diff --git a/.env.example b/.env.example index 2b32655..0aa1fc8 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,79 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env for a local synthetic environment. Never commit .env. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +DATABASE_NAME=messenger +DATABASE_USER=messenger +DATABASE_PASSWORD=replace-with-a-long-random-database-password + +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=replace-with-a-long-random-rabbitmq-password +RABBITMQ_VHOST=/ +ATTACHMENT_JOB_EXCHANGE=attachments.v1 +ATTACHMENT_JOB_QUEUE=attachments.process.level4.v1 +ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1 +ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1 +ATTACHMENT_DLQ_QUEUE=attachments.failed.v1 +ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1 +ATTACHMENT_PIPELINE_VERSION=2 +WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 +JOB_MAX_ATTEMPTS=5 +JOB_RETRY_BASE_DELAY=1s +JOB_RETRY_MAX_DELAY=30s +JOB_TIMEOUT=60s +JOB_LEASE_DURATION=90s +JOB_LEASE_REAPER_INTERVAL=5s +WORKER_MAX_IMAGE_PIXELS=40000000 +WORKER_MEMORY_BUDGET_BYTES=536870912 +WORKER_MEMORY_WAIT=2s + +# Fault injection is opt-in and intended only for an isolated test stand. +FAULT_CRASH_AFTER_OUTBOX_COMMIT=false +FAULT_CRASH_BEFORE_SIDE_EFFECT=false +FAULT_CRASH_AFTER_SIDE_EFFECT=false +FAULT_CRASH_AFTER_RESULT_COMMIT=false + S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=replace-with-a-long-random-api-secret +MINIO_ROOT_USER=minio-root +MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret + +SESSION_TTL_SECONDS=3600 +IDEMPOTENCY_TTL_SECONDS=86400 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +MAX_CONCURRENT_UPLOADS=3 +UPLOAD_PERMIT_WAIT_MILLIS=10000 + +# Staging objects are deleted only after the final result is committed. +# A separate lifecycle/cleanup policy may remove abandoned staging objects +# after this retention interval; it must never remove active jobs. STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +ATTACHMENT_CLEANUP_GRACE_SECONDS=3600 +ATTACHMENT_CLEANUP_DRY_RUN=false +ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000 +ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000 + +# Observability (lab6). The worker must set a distinct OTEL_SERVICE_NAME. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +OTEL_TRACES_SAMPLER=parentbased_traceidratio +# Balanced lab profile: keep 25% of complete parent-based traces; keep all logs/metrics. +OTEL_TRACES_SAMPLER_ARG=0.25 +OTEL_METRIC_EXPORT_INTERVAL=30000 +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512 +OTEL_BSP_EXPORT_TIMEOUT=5000 +OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=256 +OBSERVABILITY_QUEUE_POLL_INTERVAL=5s + +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=replace-with-a-long-random-grafana-password +PROMETHEUS_PORT=9090 +LOKI_PORT=3100 +TEMPO_PORT=3200 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..a07316e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,59 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: EgorAnti / БПИ2301 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка (3/4/5): 5 -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Трек 5, если заявлен: «Цена наблюдаемости» — сравнение telemetry off, full 100%/10s, economy 5%/60s и balanced 25%/30s при одинаковом профиле нагрузки. + +Что реализовано и что перенесено: + +- из lab5 перенесён и сохранён работающий минимум API/worker, PostgreSQL, Redis, RabbitMQ, MinIO, outbox/retry/DLQ и публичные тесты; +- добавлены OpenTelemetry Java agent/API для API и worker, W3C trace context через RabbitMQ и durable retry, структурные OTLP logs, metrics и реальные traces; +- в общий Compose добавлены OTel Collector с memory limiter/batch/bounded queues, Prometheus, Loki, Tempo и Grafana с persistent volumes; +- provisioned datasources, импортируемый dashboard, p50/p95/p99 из histogram, error ratio, queue depth/age, DB pool, worker saturation, SLI/SLO и alert rules; +- воспроизведены отказ worker, отказ PostgreSQL, retry через MinIO и недоступность Loki/Tempo с firing/resolved evidence; +- проведён эксперимент стоимости телеметрии, проверены ingest/CPU/RAM/latency/sampling loss и выбран balanced-профиль: 25% parent-based traces, metrics 30s, все logs. + +Команды запуска из чистого clone: + +```powershell +Copy-Item .env.example .env +# Заменить replace-with-* уникальными локальными значениями. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +$env:PYTHONUTF8 = "1" +make check PYTHON=python +make test BASE_URL=http://localhost:8080 PYTHON=python +powershell -File scripts/experiment-observability-cost.ps1 ` + -BaseUrl http://localhost:8080 -LoadSeconds 45 -Concurrency 4 +``` + +Результаты проверок и ссылка на REPORT.md: + +- `make check PYTHON=python`: `PASS: offline skeleton structure, local refs and Python syntax`; +- `make test BASE_URL=http://127.0.0.1:18080 PYTHON=python`: `PASS: lab6 public API scenario`; +- `docker compose --env-file .env config --quiet`: успешно; +- balanced работает с `OTEL_TRACES_SAMPLER_ARG=0.25`, `OTEL_METRIC_EXPORT_INTERVAL=30000`; итоговый `make test` прошёл; +- во всех четырёх cost-режимах 0 load errors и 0 Collector transport-lost spans; normalized Tempo+Loki ingest balanced — 2200.52 B/request против 6918.72 B/request в full; +- sampling-probe: 40 контролируемых HTTP 500, 8 sampled и 32 unsampled logs; sampled trace найден в Tempo (200), unsampled отсутствует (404); +- полный отчёт: [`REPORT.md`](../REPORT.md); raw evidence: [`evidence/lab6-20261119-203024-observability-cost/`](../evidence/lab6-20261119-203024-observability-cost/). + +Ограничения / известные ошибки: + +- стенд локальный и однорепличный, telemetry retention — 72 часа/3 дня; +- cost experiment — один последовательный closed-loop прогон на Docker Desktop; абсолютные latency/CPU требуют повторов и confidence intervals перед production sizing; +- head sampling принимает решение до outcome/duration: balanced может не сохранить около 75% единичных ошибок и редких slow traces, однако их logs и metric aggregates сохраняются; +- на Windows перед `make check` нужен `PYTHONUTF8=1`, иначе Python с CP1251 не читает UTF-8 OpenAPI JSON. + +Источники и использование ИИ: + +- использованы официальные спецификации/документация OpenTelemetry, W3C Trace Context, Prometheus histograms, Grafana provisioning, Loki и Tempo; ссылки зафиксированы в `contracts/observability.md` и `REPORT.md`; +- OpenAI Codex использовался для переноса lab5, реализации конфигурации/инструментации, подготовки воспроизводимых scripts, запуска проверок и анализа evidence; итог подтверждён командами и raw-артефактами выше. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a7189..affc550 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +.gradle/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a48ffd3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# Stage 1: Build +FROM eclipse-temurin:23-jdk AS build + +WORKDIR /app + +COPY gradlew . +COPY gradle gradle +COPY build.gradle settings.gradle ./ + +COPY src src + +RUN --mount=type=cache,target=/root/.gradle \ + chmod +x gradlew && ./gradlew bootJar --no-daemon + + +# Stage 2: Runtime +FROM eclipse-temurin:23-jre + +WORKDIR /app + +COPY --from=build /app/build/libs/*.jar app.jar + +ARG OTEL_JAVA_AGENT_VERSION=2.33.1 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends curl \ + && curl --fail --location --silent --show-error \ + "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_JAVA_AGENT_VERSION}/opentelemetry-javaagent.jar" \ + --output /opt/opentelemetry-javaagent.jar \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --create-home messenger + +COPY --chmod=0755 infra/otel/docker-entrypoint.sh /usr/local/bin/messenger-entrypoint + +USER messenger + +EXPOSE 8080 + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD curl --fail --silent --show-error http://localhost:8080/health/ready || exit 1 + +ENTRYPOINT ["messenger-entrypoint"] diff --git a/REPORT.md b/REPORT.md index b97ae42..b3ecf95 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,169 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: EgorAnti / БПИ2301 +- Целевая оценка: 5 (трек «Цена наблюдаемости») +- Commit/PR: заполнить после публикации +- Реализация: OpenTelemetry Java agent/API, OTel Collector, Prometheus, Loki, Tempo, Grafana ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker с Compose v2, Java 23 только для локальной Gradle-сборки и Python 3.9+ для публичных проверок. -## Архитектура и решения +```powershell +Copy-Item .env.example .env +# Заменить все replace-with-* уникальными локальными значениями. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +docker compose ps +python tests/smoke.py --base-url http://localhost:8080 +python tests/load.py --base-url http://localhost:8080 --duration 60 --concurrency 8 ` + --output evidence/lab6-http-load.json +python tests/attachment_load.py --base-url http://localhost:8080 --count 20 ` + --output evidence/lab6-attachment-load.json +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Grafana: `http://localhost:3000`; Prometheus: `http://localhost:9090`. Учётные данные Grafana берутся только из `.env`. Безопасная остановка сохраняет volumes: `docker compose down`. Намеренное удаление учебных данных: `docker compose down --volumes`. + +## Архитектура и границы + +API и worker запускаются из одного образа, но с разными `MESSENGER_ROLE` и `OTEL_SERVICE_NAME`. В образ встроен закреплённый Java agent 2.33.1. Он создаёт реальные server/client/JDBC/RabbitMQ producer/consumer spans, переносит W3C `traceparent`/`tracestate` в AMQP headers, экспортирует runtime/HTTP/DB metrics и связывает logs с активным span. Для durable retry контекст сохраняется в `attachment_jobs.trace_parent/trace_state` (Flyway V8) и восстанавливается перед повторным publish: retry producer становится потомком предыдущего consumer span. Собственные метрики worker создаются через стабильный OTel API 1.67.0. + +Collector принимает OTLP/HTTP и OTLP/gRPC. Перед экспортом применяются memory limiter, удаление потенциально чувствительных attributes и batch. Metrics доступны Prometheus через Collector exporter, logs отправляются OTLP в Loki, traces — OTLP в Tempo. Grafana provisioned из Git и не является хранилищем. + +| Компонент | Назначение | +| --- | --- | +| Prometheus | Хранит числовые временные ряды и вычисляет rate, error ratio, histogram percentiles и alert expressions. | +| Loki | Хранит структурные API/worker logs и позволяет найти событие по `service_name`, `event` и `trace_id`. | +| Tempo | Хранит распределённые traces/spans и показывает причинную цепочку HTTP → broker → worker → DB/S3. | +| Grafana | Запрашивает три источника, связывает log ↔ trace и отображает dashboard; сама не заменяет эти хранилища. | + +Collector имеет лимит 384 MiB, batch 512 и exporter queues по 2048 элементов. Retry ограничен 60 секундами. SDK использует bounded span queue 2048, batch 512 и export timeout 5 секунд. Поэтому отказ telemetry backend не блокирует business threads и не вызывает неограниченный рост RAM; после исчерпания буферов допустима измеримая потеря telemetry. + +## Соответствие сигналов + +| Контракт | Реальный сигнал / запрос | +| --- | --- | +| HTTP counter | `http_server_request_duration_seconds_count`, labels `http_request_method`, `http_route`, `http_response_status_code` | +| HTTP duration | `http_server_request_duration_seconds_bucket`; p50/p95/p99 через `histogram_quantile` по сумме buckets реплик | +| Jobs processed/failed | `messenger_jobs_processed_total`, labels только `job_type` и `outcome` | +| Job duration | `messenger_job_processing_duration_seconds_bucket` | +| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket` | +| Queue | `messenger_queue_depth`, `messenger_queue_oldest_job_age_seconds`; источник — durable DB queue | +| Worker saturation | `messenger_worker_busy / messenger_worker_capacity` | +| DB pool/runtime | auto-instrumentation `db_client_connections_*`, `jvm_*`, `process_*` | +| Logs | OTLP logs в Loki с `service_name`, `severity`, `event`, `trace_id`, `span_id`, `request_id` | +| Traces | Tempo: servlet server, RabbitMQ producer/consumer, JDBC/Redis/S3 client spans | + +`user_id`, `chat_id`, `message_id`, `attachment_id`, request ID, filename и raw URL не используются как metric labels. Authorization/session tokens, bodies, изображения, S3 credentials и signed URLs не захватываются. Collector дополнительно удаляет известные sensitive keys. Технические job/attachment UUID также удалены из новых structured worker logs. + +## Dashboard: запросы и единицы + +| Панель | PromQL | Единица | +| --- | --- | --- | +| API RPS | `sum by (http_request_method) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m]))` | requests/s | +| API 5xx ratio | `(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5..",http_route!~"/health/.*"}[5m])) or vector(0)) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m])), 0.001)` | ratio 0..1, в Grafana показывается как %; без 5xx возвращает 0, а не `No data` | +| API p50/p95/p99 | `histogram_quantile(Q, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api",http_route!~"/health/.*"}[5m])))`, где `Q=0.50/0.95/0.99` | seconds | +| Queue depth | `max(messenger_queue_depth{service_name="messenger-api"})` | jobs (count) | +| Oldest job age | `max(messenger_queue_oldest_job_age_seconds{service_name="messenger-api"})` | seconds | +| DB pool | `sum by (service_name, state) (db_client_connections_usage{service_name=~"messenger-(api|worker)"})` и `sum by (service_name) (db_client_connections_pending_requests{service_name=~"messenger-(api|worker)"})` | connections/requests (count) | +| Worker saturation | `messenger_worker_busy / clamp_min(messenger_worker_capacity, 1)` с фильтром `service_name="messenger-worker"` | ratio 0..1, в Grafana показывается как % | + +Percentile вычисляются из cumulative histogram buckets: сначала `rate(..._bucket[5m])`, затем `sum by (le)` объединяет buckets всех API-реплик, и только после этого применяется `histogram_quantile`. Готовые p50/p95/p99 отдельных реплик не вычисляются и не усредняются. + +## SLI/SLO и alerts + +API availability SLI: число 5xx business responses / все business responses за 5 минут; health routes исключены, 4xx не считаются отказом сервера. Учебный SLO — не менее 99% успешных business responses. Warning срабатывает при error ratio >5% в течение 2 минут. + +Attachment readiness SLI: p95 времени от durable acceptance до `ready` за 10 минут. Учебная цель — p95 <60 секунд. Alert `AttachmentQueueLagHigh` срабатывает, если возраст старейшей незавершённой job >60 секунд ещё 1 минуту. Есть отдельные alerts на worker saturation и потерю Collector export. + +| Alert | Условие и окно | `for` | Проверка | +| --- | --- | --- | --- | +| `MessengerApiHighErrorRate` | 5xx / все business requests за 5m >5%; health исключены | 2m | PostgreSQL stop: firing value 0.964; после восстановления ratio 0 и alert resolved | +| `AttachmentQueueLagHigh` | oldest durable job age >60s | 1m | worker stop: depth 3, age 138s, firing; после drain depth/age 0 и alert resolved | +| `AttachmentWorkerSaturated` | busy / capacity ≥100% | 2m | правило загружено; runbook сохранён | +| `TelemetryExportFailures` | скорость failed trace/log exports >0 за 5m | 1m | правило загружено; bounded exporter runbook сохранён | + +## Воспроизводимые инциденты и доказательства + +### 1. Worker остановлен + +```powershell +powershell -File scripts/incident-worker-stop.ps1 -DurationSeconds 150 -JobCount 10 +``` + +Скрипт останавливает только worker, создаёт синтетические attachment jobs, ждёт firing queue-lag alert, сохраняет Prometheus values/alerts, запускает worker и сохраняет состояние после drain. Результат помещается в `evidence/lab6--worker-stop/` и не содержит credentials. + +Проверено повторно 19.11.2026: API принял 3/3 uploads при остановленном worker; `queue_depth=3`, `oldest_job_age=138`, alert `AttachmentQueueLagHigh` был `firing`. После восстановления `queue_depth=0`, `oldest_job_age=0`, active alert отсутствует (`alert_resolved=true`). Raw evidence: `evidence/lab6-20261119-192857-worker-stop/`. + +### 2. PostgreSQL недоступен + +```powershell +powershell -File scripts/incident-database-stop.ps1 -DurationSeconds 165 +``` + +Скрипт ограниченно останавливает PostgreSQL, опрашивает readiness и business endpoint, сохраняет status/latency, Prometheus, Loki и Tempo snapshots, затем обязательно запускает БД в `finally` и проверяет восстановление. + +Проверено повторно 19.11.2026: 33 readiness/business probes сформировали контролируемые 503; `MessengerApiHighErrorRate` перешёл в `firing` со значением 0.964. Loki/Tempo сохранили JDBC connection failures и error spans. После запуска PostgreSQL readiness восстановился, пятиминутное error ratio стало 0, active alert исчез (`alert_resolved=true`). Raw evidence: `evidence/lab6-20261119-193359-database-stop/`. + +### 3. Отказ telemetry backend + +```powershell +powershell -File scripts/incident-telemetry-backend.ps1 -BaseUrl http://localhost:8080 +``` + +Проверено 19.11.2026 остановкой Loki и Tempo при работающих Collector/Prometheus. `make test` прошёл; за 75 секунд выполнено 3060/3060 успешных POST messages, ошибок 0. По 14 samples максимум RSS Collector составил 173644186 bytes (165.6 MiB) при container limit 512 MiB и memory limiter 384 MiB. Максимальная exporter queue — 66 из capacity 2048. После 60-секундного retry budget Collector учёл 3364 failed spans и 926 failed log records: это измеримая ограниченная потеря, а не блокировка business threads или бесконечный рост памяти. После запуска backend обе `/ready` вернули 200, queue size вернулся к 0. Raw evidence: `evidence/lab6-20261119-194742-telemetry-backend/`. + +Поведение ограничено на двух уровнях: SDK имеет BSP queue 2048/batch 512/export timeout 5s; Collector имеет memory limiter 384 MiB, batch 512, exporter queue 2048 и retry не дольше 60s. Когда backend не возвращается в budget, элементы отбрасываются с ростом `otelcol_exporter_send_failed_*`; сообщения приложения при этом продолжают фиксироваться в PostgreSQL/RabbitMQ. + +### Сквозной retry trace + +```powershell +powershell -File scripts/incident-retry-trace.ps1 -BaseUrl http://localhost:8080 +``` + +Скрипт принимает один синтетический upload при остановленном worker, кратко делает MinIO недоступным для первой обработки, автоматически восстанавливает сервисы и проверяет parent-цепочку в Tempo/Loki. Проверено 19.11.2026: trace `a3dc1f8fabc7dc732fc3f502007f7654` содержит 3 producer и 3 consumer spans. Каждый retry consumer имеет retry producer как parent; финальная попытка содержит S3 PUT/DELETE и JDBC result commit. Raw evidence: `evidence/lab6-20261119-191738-retry-trace/`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда | Артефакт | +| --- | --- | --- | +| Compose воспроизводим | `docker compose --env-file .env config --quiet` | вывод команды | +| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8080` | вывод команды | +| Metrics/logs/traces | HTTP и attachment load; dashboard `messenger-observability` | `evidence/README.md` и timestamped raw evidence | +| API → queue → worker trace + retry | `scripts/incident-retry-trace.ps1` | trace `a3dc1f8fabc7dc732fc3f502007f7654`, raw summary в `evidence/lab6-20261119-191738-retry-trace/` | +| Worker failure/recovery | `scripts/incident-worker-stop.ps1` | timestamped evidence directory | +| DB failure/recovery | `scripts/incident-database-stop.ps1` | timestamped evidence directory | +| Importable dashboard | автоматический provisioning | `infra/otel/grafana/dashboards/messenger-overview.json` | +| Alert rules/runbooks | Prometheus rule load | `infra/otel/prometheus-alerts.yaml`, `infra/otel/runbooks.md` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Цена наблюдаемости и выбранный профиль -## Эксперименты и отказы +Воспроизводимый сценарий `scripts/experiment-observability-cost.ps1` последовательно пересоздаёт только API и worker, делает одинаковый 10-секундный прогрев и запускает для каждого режима один и тот же closed-loop профиль: 45 секунд, concurrency 4, POST messages, без клиентских retry. JVM в каждом режиме холодная; БД, broker, Collector и хранилища общие. Ingest сравнивается как дельта монотонных счётчиков и дополнительно нормализуется на успешный запрос. CPU — среднее `docker stats` за измеряемую фазу, RAM — peak RSS. Это локальный сравнительный опыт, а не capacity benchmark: closed-loop имеет coordinated omission, а один прогон не позволяет ранжировать близкие sampling-профили по latency. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверенный запуск: `evidence/lab6-20261119-203024-observability-cost/`. Во всех режимах ошибок нагрузки было 0, а Collector transport loss был 0 spans. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Режим | Traces / metrics | RPS | p50 / p95 / p99, ms | API CPU avg / RAM peak | Collector CPU avg / RAM peak | +| --- | --- | ---: | ---: | ---: | ---: | +| off | agent/SDK выключен | 40.55 | 97 / 198 / 275 | 66.50% / 363.0 MiB | 0.05% / 144.5 MiB | +| full | 100% / 10s | 22.42 | 185 / 315 / 406 | 69.18% / 405.7 MiB | 2.17% / 150.2 MiB | +| economy | 5% / 60s | 14.62 | 287 / 406 / 502 | 68.56% / 411.8 MiB | 0.88% / 152.2 MiB | +| balanced | 25% / 30s | 16.93 | 210 / 391 / 409 | 67.62% / 406.6 MiB | 0.54% / 148.5 MiB | -## Ограничения и следующие шаги +Одинаковая concurrency насыщала лимит API, поэтому CPU близок, а цена видна в throughput/latency: относительно off full дал -44.7% RPS и +59.1% p95; balanced — -58.2% RPS и +97.6% p95. Немонотонность full/economy/balanced означает, что основная application-side цена — сам agent, создание metrics/logs/context и экспорт, а не только число сохранённых spans; различия одного локального прогона нельзя приписывать sampler без повторов. Память API с agent выросла примерно на 43–49 MiB, worker — на 81–109 MiB. Sampling прежде всего снижает Collector/backend cost. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +| Режим | Accepted spans / logs / metric points | Нормализованное сохранение traces | Tempo+Loki bytes / успешный запрос | Потеря Collector | +| --- | ---: | ---: | ---: | ---: | +| off | 0 / 0 / 0 | 0% | 0 B | 0 spans | +| full | 12346 / 1019 / 711 | 100% | 6918.72 B | 0 spans | +| economy | 447 / 671 / 115 | 5.52% | 1108.25 B | 0 spans | +| balanced | 2200 / 775 / 238 | 23.48% | 2200.52 B | 0 spans | + +Нормализованный balanced ingest trace+log оказался на 68.2% ниже full. Логи не sample-ятся, поэтому создают полезный нижний предел ingest. Metrics — cumulative counters/histograms; интервал 30 секунд уменьшает число export points, но не превращает percentile в среднее и остаётся короче минутного queue-lag alert window. Интервал 60 секунд откладывает свежесть почти на весь alert window и вместе с 5% traces слишком сильно ухудшает диагностику. + +`parentbased_traceidratio` принимает решение в начале root trace, до знания HTTP status и duration. Поэтому при balanced один отдельный 5xx или редкий медленный запрос имеет примерно 75% шанс не попасть в Tempo; при economy — примерно 95%. Если таких событий `k`, вероятность не сохранить ни одного при balanced равна `0.75^k`. Проверочная серия 40 контролируемых 500 responses дала 8 sampled и 32 unsampled logs: sampled trace `00d814f7907510c8066b49c89cefd0ec` найден в Tempo (HTTP 200), unsampled `495125068d65af4c622b34ada5196db5` отсутствовал (HTTP 404). При этом структурный log есть для всех 40, а metrics продолжают считать status и histogram duration. + +Выбран balanced профиль: parent-based 25% traces, metrics export 30s, logs без sampling. Он сохраняет целую API → broker → worker parent-цепочку для выбранного trace и уменьшает storage ingest, не задерживая minute-scale alerts. Tail sampling мог бы гарантированно оставлять error/slow traces, но для много минут живущих async retry traces потребовал бы большого decision window и памяти Collector; короткое окно разрезало бы цепочку. Поэтому здесь риск редких пропусков принят явно, а симптомы остаются в несэмплируемых logs/metrics. Для production следующий шаг — отдельные bounded tail policies либо error-triggered trace retention с проверкой длинных queue traces. + +## Ограничения + +Локальный стенд однорепличный для telemetry storage и не является production HA. Retention — 72 часа/3 дня. Queue age берётся из durable DB job queue, а не peek первого RabbitMQ message; это сохраняет измеримость при остановленном worker и соответствует outbox как источнику истины. Cost experiment использует один последовательный closed-loop прогон на общем Docker Desktop host; абсолютные CPU/latency и немонотонные различия sampling-профилей требуют повторов и confidence intervals перед production sizing. Runtime evidence в `evidence/` получено на локальном Compose 19.11.2026; для защиты можно дополнительно приложить screenshots тех же запросов в Grafana Explore. diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..a45c46b --- /dev/null +++ b/build.gradle @@ -0,0 +1,61 @@ +plugins { + id 'java' + id 'org.springframework.boot' version '4.1.1' + id 'io.spring.dependency-management' version '1.1.7' +} + +group = 'com.example' +version = '0.0.1-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(23) + } +} + +repositories { + mavenCentral() +} + +dependencies { +// implementation 'org.springframework.boot:spring-boot-starter-data-jpa' +// implementation 'org.springframework.boot:spring-boot-starter-security' + implementation 'org.springframework.boot:spring-boot-starter-webmvc' + implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-jdbc' + implementation 'org.springframework.boot:spring-boot-starter-data-redis' + implementation 'org.springframework.boot:spring-boot-starter-amqp' + implementation 'org.springframework.boot:spring-boot-flyway' + implementation 'org.flywaydb:flyway-core' + implementation 'org.flywaydb:flyway-database-postgresql' + implementation 'org.springframework.boot:spring-boot-starter-security' + implementation 'org.bouncycastle:bcprov-jdk18on:1.84' + implementation 'org.apache.commons:commons-pool2' + implementation 'io.minio:minio:8.6.0' + implementation 'net.coobird:thumbnailator:0.4.20' + implementation 'org.sejda.imageio:webp-imageio:0.1.6' + // The runtime SDK is supplied by the pinned OpenTelemetry Java agent. + // The API dependency keeps custom business metrics vendor-neutral. + implementation 'io.opentelemetry:opentelemetry-api:1.67.0' + runtimeOnly 'org.postgresql:postgresql' + + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.3' + compileOnly 'org.projectlombok:lombok' + developmentOnly 'org.springframework.boot:spring-boot-devtools' + annotationProcessor 'org.projectlombok:lombok' +// testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test' +// testImplementation 'org.springframework.boot:spring-boot-starter-security-test' + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.springframework.boot:spring-boot-starter-webmvc-test' + testCompileOnly 'org.projectlombok:lombok' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testAnnotationProcessor 'org.projectlombok:lombok' + + implementation("com.github.f4b6a3:uuid-creator:6.1.1") + +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..56e4284 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,426 @@ +name: hla-lab6 + +x-api-environment: &api-environment + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: otlp + OTEL_LOGS_EXPORTER: otlp + OTEL_PROPAGATORS: tracecontext,baggage + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-0.25} + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-30000} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-5000} + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT: ${OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT:-256} + OTEL_RESOURCE_ATTRIBUTES: service.version=0.0.1,deployment.environment.name=${APP_ENV:-lab} + OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_MDC_ATTRIBUTES_INCLUDED: request_id + OTEL_INSTRUMENTATION_COMMON_LOGGING_STRUCTURED_ATTRIBUTES_INCLUDED: event,job.type,outcome,attempt,error.code,http.request.method,http.route,http.response.status_code + OTEL_INSTRUMENTATION_HTTP_SERVER_CAPTURE_REQUEST_HEADERS: "" + OTEL_INSTRUMENTATION_HTTP_SERVER_CAPTURE_RESPONSE_HEADERS: "" + DATABASE_URL: jdbc:postgresql://postgres:5432/${DATABASE_NAME:-messenger} + DATABASE_USER: ${DATABASE_USER:-messenger} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env} + DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10} + DATABASE_MIN_IDLE: ${DATABASE_MIN_IDLE:-2} + DATABASE_QUERY_TIMEOUT_SECONDS: ${DATABASE_QUERY_TIMEOUT_SECONDS:-5} + DATABASE_TRANSACTION_TIMEOUT_SECONDS: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:-5} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_CONNECT_TIMEOUT: ${REDIS_CONNECT_TIMEOUT:-1s} + REDIS_COMMAND_TIMEOUT: ${REDIS_COMMAND_TIMEOUT:-500ms} + REDIS_POOL_MAX_ACTIVE: ${REDIS_POOL_MAX_ACTIVE:-8} + REDIS_POOL_MAX_IDLE: ${REDIS_POOL_MAX_IDLE:-8} + REDIS_POOL_MIN_IDLE: ${REDIS_POOL_MIN_IDLE:-0} + REDIS_POOL_MAX_WAIT: ${REDIS_POOL_MAX_WAIT:-500ms} + USER_CACHE_TTL_SECONDS: ${USER_CACHE_TTL_SECONDS:-300} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: http://minio:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3} + UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000} + RABBITMQ_HOST: rabbitmq + RABBITMQ_PORT: 5672 + RABBITMQ_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + RABBITMQ_VHOST: ${RABBITMQ_VHOST:-/} + ATTACHMENT_JOB_EXCHANGE: ${ATTACHMENT_JOB_EXCHANGE:-attachments.v1} + ATTACHMENT_JOB_QUEUE: ${ATTACHMENT_JOB_QUEUE:-attachments.process.level4.v1} + ATTACHMENT_JOB_ROUTING_KEY: ${ATTACHMENT_JOB_ROUTING_KEY:-attachment.process.v1} + ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1} + ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1} + ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1} + ATTACHMENT_PIPELINE_VERSION: ${ATTACHMENT_PIPELINE_VERSION:-2} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_RETRY_BASE_DELAY: ${JOB_RETRY_BASE_DELAY:-1s} + JOB_RETRY_MAX_DELAY: ${JOB_RETRY_MAX_DELAY:-30s} + JOB_TIMEOUT: ${JOB_TIMEOUT:-60s} + JOB_LEASE_DURATION: ${JOB_LEASE_DURATION:-90s} + JOB_LEASE_REAPER_INTERVAL: ${JOB_LEASE_REAPER_INTERVAL:-5s} + WORKER_MAX_IMAGE_PIXELS: ${WORKER_MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_BUDGET_BYTES: ${WORKER_MEMORY_BUDGET_BYTES:-536870912} + WORKER_MEMORY_WAIT: ${WORKER_MEMORY_WAIT:-2s} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} + ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false} + ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000} + ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:-60000} + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_DB: ${DATABASE_NAME:-messenger} + POSTGRES_USER: ${DATABASE_USER:-messenger} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + cpus: 1.0 + mem_limit: 768m + networks: [backend] + + redis: + image: redis:7.4-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + cpus: 0.25 + mem_limit: 256m + networks: [backend] + + rabbitmq: + image: rabbitmq:4.1.4-management-alpine + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/} + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2" + volumes: + - rabbitmq-data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 10s + start_period: 60s + retries: 12 + restart: unless-stopped + cpus: 0.5 + mem_limit: 1024m + networks: [backend] + + minio: + image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + volumes: + - minio-data:/bitnami/minio/data + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:9000/minio/health/live"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + cpus: 0.75 + mem_limit: 512m + networks: [backend] + + minio-init: + image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc + depends_on: + minio: + condition: service_healthy + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + volumes: + - ./infra/minio-api-policy.json:/policy/minio-api-policy.json:ro + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json + attempt=0 + until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" \ + && mc mb --ignore-existing "local/$$S3_BUCKET" \ + && mc anonymous set none "local/$$S3_BUCKET" \ + && (mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" \ + || mc admin user info local "$$S3_ACCESS_KEY_ID") \ + && (mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json \ + || mc admin policy info local messenger-attachments-rw) \ + && mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID" + do + attempt=$$((attempt + 1)) + if [ "$$attempt" -ge 30 ]; then + echo "MinIO bootstrap did not complete after 30 attempts" >&2 + exit 1 + fi + sleep 2 + done + restart: "no" + networks: [backend] + + private-bucket-check: + profiles: ["verification"] + image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc + depends_on: + minio-init: + condition: service_completed_successfully + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set anonymous http://minio:9000 + if mc ls "anonymous/$$S3_BUCKET" >/dev/null 2>&1; then + echo "FAIL: anonymous bucket listing succeeded" >&2 + exit 1 + fi + mc alias set restricted http://minio:9000 "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + mc ls "restricted/$$S3_BUCKET" >/dev/null + if mc admin info restricted >/dev/null 2>&1; then + echo "FAIL: API credentials have administrative access" >&2 + exit 1 + fi + echo "PASS: bucket is private and API credentials are non-admin" + restart: "no" + networks: [backend] + + # One-shot Flyway runner. It is invoked explicitly before API replicas. + migrate: + image: hla-lab6-app:local + build: + context: . + environment: + <<: *api-environment + FLYWAY_ENABLED: "true" + MIGRATE_ONLY: "true" + MESSENGER_ROLE: migration + SPRING_MAIN_WEB_APPLICATION_TYPE: none + OTEL_SDK_DISABLED: "true" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy + restart: "no" + cpus: 0.75 + mem_limit: 768m + networks: [backend] + + api: + image: hla-lab6-app:local + build: + context: . + environment: + <<: *api-environment + FLYWAY_ENABLED: "false" + MESSENGER_ROLE: api + OTEL_SERVICE_NAME: messenger-api + FAULT_CRASH_AFTER_OUTBOX_COMMIT: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:-false} + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 180s + init: true + stop_grace_period: 12s + restart: unless-stopped + cpus: 0.75 + mem_limit: 768m + networks: [backend] + + worker: + image: hla-lab6-app:local + environment: + <<: *api-environment + FLYWAY_ENABLED: "false" + MESSENGER_ROLE: worker + OTEL_SERVICE_NAME: messenger-worker + SPRING_MAIN_WEB_APPLICATION_TYPE: none + JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70 + FAULT_CRASH_BEFORE_SIDE_EFFECT: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:-false} + FAULT_CRASH_AFTER_SIDE_EFFECT: ${FAULT_CRASH_AFTER_SIDE_EFFECT:-false} + FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false} + depends_on: + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: + disable: true + init: true + stop_grace_period: 20s + restart: unless-stopped + cpus: 1.0 + mem_limit: 1024m + networks: [backend] + + nginx: + image: nginx:1.27.4-bookworm + ports: + - "127.0.0.1:${APP_PORT:-8080}:80" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 2s + timeout: 2s + retries: 15 + stop_grace_period: 10s + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + networks: [backend] + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.165.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + networks: [backend] + + prometheus: + image: prom/prometheus:v3.14.0 + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=3d + - --web.enable-lifecycle + volumes: + - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/prometheus-alerts.yaml:/etc/prometheus/prometheus-alerts.yaml:ro + - prometheus-data:/prometheus + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + networks: [backend] + + loki: + image: grafana/loki:3.7.8 + command: ["-config.file=/etc/loki/local-config.yaml"] + ports: + - "127.0.0.1:${LOKI_PORT:-3100}:3100" + volumes: + - ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro + - loki-data:/loki + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + networks: [backend] + + tempo: + image: grafana/tempo:3.1.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + ports: + - "127.0.0.1:${TEMPO_PORT:-3200}:3200" + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo-data:/var/tempo + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + networks: [backend] + + grafana: + image: grafana/grafana:13.3.0 + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_UNIFIED_ALERTING_ENABLED: "true" + volumes: + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + networks: [backend] + +networks: + backend: + driver: bridge + +volumes: + postgres-data: + redis-data: + minio-data: + rabbitmq-data: + prometheus-data: + loki-data: + tempo-data: + grafana-data: diff --git a/evidence/README.md b/evidence/README.md index 8df7001..646cb21 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,82 @@ -# Доказательства +# Проверенные наблюдения lab6 — 19.11.2026 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Все данные ниже получены из запущенного Compose-стека, а не из предположений. Нагрузка использовала только синтетические PNG и пользователей; credentials и bodies в evidence не записывались. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +Grafana 13.3.0 отдельно прошла runtime provisioning check: `/api/health` вернул `database=ok`, API перечислил datasources `Loki`, `Prometheus`, `Tempo` и dashboard `Messenger observability`. + +`make test BASE_URL=http://127.0.0.1:18080 PYTHON=python` выполнен с включённым Java agent и работающим Collector; публичный сценарий завершился `PASS`. + +Свежий trace этого запуска `128cf1e355c405bc421ef4e0b5daa0e0` найден одновременно в Tempo и Loki. API log `http_request_completed` содержит route template `/api/v1/chats/{chatId}/attachments`, status 202 и `span_id=cd375b5c0e8ea3ab`; worker log `attachment_job_completed` содержит outcome `success` и `span_id=10bac10a3d80b9a6`. В Tempo producer `attachments.v1 publish` является родителем consumer `attachment.process.v1 process`; в той же trace присутствуют JDBC и S3 PUT/DELETE spans обоих сервисов. + +## Сквозная корреляция + +Smoke прошёл за 8.35 с после прогрева JVM. Для attachment upload Tempo вернул trace `376c8caf07126a6950cf8c826a9d1b6e` с одной цепочкой: + +`messenger-api POST /api/v1/chats/{chatId}/attachments` → RabbitMQ producer → `messenger-worker attachment.process.v1 process` → JDBC/S3 child spans. + +Loki вернул structured event `attachment_job_completed` с тем же `trace_id`, `span_id=a8a1bffac4fb70f1`, `service_name=messenger-worker`, `outcome=success`. Collector internal metrics на момент проверки: 1830 accepted spans и 55 accepted log records. В metric series использовались route templates и low-cardinality labels; UUID, request ID, filename и raw URL отсутствовали в labels. + +## Отказ 1: worker остановлен + +Raw files: `lab6-20261119-131448-worker-stop/`. + +- workload: 5 requested, 5 accepted, 0 errors; +- failure: queue depth 5, oldest age 139 s, `AttachmentQueueLagHigh` firing; +- recovery: queue depth 0, oldest age 0, successful jobs снова увеличиваются. + +Причина по сохранённым наблюдениям: API продолжал принимать задачи, но consumer отсутствовал; росли только durable queue depth/age. После возврата consumer backlog исчез. + +## Отказ 2: PostgreSQL остановлен + +Raw files: `lab6-20261119-131929-database-stop/`. + +- семь readiness/business probes подряд вернули HTTP 503; +- цикл probe занимал 3988.8–4537.5 ms; +- Prometheus сохранил ненулевую скорость 5xx; +- Loki сохранил `Queue telemetry sample failed: CannotGetJdbcConnectionException` и `HikariPool-1 - Failed to validate connection ... This connection has been closed`, включая коррелируемый trace `7b47023dcfebb5738de24578b483931a`; +- Tempo TraceQL `{ span.db.system = "postgresql" && status = error }` вернул PostgreSQL error spans; trace `7b47023dcfebb5738de24578b483931a` длился 5030 ms. Запрос `{ name = "GET /health/ready" && status = error }` вернул 5-секундные error traces; +- после запуска PostgreSQL readiness снова прошёл (`recovery.txt`). + +Причина по сохранённым наблюдениям: задержка находилась в PostgreSQL/JDBC connection path, а не в CPU-работе HTTP handler. + +## W3C context через durable retry + +Raw files: `lab6-20261119-191738-retry-trace/`. + +Один синтетический upload был принят при остановленном worker, затем MinIO кратко останавливался. Trace `a3dc1f8fabc7dc732fc3f502007f7654` содержит 3 producer и 3 consumer spans в одной trace: + +`upload server` → `initial publish` → `consumer #1` → `retry publish` → `consumer #2` → `retry publish` → `consumer #3` → `S3 PUT/DELETE + DB commit`. + +Parent IDs в `retry-trace-summary.json` показывают, что каждый consumer является ребёнком соответствующего producer, а каждый retry producer — ребёнком предыдущего consumer. Loki в этой же trace содержит `attachment_job_retry_scheduled` (`attempt=2`, `error.code=processing_failed`) и финальный `attachment_job_completed` (`outcome=success`). + +## Dashboard и PromQL + +Grafana runtime API загрузил dashboard version 3. Все 13 Prometheus targets dashboard были выполнены через `/api/v1/query` со статусом `success`: RPS, error ratio, p50/p95/p99, job outcomes/duration, time-to-ready, queue depth/age, worker saturation и DB pool. В здоровом состоянии error ratio возвращает одну series со значением `0`, а не `No data`. + +Панель API latency использует `histogram_quantile` после `sum by (le)` над `rate(..._bucket[5m])`; percentile реплик не усредняются. Grafana units проверены через API: latency/oldest age — seconds, error ratio/worker saturation — percent, queue depth и DB pool — count. Для совмещённой queue-панели depth использует левую ось, age — правую. + +## Alert lifecycle: firing → resolved + +`AttachmentQueueLagHigh`: raw files `lab6-20261119-192857-worker-stop/`. При остановленном worker API принял 3/3 jobs, depth был 3, oldest age 138s, состояние alert — `firing`. После запуска worker depth/age стали 0, в recovery snapshot active alert отсутствует и `alert_resolved=true`. + +`MessengerApiHighErrorRate`: raw files `lab6-20261119-193359-database-stop/`. При остановленном PostgreSQL выполнено 33 контролируемых readiness/business probe; alert достиг `firing` со значением 0.964. После восстановления БД readiness стал успешным, error ratio вернулся к 0, active alert отсутствует и `alert_resolved=true`. + +Внешняя доставка намеренно не настраивалась: критерий проверяется локальными Prometheus rule states и timestamped JSON snapshots. + +## Отказ telemetry backend + +Raw files: `lab6-20261119-194742-telemetry-backend/`. + +Loki и Tempo были остановлены при работающих Collector/Prometheus. `make test` завершился `PASS`; 75-секундная нагрузка записала 3060/3060 сообщений без ошибок. В 14 последовательных samples Collector достиг максимум 173644186 bytes (165.6 MiB) RSS при лимитах 384 MiB memory limiter / 512 MiB container. Exporter queue достигала 66 из 2048. + +После исчерпания ограниченного retry budget счётчики показали 3364 failed spans и 926 failed log records. Это ожидаемая измеримая потеря telemetry: business traffic не блокировался, RAM и очереди оставались ограниченными. После восстановления Loki/Tempo оба `/ready` вернули 200, exporter queue стала 0. Воспроизводимый сценарий: `scripts/incident-telemetry-backend.ps1`. + +## Цена наблюдаемости + +Raw files: `lab6-20261119-203024-observability-cost/`; сценарий: `scripts/experiment-observability-cost.ps1`. + +При одинаковых duration 45s / concurrency 4 / workload профиль off показал p95 198.133ms и 40.547 RPS; full (100% traces, metrics 10s) — 315.289ms и 22.419 RPS; economy (5%/60s) — 405.639ms и 14.622 RPS; balanced (25%/30s) — 391.490ms и 16.934 RPS. Все четыре режима завершились без ошибок. CPU/RAM взяты из raw `*-resource-samples.json`; итоговая таблица находится в `comparison.csv`/`summary.json`. + +Normalized trace retention составил 5.52% для economy и 23.48% для balanced. Tempo+Loki ingest на успешный request: full 6918.72B, economy 1108.25B, balanced 2200.52B. Во всех режимах дельта Collector failed/enqueue-failed/refused была 0: недостающие spans — результат решения SDK sampler, а не транспортная потеря. + +На balanced выполнено 40 контролируемых 500 requests: logs содержали 8 sampled и 32 unsampled trace IDs. Sampled trace вернул Tempo HTTP 200, unsampled — 404. Это прямое подтверждение, что head sampler принимает решение до outcome и может потерять отдельную ошибку или редкий slow trace; logs/metrics при этом остаются. После эксперимента `make test` прошёл. Выбран профиль 25% traces / metrics 30s / все logs; детали компромисса и ограничения одного локального прогона описаны в `REPORT.md`. diff --git a/evidence/failover-each-2026-11-19.json b/evidence/failover-each-2026-11-19.json new file mode 100644 index 0000000..a8241bd --- /dev/null +++ b/evidence/failover-each-2026-11-19.json @@ -0,0 +1,24 @@ +{ + "scenario": "sequential failover of each API replica", + "replicas": 2, + "results": [ + { + "target_instance_id": "8e081627dfc3", + "window_seconds": 15.775, + "successful_reads": 3, + "successful_writes": 4, + "failed_attempts": 0, + "successful_rps": 0.444, + "first_recovery_seconds": 5.018 + }, + { + "target_instance_id": "95b5db854bee", + "window_seconds": 16.774, + "successful_reads": 3, + "successful_writes": 3, + "failed_attempts": 1, + "successful_rps": 0.358, + "first_recovery_seconds": 6.15 + } + ] +} diff --git a/evidence/lab4-2026-11-11.md b/evidence/lab4-2026-11-11.md new file mode 100644 index 0000000..c386a7a --- /dev/null +++ b/evidence/lab4-2026-11-11.md @@ -0,0 +1,79 @@ +# Lab4 evidence — 2026-11-11 + +Стенд: Docker Compose, nginx на `127.0.0.1:18084`, API ×2, PostgreSQL, Redis, MinIO. Секреты и session tokens в evidence не сохранялись. + +## Bootstrap и миграции + +- MinIO server image: `bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd…e0cab`. +- `minio-init`: bucket `messenger-attachments` создан, anonymous access установлен `private`, пользователь `messenger-api` создан, policy `messenger-attachments-rw` назначена. +- Flyway: успешно проверены и применены 4 миграции, итоговая версия schema `v4`. +- `docker compose ps`: две здоровые API-реплики; MinIO/PostgreSQL/Redis/nginx healthy; наружу опубликован только nginx. + +## Проверки + +```text +AttachmentServiceTest + AttachmentReconcilerTest: BUILD SUCCESSFUL, 10 tests. +tests/smoke.py: PASS: lab4 public API scenario. +student_tests/lab4_negative.py: +PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, +byte limit and filename checks. +``` + +Persistence hook останавливал nginx, обе API, Redis, PostgreSQL и MinIO, затем запускал те же named volumes и повторял idempotent bucket bootstrap: + +```text +PASS: persistence preserved confirmed data and existing session. +``` + +Failover hook остановил replica из `X-Instance-Id`, обслужившую тестовый запрос: + +```json +{"observed_instances_before":2,"successful_writes":102,"failed_attempts":0,"first_recovery_seconds":1.386} +``` + +```text +PASS: failover preserved confirmed data and existing session. +``` + +При остановленном MinIO: + +```text +PASS: unavailable S3 returns contract JSON 503 +``` + +После запуска того же MinIO volume публичный smoke повторно завершился PASS, checksum скачанного original совпал. + +## Level 4: consistency, memory and private S3 + +Unit tests inject an interrupted request stream before S3 PUT and a repository +failure after S3 PUT. The first leaves storage empty; the second verifies the +compensating DELETE. Reconciler tests prove that an orphan younger than the +one-hour grace period survives, an older orphan is deleted, dry-run never +deletes, and a missing object is reported. + +Bootstrap was executed again against the existing bucket and completed with +exit code 0. The direct S3 probe (without application API) produced: + +```text +PASS: bucket is private and API credentials are non-admin +``` + +Full Gradle suite using the live Compose PostgreSQL/Redis/MinIO network: + +```text +BUILD SUCCESSFUL in 11m 47s +``` + +Concurrent memory experiment (`evidence/upload-memory-2026-11-11.json`): + +```json +{"concurrency":12,"successful_uploads":12,"payload_bytes_each":6835833, + "duration_seconds":10.156,"api_replicas":2,"memory_samples":5, + "baseline_api_memory_bytes":668467200,"peak_api_memory_bytes":839909375, + "peak_increase_bytes":171442175,"configured_max_concurrent_uploads_per_replica":3} +``` + +The memory values are the aggregate of both API containers from `docker stats`. +Sampling can miss a shorter peak, so this is an operational bound observation, +not a JVM heap proof. The code-level bounds are 10 MiB/request, 40M pixels and +three active uploads per replica. diff --git a/evidence/lab5-2026-11-18.md b/evidence/lab5-2026-11-18.md new file mode 100644 index 0000000..a782c2f --- /dev/null +++ b/evidence/lab5-2026-11-18.md @@ -0,0 +1,41 @@ +# Lab5 verification — 2026-11-18 + +Среда: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23, Spring Boot 4.1.1, RabbitMQ 4.1.4. + +## Статические и unit-проверки + +```text +docker compose --env-file .env.example config --quiet +PASS (exit 0) + +gradlew test --tests '*AttachmentServiceTest' --tests '*ThumbnailProcessorTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest' +BUILD SUCCESSFUL +``` + +## Публичный сценарий + +```text +python tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +``` + +В момент проверки отдельные services `api`, `worker`, `rabbitmq`, `postgres`, `redis`, `minio` и `nginx` были запущены; RabbitMQ queue `attachments.process.v1` имела `durable=true`. + +## Остановка worker, restart и redelivery + +```text +python student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery +``` + +Сценарий теста: + +1. останавливает worker; +2. получает upload `202 queued`, проверяет `409` content и привязку к сообщению; +3. перезапускает API и RabbitMQ без удаления volumes; +4. запускает worker и ждёт `ready`; +5. сверяет original bytes и единственный thumbnail; +6. публикует тот же `job_id` повторно; +7. проверяет terminal `ready` и ровно одну запись thumbnail. + +Синтетические пароли и токены тест не печатает и в evidence не сохраняет. diff --git a/evidence/lab5-level4-2026-11-19.md b/evidence/lab5-level4-2026-11-19.md new file mode 100644 index 0000000..00b9be6 --- /dev/null +++ b/evidence/lab5-level4-2026-11-19.md @@ -0,0 +1,56 @@ +# Lab5 level 4 verification — 2026-11-19 + +Среда: Windows, Docker Compose, Java 23, Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, MinIO. + +## Статические и unit-проверки + +```text +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +docker compose --env-file .env.example config --quiet +exit 0 + +gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' \ + --tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' \ + --tests '*StagingCleanerTest' +BUILD SUCCESSFUL +``` + +`ImageVariantProcessorTest` проверяет реальные JPEG/WebP signatures, ImageIO decode и размеры thumbnail/crop/optimized. `RetryPolicyTest` проверяет bounded exponential backoff и jitter. + +## API и recovery + +```text +python -X utf8 tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery +``` + +Recovery test останавливает worker, получает `202 queued`, перезапускает API и RabbitMQ без удаления volumes, запускает worker и сверяет original SHA-256, thumbnail 256×160, crop 128×128, WebP signature. Повторная публикация того же `job_id` оставляет `ready` и ровно три variant rows. + +## Доказанные окна отказа + +```text +python -X utf8 student_tests/lab5_fault_injection.py +PASS: outbox commit/publish crash and result commit/ack crash recovered +``` + +1. API завершается после COMMIT attachment/outbox до publish. Перезапущенный API republisher публикует сохранённое намерение; новый upload не создаётся вручную. +2. Worker завершается после записи объектов и atomic ready COMMIT до ack. Redelivery завершает тот же effect; variant rows остаются уникальными. + +## DLQ, replay и lease + +```text +python -X utf8 student_tests/lab5_dlq_replay_lease.py +PASS: DLQ, guarded replay and expired lease recovery +``` + +- malformed persistent JSON был permanent failure и увеличил `attachments.failed.v1` ровно на 1; +- failed job replayed один раз, audit count = 1; повтор в queued/ready дал no-op; +- после replay получены те же три variant names без второго набора; +- искусственно истёкший processing lease был requeued, attempt увеличился, job дошёл до ready. + +Секреты, session tokens и contents сообщений в evidence не сохранялись. Docker volumes в проверках не удалялись. diff --git a/evidence/lab5-level5-2026-11-19.md b/evidence/lab5-level5-2026-11-19.md new file mode 100644 index 0000000..438c45b --- /dev/null +++ b/evidence/lab5-level5-2026-11-19.md @@ -0,0 +1,213 @@ +# Lab5, трек 5.2 — доказанная идемпотентность pipeline + +Дата: 2026-11-19. Исходная точка: `75d2f78` + рабочее дерево с изменениями lab5. +Стенд: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23 (контейнеры и Gradle +toolchain), Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, Redis 7.4, +MinIO `bitnamilegacy/minio:2025.4.22-debian-12-r1`. + +Все пользователи, сообщения и изображения синтетические. Секреты, session tokens, +содержимое сообщений и signed URL в evidence не сохранялись. + +## Гипотеза и инварианты + +Гипотеза: при versioned pipeline и fencing по `(attachment_id, pipeline_version, +lease)` повторная доставка, перестановка сообщений, crash в окнах до/после side +effect и устаревшая job не теряют подтверждённую работу и не заменяют результат +новой версии старым. + +Проверяемые инварианты: + +1. `attachments.status='ready'` для активной версии означает ровно три variant + rows этой версии (`thumbnail`, `crop`, `optimized`) и отсутствие + `queued/processing` jobs у вложения. +2. Для вложения существует не более одной активной (`queued/processing`) job. +3. Ключ variant rows — `(attachment_id, pipeline_version, name)`; повторная + запись не создаёт дубль. +4. Объектные ключи версионированы: `thumbnail-vN.jpg`, `crop-vN.jpg`, + `optimized-vN.webp`; `original` общий и неизменяемый. Поэтому v1 не может + перезаписать v2. +5. `attachments.pipeline_version` монотонно повышается только аудированной + транзакцией `promote-pipeline`; старая `queued/processing` job получает + `superseded` и `published=true`. +6. Все подтверждённые тестом задачи (`status='ready'`) остаются `ready` после + duplicate/reorder/stale доставок. +7. В active version нет variant rows с `pipeline_version` больше + `attachments.pipeline_version`. +8. Одна и та же `pipeline_version` детерминирована; изменение профиля + оформляется новой версией, потому что `ON CONFLICT DO NOTHING` не перезаписывает + уже зафиксированные variant rows. + +## Команды и результаты + +```text +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +docker compose --env-file .env.example config --quiet +exit 0 + +.\gradlew.bat test +BUILD SUCCESSFUL (30 tests; full suite against live Compose PostgreSQL/Redis/MinIO) +``` + +Для полного Gradle suite хранилища Compose не публикуют порты наружу; на время +прогона использовались временные localhost-туннели к `postgres:5432`, +`redis:6379`, `rabbitmq:5672`, `minio:9000`. После прогона туннели удалены. + +### Основной эксперимент трека + +```text +python -X utf8 student_tests/lab5_pipeline_idempotency.py +PASS: version fencing, superseded, duplicate/reorder and pre/post-side-effect crashes +``` + +Тест автоматически: + +1. создаёт v1 attachment, дожидается `ready` и снимает snapshot трёх variant rows; +2. останавливает worker, повышает версию до v2 через `scripts/promote-pipeline`, + публикует `v2, v2, v1` (порядок доставки не совпадает с порядком создания) и + запускает worker; +3. проверяет, что v2 дал отдельный результат, в БД ровно 6 rows (3 v1 + 3 v2), + active version = 2, а повторные stale v1 / duplicate v2 не меняют snapshot; +4. отдельно повышает ещё **queued** v1 job: она становится + `superseded|published=true|superseded_by_v2`, v1 variant rows не появляются, + v2 доходит до `ready` с тремя вариантами; +5. вставляет crash до S3 side effect, после S3 side effect и после result + COMMIT; каждый раз проверяет восстановление через lease/reaper/redelivery; +6. проверяет, что все 5 подтверждённых вложений остались `ready`. + +### Наблюдаемое состояние после прогона + +Пример versioned-attachment (сокращённые ID приведены для читаемости): + +```text +attachment A (v1 -> v2, сначала завершён v1) + active=2 ready + job v1 ready published=t attempt=1 + job v2 ready published=t attempt=1 + variant keys: crop-v1, optimized-v1, thumbnail-v1, + crop-v2, optimized-v2, thumbnail-v2 + +attachment B (queued v1 повышен до v2) + active=2 ready + job v1 superseded published=t last_error=superseded_by_v2 + job v2 ready published=t attempt=1 + variant keys: crop-v2, optimized-v2, thumbnail-v2 (v1 rows отсутствуют) +``` + +SQL-инварианты по всем attachment, созданным 2026-11-19 в ходе финального +прогона и регрессии lab5: + +```text +superseded_jobs|1 +today_promotions|9 +today_ready|55 +today_ready_bad_variants|0 +ready_with_nonterminal_job|0 +attachments_with_two_active_jobs|0 +duplicate_variant_keys|0 +stale_results_for_active_version|0 +``` + +Полный volume содержит также 3 attachment, созданных 2026-11-18 до миграции V7 +(тогда уровень 3 сохранял только original + thumbnail). Это исторические данные +предыдущего стенда; они не являются результатом текущего versioned pipeline. +Запросы инвариантов выше ограничены `created_at >= '2026-11-19'`; на чистом +volume исторических строк нет. + +### Точки отказа + +| Fault | Где срабатывает | Наблюдение | +| --- | --- | --- | +| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker, до чтения staging/S3 PUT, `halt(84)` | 0 variant rows, 0 объектов; reaper возвращает job в `queued`, attempt увеличивается, задача доходит до `ready` | +| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker, после 4 S3 PUT, до result COMMIT, `halt(85)` | objects=4, variant rows=0; повтор пишет те же deterministic keys и коммитит результат, objects остаётся 4 | +| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker, после atomic `ready` COMMIT, до ack, `halt(86)` | redelivery видит `ready` и завершает no-op; 3 variant rows не дублируются | +| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API, после COMMIT attachment/outbox, до publish, `halt(87)` | republisher находит `published=false` и публикует сохранённое намерение; задача доходит до `ready` | + +Crash-точки включаются только переменными окружения и по умолчанию выключены; +они предназначены для изолированного учебного стенда. + +## Регрессия уровней 3–4 + +```text +python -X utf8 tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 student_tests/lab5_worker_recovery.py +PASS: stopped worker, durable restart recovery and duplicate delivery + +python -X utf8 student_tests/lab5_fault_injection.py +PASS: outbox commit/publish crash and result commit/ack crash recovered + +python -X utf8 student_tests/lab5_dlq_replay_lease.py +PASS: DLQ, guarded replay and expired lease recovery + +python -X utf8 student_tests/idempotency_concurrent.py +PASS: 32 concurrent requests used 2 replicas and created one message + +python -X utf8 student_tests/readiness_gate.py +PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration + +python -X utf8 student_tests/failover_each.py --action-script scripts/stop-one.cmd \ + --restore-script scripts/restore-api.cmd --window 15 \ + --output evidence/failover-each-2026-11-19.json +(JSON: 2 replicas sequentially stopped and restored; confirmed writes preserved) + +python -X utf8 student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \ + --duration 180 --concurrency 4 \ + --output evidence/rolling-update-2026-11-19.json +(JSON: 4 observed replacement instance IDs; 335 successful writes, 3 transitional failures) + +python -X utf8 student_tests/lab4_negative.py +PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks + +python -X utf8 student_tests/concurrent_upload_memory.py \ + --output evidence/upload-memory-2026-11-19.json +PASS: concurrent upload memory experiment (12/12 uploads accepted, 2 API replicas) +``` + +`readiness_gate.py` проверяет Google-подобный gate по обязательной схеме: +`ReadinessService` теперь пробует обе таблицы — `message_idempotency` (lab3) и +`attachments` (lab4/lab5). `concurrent_upload_memory.py` обновлён с lab4-контракта +`201` на lab5-контракт `202`. + +## Почему это at-least-once, а не exactly-once + +RabbitMQ даёт at-least-once: сообщение может быть доставлено повторно после +потери соединения или падения worker до ack; порядок между разными сообщениями +не является бизнес-порядком. Поэтому duplicate и reorder — нормальные состояния, +а не аномалии. + +Наш pipeline делает **эффект** идемпотентным, но не делает доставку exactly-once: + +- publisher confirm подтверждает только приём сообщения брокером, а consumer ack + отправляется после PostgreSQL COMMIT; между S3 PUT и COMMIT нет распределённой + транзакции; +- crash после S3 PUT и до COMMIT оставляет объекты, а crash после COMMIT и до ack + вызывает redelivery; оба окна специально инжектируются тестом; +- повторная обработка может повторно выполнить S3 PUT, но deterministic keys, + `ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` не создают + второго набора результатов и не меняют active version; +- stale job ограничена `pipeline_version` и lease в каждом переходе состояния, + поэтому не может записать результат поверх новой версии. + +Корректная формулировка: **at-least-once delivery + idempotent effect = +effectively-once наблюдаемый результат для активной версии**. Exactly-once +доставка потребовала бы транзакционной связи RabbitMQ, PostgreSQL и S3 (или +эквивалентного распределённого протокола), которой в этой учебной архитектуре +нет и которая не нужна для данных инвариантов. + +## Границы + +- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO; + эксперимент не доказывает отказоустойчивость к потере host/диска. +- Versioned S3 objects старой версии после promotion «в полёте» могут остаться + orphan-объектами: active result они не перезаписывают, но отдельного GC для + versioned objects нет (чистится только staging). +- `superseded`/promotion — операторская SQL-транзакция через скрипт, не публичный + HTTP API. +- Числа failover/rolling в JSON — проверка сохранности и восстановления на одном + Windows-хосте, не benchmark пропускной способности; latency включает + особенность localhost dual-stack в Docker Desktop. +- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант; + на чистом volume и для новых данных действует инвариант трёх вариантов. diff --git a/evidence/lab6-20261119-131448-worker-stop/failure-observation.json b/evidence/lab6-20261119-131448-worker-stop/failure-observation.json new file mode 100644 index 0000000..bdb8a05 --- /dev/null +++ b/evidence/lab6-20261119-131448-worker-stop/failure-observation.json @@ -0,0 +1,66 @@ +{ + "captured_at": "2026-11-19T10:17:21.9038708Z", + "queue_depth": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795083444.15, + "5" + ] + } + ] + } + }, + "oldest_job_age": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795083446.324, + "139" + ] + } + ] + } + }, + "alerts": { + "status": "success", + "data": { + "alerts": [ + { + "labels": { + "alertname": "AttachmentQueueLagHigh", + "deployment_environment_name": "lab", + "exported_instance": "943917ff1db9", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "otel-collector-exported-metrics", + "messaging_destination_name": "attachments.process.level4.v1", + "otel_scope_name": "messenger-queue", + "otel_scope_version": "1.0.0", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity": "warning", + "slo": "attachment-time-to-ready" + }, + "annotations": { + "description": "The durable worker queue has not drained within the SLO threshold.", + "runbook": "infra/otel/runbooks.md#queue-lag", + "summary": "Oldest attachment job is over 60 seconds old" + }, + "state": "firing", + "activeAt": "2026-11-19T10:16:07.684164819Z", + "value": "1.39e+02" + } + ] + } + } +} diff --git a/evidence/lab6-20261119-131448-worker-stop/load.json b/evidence/lab6-20261119-131448-worker-stop/load.json new file mode 100644 index 0000000..a4e3f14 --- /dev/null +++ b/evidence/lab6-20261119-131448-worker-stop/load.json @@ -0,0 +1,8 @@ +{ + "workload": "synthetic PNG attachment uploads", + "requested": 5, + "accepted": 5, + "errors": 0, + "elapsed_seconds": 1.063, + "contains_credentials": false +} diff --git a/evidence/lab6-20261119-131448-worker-stop/recovery-observation.json b/evidence/lab6-20261119-131448-worker-stop/recovery-observation.json new file mode 100644 index 0000000..7bbd07c --- /dev/null +++ b/evidence/lab6-20261119-131448-worker-stop/recovery-observation.json @@ -0,0 +1,50 @@ +{ + "captured_at": "2026-11-19T10:18:32.6133979Z", + "queue_depth": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795083514.639, + "0" + ] + } + ] + } + }, + "oldest_job_age": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795083516.701, + "0" + ] + } + ] + } + }, + "jobs": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "outcome": "success" + }, + "value": [ + 1795083518.741, + "4.145557430652597" + ] + } + ] + } + } +} diff --git a/evidence/lab6-20261119-131929-database-stop/failure-observation.json b/evidence/lab6-20261119-131929-database-stop/failure-observation.json new file mode 100644 index 0000000..d2351b3 --- /dev/null +++ b/evidence/lab6-20261119-131929-database-stop/failure-observation.json @@ -0,0 +1,20 @@ +{ + "captured_at": "2026-11-19T10:20:07.2547700Z", + "api_5xx": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795083609.264, + "0.15208333333333335" + ] + } + ] + } + }, + "jdbc_spans_hint": "Grafana Explore -> Tempo: { resource.service.name = \"messenger-api\" && span.db.system.name = \"postgresql\" }", + "correlated_logs_hint": "Grafana Explore -> Loki: {service_name=\"messenger-api\"} | json | level=\"ERROR\"" +} diff --git a/evidence/lab6-20261119-131929-database-stop/http-samples.json b/evidence/lab6-20261119-131929-database-stop/http-samples.json new file mode 100644 index 0000000..32a0457 --- /dev/null +++ b/evidence/lab6-20261119-131929-database-stop/http-samples.json @@ -0,0 +1,44 @@ +[ + { + "at": "2026-11-19T10:19:31.3274597Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4537.5 + }, + { + "at": "2026-11-19T10:19:36.9271045Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4098.9 + }, + { + "at": "2026-11-19T10:19:42.0396820Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 3988.8 + }, + { + "at": "2026-11-19T10:19:47.0377275Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4011.4 + }, + { + "at": "2026-11-19T10:19:52.0526176Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4038.2 + }, + { + "at": "2026-11-19T10:19:57.1033752Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4058.3 + }, + { + "at": "2026-11-19T10:20:02.1668525Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4049.2 + } +] diff --git a/evidence/lab6-20261119-131929-database-stop/loki-api-logs.json b/evidence/lab6-20261119-131929-database-stop/loki-api-logs.json new file mode 100644 index 0000000..aad5f1c --- /dev/null +++ b/evidence/lab6-20261119-131929-database-stop/loki-api-logs.json @@ -0,0 +1,563 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083580665709480", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "4e994b1e2de6ea26", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "c3307b04e636e33828d63bc789e863b8" + }, + "values": [ + [ + "1795083580643209485", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083585672714591", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "36f0c3d0073685cf", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "8d45b62708d37ee1b98b38733d488365" + }, + "values": [ + [ + "1795083585670851940", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083595680182050", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "3b18672eedc0d813", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "0e9e710a9decc517e4c19b26ca8976c6" + }, + "values": [ + [ + "1795083595678179181", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083600685613922", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "1b7757646a541419", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "93100e12c7a8800d880867d4ffbe4a09" + }, + "values": [ + [ + "1795083600683377175", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083575542909551", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "5bb939bc64521236", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "7b47023dcfebb5738de24578b483931a" + }, + "values": [ + [ + "1795083575541296003", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083590676105708", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "7ceec07d73695135", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "243ff15610e01771b8784440d0255eb5" + }, + "values": [ + [ + "1795083590675921437", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083605690135533", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "55d69d550d51484a", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "113194c82bf52dc7045c4ddcca64416e" + }, + "values": [ + [ + "1795083605689816022", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "warn", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083570548641327", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "5bb939bc64521236", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "7b47023dcfebb5738de24578b483931a" + }, + "values": [ + [ + "1795083570548127003", + "HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@6908cf44 (This connection has been closed.). Possibly consider using a shorter maxLifetime value." + ] + ] + }, + { + "stream": { + "container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62", + "deployment_environment_name": "lab", + "detected_level": "warn", + "flags": "3", + "host_arch": "amd64", + "host_name": "943917ff1db9", + "observed_timestamp": "1795083570555957813", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "8", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "943917ff1db9", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "5bb939bc64521236", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "7b47023dcfebb5738de24578b483931a" + }, + "values": [ + [ + "1795083570555778998", + "HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@51c569d9 (This connection has been closed.). Possibly consider using a shorter maxLifetime value." + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 40976, + "linesProcessedPerSecond": 136, + "totalBytesProcessed": 2703, + "totalLinesProcessed": 9, + "execTime": 0.065965, + "queueTime": 0.00129, + "subqueries": 0, + "totalEntriesReturned": 9, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 9, + "totalStructuredMetadataBytesProcessed": 1992 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "querierExecTime": 0.047749 + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 1, + "totalBatches": 2, + "totalLinesSent": 9, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 2703, + "headChunkLines": 9, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 9, + "headChunkStructuredMetadataBytes": 1992, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 492747, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "recvWaitTime": 0.008569 + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false, + "totalStreams": 0, + "chunkRefsLookupTime": 0, + "bloomFilterTime": 0 + } + } + } +} diff --git a/evidence/lab6-20261119-131929-database-stop/recovery.txt b/evidence/lab6-20261119-131929-database-stop/recovery.txt new file mode 100644 index 0000000..df881dd --- /dev/null +++ b/evidence/lab6-20261119-131929-database-stop/recovery.txt @@ -0,0 +1 @@ +recovered diff --git a/evidence/lab6-20261119-131929-database-stop/tempo-traces.json b/evidence/lab6-20261119-131929-database-stop/tempo-traces.json new file mode 100644 index 0000000..a0023fb --- /dev/null +++ b/evidence/lab6-20261119-131929-database-stop/tempo-traces.json @@ -0,0 +1,697 @@ +{ + "traces": [ + { + "traceID": "2f7370e029e3f7a16b43878d68ae8e20", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083242196014220", + "durationMs": 32 + }, + { + "traceID": "4896829563bf7a106ee8d143ac84832e", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083239923257437", + "durationMs": 6 + }, + { + "traceID": "198b1479aed61af14a84c7b7a2ed9b1e", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083239570603967", + "durationMs": 1 + }, + { + "traceID": "21502c0de7d46dc0269c8ba624a6c23b", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083233700846631", + "durationMs": 5 + }, + { + "traceID": "3410d7364dba96b2aacfc2aadfea97be", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083229579273036" + }, + { + "traceID": "4c35f7b7fc7e631b7b6ba39f98c579cf", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083229565470275", + "durationMs": 5 + }, + { + "traceID": "20db1bec473cb94e969018b15be1a01f", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083229470223000", + "durationMs": 1 + }, + { + "traceID": "1552c3687c6de569229b530f24a4e3f1", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083223342807944", + "durationMs": 9 + }, + { + "traceID": "2b55cb82eb4404af16918e5a5d0cdd57", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083219563846030", + "durationMs": 1 + }, + { + "traceID": "21224d7bdfd6538117436f0d7d894709", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083219401117603", + "durationMs": 1 + }, + { + "traceID": "1ef30879f867c99a5f768bf2d67de7c4", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083214510024043", + "durationMs": 1 + }, + { + "traceID": "a223dd910754a9dd2d82e7bdb4424dc", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083214497015475", + "durationMs": 1 + }, + { + "traceID": "90dae0d5256b8b84bfd2a8339d6be34", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083214398206478", + "durationMs": 2 + }, + { + "traceID": "4cd4879aceb0549998b84df6f5d778bd", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083210923828917", + "durationMs": 2 + }, + { + "traceID": "28e809860332b12c9835ccf41dddc01e", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083209509201945" + }, + { + "traceID": "52d228b9188561777ea6de6c84f60b96", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083204495117245" + }, + { + "traceID": "1317c6519ef468e6012b6d2791feea2d", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083202718212905", + "durationMs": 4 + }, + { + "traceID": "2f77661745383a05a210a31eeb10e9d3", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083194505678372", + "durationMs": 1 + }, + { + "traceID": "34612903902c205b2c8493baf36d8533", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083194492176788", + "durationMs": 1 + }, + { + "traceID": "29d3aa01326c8a54ddc239c54c429cff", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083192125513698", + "durationMs": 6 + }, + { + "traceID": "4b88e4110e4db4e24c776b7f446c4e7a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083190034891922", + "durationMs": 5 + }, + { + "traceID": "34e785064a8dc9639a8a8f892adbde3c", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083189490866815" + }, + { + "traceID": "12e6eb2f7dc204f7cd4faf2dedd0c126", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083187892313453", + "durationMs": 13 + }, + { + "traceID": "235564101cd94c4ee16ce1f6e61e6573", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083184436384728", + "durationMs": 1 + }, + { + "traceID": "557b661ca4b8a55da2b22488d5d1a76e", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083184423130530" + }, + { + "traceID": "233215abd278bff45d2750c745563535", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083183754090813", + "durationMs": 5 + }, + { + "traceID": "7009d65bdcb674eadd9b54f30c40307", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083179425828960", + "durationMs": 10 + }, + { + "traceID": "39ca7029a7af8f73cdbfdc6dd957a638", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083177461848482", + "durationMs": 3 + }, + { + "traceID": "396d27a55614c7e6cc8e929a8e6642b7", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083174419724867" + }, + { + "traceID": "469f72453a60129e164dc2e9e21ed9f2", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083173349996485", + "durationMs": 3 + }, + { + "traceID": "21524c6bad04c5393ed93d1b32a7ddfb", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083171300590874", + "durationMs": 7 + }, + { + "traceID": "35d793b9eb0d9671db56664143632461", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083169240643784", + "durationMs": 4 + }, + { + "traceID": "2d189acf2529e1e5070a34283fc27aab", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083167170951206", + "durationMs": 4 + }, + { + "traceID": "3510522263a30ca9821d3282fa84df6e", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083163056985121", + "durationMs": 3 + }, + { + "traceID": "20e8f9da366714e56573936da9821eee", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083159419911892", + "durationMs": 1 + }, + { + "traceID": "4684c71727338fddbcc92e81c3b66fde", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083159247657934", + "durationMs": 1 + }, + { + "traceID": "19b2c69c87cfcbcb7fba87cf79f6e056", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083154350059891", + "durationMs": 1 + }, + { + "traceID": "4edb69a63d986c07bb80252c1e3756fd", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083154263143826", + "durationMs": 6 + }, + { + "traceID": "47ba154a890c3cb5eadd06e5c2177290", + "rootServiceName": "messenger-api", + "rootTraceName": "messenger", + "startTimeUnixNano": "1795083154084280265" + }, + { + "traceID": "3c443e3d5e4b067ebe65209271d29870", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083145901013447", + "durationMs": 32 + }, + { + "traceID": "b105a97cc15b8dbd1beab3907e064ec", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083144348529368" + }, + { + "traceID": "3d332022d66852a32492b2bb50e4a49c", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083139347599576" + }, + { + "traceID": "32aeaf15c5816848ead7dc9cb0d317d2", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083134345037608", + "durationMs": 1 + }, + { + "traceID": "307377d165adba665b634e5a787f8faa", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083133489684589", + "durationMs": 10 + }, + { + "traceID": "fd53f3fac974e14a9b0bc6b7fb2b3da", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083129344879817", + "durationMs": 1 + }, + { + "traceID": "1eae20965992f2a3b00292e96763a1b7", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083125000318817", + "durationMs": 4 + }, + { + "traceID": "52ae20055b7d94742c82e7e84a38e735", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083114655214727", + "durationMs": 7 + }, + { + "traceID": "f9c0c2521f8b128e6ad9267eaea7c31", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083114270828828", + "durationMs": 1 + }, + { + "traceID": "553567508997161c44dde4db81e9e7a1", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083114156439582", + "durationMs": 2 + }, + { + "traceID": "52a46cc6567688c8f1a61f4f2da6b7b6", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083112579003220", + "durationMs": 5 + }, + { + "traceID": "20609217d4e9741a70699a93d47ee262", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083109270416562" + }, + { + "traceID": "1a21f5d8b71f0f85cf09f3ea38b9e14e", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083108445836642", + "durationMs": 5 + }, + { + "traceID": "444b20682dddb90d8f199c5706753a9", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083106389151714", + "durationMs": 7 + }, + { + "traceID": "3d802498b048b6c57522373e84d66500", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083104269589692" + }, + { + "traceID": "2913d2fde17689ac2914f12dbbd50bc1", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083104151632742", + "durationMs": 2 + }, + { + "traceID": "228511d2e5d949660507d602f0ddef90", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083102279505914", + "durationMs": 9 + }, + { + "traceID": "1b12ae26787f10f67057fc1015c31917", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083099267378928", + "durationMs": 1 + }, + { + "traceID": "43062e16f5b379215b34b5bb2fc2e407", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083098068647506", + "durationMs": 11 + }, + { + "traceID": "3d8bbe120b5653959c4dabed68f8cbbc", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083094201070242" + }, + { + "traceID": "1feda03af32692e2cb8960e623e387f9", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083094198945116", + "durationMs": 2 + }, + { + "traceID": "2e0cffc7ec23628b6002ed62b1bfc794", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083093973006592", + "durationMs": 3 + }, + { + "traceID": "3d443a0cd84aacb6a25f3988c02c0b22", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083087825041036", + "durationMs": 5 + }, + { + "traceID": "3c05f026d5daa80a273f11e2d1f15a1b", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083084195702312", + "durationMs": 1 + }, + { + "traceID": "3377fff79205c50913da9523e86cf267", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083081595922893", + "durationMs": 9 + }, + { + "traceID": "26186c174d50642629713908cc0b22f7", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083079195725678" + }, + { + "traceID": "fab56b55c29cc155c83aad521a338fe", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083074193776114", + "durationMs": 1 + }, + { + "traceID": "50e8e14ee65caf9671d55d7436644ad6", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083073356737601", + "durationMs": 3 + }, + { + "traceID": "2ec40aa182ea167e2f5a21f958c4ef94", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083071308395569", + "durationMs": 6 + }, + { + "traceID": "1ac3d4136a09f7f4bf756f6846bb2ccb", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083069216469144", + "durationMs": 8 + }, + { + "traceID": "1b5e3b8f90874b5011b7b12483238ec2", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083067098088469", + "durationMs": 8 + }, + { + "traceID": "ab2a39ce37c7fd72a20f377cdfb284b", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083065014158531", + "durationMs": 8 + }, + { + "traceID": "1cf32cac103b8da37d0a36b742a4c0ba", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083064125248315", + "durationMs": 2 + }, + { + "traceID": "18201f914e73079877fe4eaecf08f654", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083064003875969", + "durationMs": 1 + }, + { + "traceID": "2a7711fe81236abc4a73fe4a6e8ddb40", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083062950592934", + "durationMs": 7 + }, + { + "traceID": "460a4763151dd74b518c962f0ea45435", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083059124764109", + "durationMs": 1 + }, + { + "traceID": "25c3e39f4b20e8d0d3cb078ff5df2ad7", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083058999411000", + "durationMs": 4 + }, + { + "traceID": "3ed3706b294ebabc65ef639c6040c067", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083058731005797", + "durationMs": 16 + }, + { + "traceID": "7d9a5a43868eaf91e3e3bc229126d94", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083054490241800", + "durationMs": 10 + }, + { + "traceID": "106606ab7649a3687c746c4522500278", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083054115882944", + "durationMs": 4 + }, + { + "traceID": "441c134a0bd1992ddfe2e444fa00a396", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083049115458312", + "durationMs": 1 + }, + { + "traceID": "2166253a92c073ec9134f2bdd629b69", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083049113615406", + "durationMs": 1 + }, + { + "traceID": "1bf168c1495a1aed48983e09c32ad8fd", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083048185194380", + "durationMs": 4 + }, + { + "traceID": "271d3c0780a876ba064fb3e4ee5b5277", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083046116869717", + "durationMs": 6 + }, + { + "traceID": "16cb9a3288e21109cff7ecbcc24b985d", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083039109138417", + "durationMs": 1 + }, + { + "traceID": "2c5c1ca5cc5238040df8864a052850a0", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083033910282803", + "durationMs": 3 + }, + { + "traceID": "3805ef57104cfb8eb7d4463ea9285ed8", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083033473941705", + "durationMs": 14 + }, + { + "traceID": "56075bd17bad6bf5d7da94e0c27d4f1e", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083029041703172", + "durationMs": 1 + }, + { + "traceID": "480532461c728593952b19efd3ebee14", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083028967837404", + "durationMs": 265 + }, + { + "traceID": "25f19f85951133faaec5ca7de7967394", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083028936172779", + "durationMs": 105 + }, + { + "traceID": "2437cad488ed6ea0b9a8633d82205c4", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083025696649794", + "durationMs": 4 + }, + { + "traceID": "49e629dcff8000c3ffc9c6850b7e3a6d", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083023439629709", + "durationMs": 8 + }, + { + "traceID": "2e684b669291ea59269620d2d49f1939", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083021062422221", + "durationMs": 5 + }, + { + "traceID": "23daf7d8e1d72a77599a9a4c3ad0a2bd", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083018975339309", + "durationMs": 11 + }, + { + "traceID": "3e2c8a85df9d466ed068290a341eaceb", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083018949869044", + "durationMs": 1 + }, + { + "traceID": "3a251a6694c7e66a5486f75866ce0b11", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083017702814031", + "durationMs": 3 + }, + { + "traceID": "2f3ffbe6bcba1490803df66b72bb6351", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083009553293092", + "durationMs": 5 + }, + { + "traceID": "3b2652b81e65bceffc7308804b6e5b3d", + "rootServiceName": "messenger-api", + "rootTraceName": "QueueTelemetry.sample", + "startTimeUnixNano": "1795083007643550929", + "durationMs": 2 + }, + { + "traceID": "31ddb25055f8bcde77faa3d25717c728", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795083007413274745", + "durationMs": 9 + }, + { + "traceID": "1863383c2292bd14bbec60f475b2e2c0", + "rootServiceName": "messenger-api", + "rootTraceName": "UnpublishedJobRepublisher.publishPending", + "startTimeUnixNano": "1795083003890520986", + "durationMs": 1 + }, + { + "traceID": "30b2935430bc98695edd4770944f602e", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795083003869406925", + "durationMs": 2 + } + ], + "metrics": { + "inspectedTraces": 333, + "inspectedBytes": "27485", + "completedJobs": 1, + "totalJobs": 3 + } +} diff --git a/evidence/lab6-20261119-191738-retry-trace/load.json b/evidence/lab6-20261119-191738-retry-trace/load.json new file mode 100644 index 0000000..474df11 --- /dev/null +++ b/evidence/lab6-20261119-191738-retry-trace/load.json @@ -0,0 +1,8 @@ +{ + "workload": "synthetic PNG attachment uploads", + "requested": 1, + "accepted": 1, + "errors": 0, + "elapsed_seconds": 0.181, + "contains_credentials": false +} diff --git a/evidence/lab6-20261119-191738-retry-trace/retry-trace-summary.json b/evidence/lab6-20261119-191738-retry-trace/retry-trace-summary.json new file mode 100644 index 0000000..2bd15e3 --- /dev/null +++ b/evidence/lab6-20261119-191738-retry-trace/retry-trace-summary.json @@ -0,0 +1,371 @@ +{ + "captured_at": "2026-11-19T16:19:25.6100693+00:00", + "trace_id": "a3dc1f8fabc7dc732fc3f502007f7654", + "retry_event": "attachment_job_retry_scheduled", + "retry_attempt": "2", + "retry_error_code": "processing_failed", + "completion_event": "attachment_job_completed", + "producer_spans": 3, + "consumer_spans": 3, + "same_trace": true, + "synthetic_data_only": true, + "spans": [ + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chatId}/attachments", + "kind": "SPAN_KIND_SERVER", + "span_id": "FnoyHc+3lfY=", + "parent_span_id": null + }, + { + "service": "messenger-api", + "name": "SELECT messenger.sessions", + "kind": "SPAN_KIND_CLIENT", + "span_id": "AwIhXSCtyas=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "SELECT messenger.users", + "kind": "SPAN_KIND_CLIENT", + "span_id": "HcCKR1DIetM=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "1j1NfZtuv3Q=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "SELECT messenger.chats", + "kind": "SPAN_KIND_CLIENT", + "span_id": "0FENqd5y7Xg=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "SELECT messenger.chat_members", + "kind": "SPAN_KIND_CLIENT", + "span_id": "fDDLjgAtoiQ=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "INSERT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "NMtDSJHphqU=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "INSERT messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "X6SdQUgT6n8=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "JW9UyNpvDLc=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "PUT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "oJlbvd5/myM=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-api", + "name": "attachments.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "h977yFuU9zE=", + "parent_span_id": "FnoyHc+3lfY=" + }, + { + "service": "messenger-worker", + "name": "attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "TqQIfXHlizY=", + "parent_span_id": "h977yFuU9zE=" + }, + { + "service": "messenger-worker", + "name": "attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "nIg4Oa+5lKY=", + "parent_span_id": "csh2DBq5I10=" + }, + { + "service": "messenger-worker", + "name": "attachment.process.v1 process", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "Bfy8+s/9Unk=", + "parent_span_id": "vcCIFAFUBAM=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "V7Aeg6wlcMA=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vxPbe/6D3BQ=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "HJI00k8Q2B4=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "4XxYraggMpU=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "AoRNFcpE608=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "J2nUmKPyp8A=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "7WwtV5gNGjs=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "ny8zFeqRljY=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "WpX199HTrxw=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "uUmkZKxHWk0=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "CzRA6Jtx9ek=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "URqheSsGi2c=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vAsiDivIg8A=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "fxdp+Lm8MRg=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "QMAfoOiUphI=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "ENOwjZ3sLF4=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "1J5W4nRH74U=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "EUzGj0Wk+mM=", + "parent_span_id": "nIg4Oa+5lKY=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "DbH/1dk+6Bs=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "bZfJhnK++2A=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "HKLGaPt22w0=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "2Q87CMlxKlw=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "MnDc4GLrCGY=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "OV/6iY4llYI=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "SELECT messenger", + "kind": "SPAN_KIND_CLIENT", + "span_id": "EZd5UKV/Kqk=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachment_jobs", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Oi+6TrQwblY=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "INSERT messenger.attachment_variants", + "kind": "SPAN_KIND_CLIENT", + "span_id": "h7/DjtpoNvk=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "INSERT messenger.attachment_variants", + "kind": "SPAN_KIND_CLIENT", + "span_id": "qqugt2fdBzY=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "INSERT messenger.attachment_variants", + "kind": "SPAN_KIND_CLIENT", + "span_id": "1CgS88Bdu2A=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "UPDATE messenger.attachments", + "kind": "SPAN_KIND_CLIENT", + "span_id": "aA89RdGHL5I=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "PUT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "VX1lxYQCfT4=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "PUT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "UeAXa0EhHpo=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "PUT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "E7t66VFyg6Q=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "PUT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "CvEetnQTWlI=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-worker", + "name": "DELETE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "TpQJoEtLY7o=", + "parent_span_id": "Bfy8+s/9Unk=" + }, + { + "service": "messenger-api", + "name": "attachments.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "csh2DBq5I10=", + "parent_span_id": "TqQIfXHlizY=" + }, + { + "service": "messenger-api", + "name": "attachments.v1 publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "vcCIFAFUBAM=", + "parent_span_id": "nIg4Oa+5lKY=" + } + ] +} diff --git a/evidence/lab6-20261119-192857-worker-stop/failure-observation.json b/evidence/lab6-20261119-192857-worker-stop/failure-observation.json new file mode 100644 index 0000000..bbb2f4d --- /dev/null +++ b/evidence/lab6-20261119-192857-worker-stop/failure-observation.json @@ -0,0 +1,67 @@ +{ + "captured_at": "2026-11-19T16:31:32.4454018Z", + "queue_depth": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795105892.467, + "3" + ] + } + ] + } + }, + "oldest_job_age": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795105892.476, + "138" + ] + } + ] + } + }, + "alerts": { + "status": "success", + "data": { + "alerts": [ + { + "labels": { + "alertname": "AttachmentQueueLagHigh", + "deployment_environment_name": "lab", + "exported_instance": "5f24a6dc-7220-429d-a093-d19540a77484", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "otel-collector-exported-metrics", + "messaging_destination_name": "attachments.process.level4.v1", + "otel_scope_name": "messenger-queue", + "otel_scope_version": "1.0.0", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity": "warning", + "slo": "attachment-time-to-ready" + }, + "annotations": { + "description": "The durable worker queue has not drained within the SLO threshold.", + "runbook": "infra/otel/runbooks.md#queue-lag", + "summary": "Oldest attachment job is over 60 seconds old" + }, + "state": "firing", + "activeAt": "2026-11-19T16:30:27.684164819Z", + "value": "1.28e+02" + } + ] + } + }, + "alert_firing": true +} diff --git a/evidence/lab6-20261119-192857-worker-stop/load.json b/evidence/lab6-20261119-192857-worker-stop/load.json new file mode 100644 index 0000000..c26e3c7 --- /dev/null +++ b/evidence/lab6-20261119-192857-worker-stop/load.json @@ -0,0 +1,8 @@ +{ + "workload": "synthetic PNG attachment uploads", + "requested": 3, + "accepted": 3, + "errors": 0, + "elapsed_seconds": 0.597, + "contains_credentials": false +} diff --git a/evidence/lab6-20261119-192857-worker-stop/recovery-observation.json b/evidence/lab6-20261119-192857-worker-stop/recovery-observation.json new file mode 100644 index 0000000..760d1e7 --- /dev/null +++ b/evidence/lab6-20261119-192857-worker-stop/recovery-observation.json @@ -0,0 +1,66 @@ +{ + "captured_at": "2026-11-19T16:32:43.2643478Z", + "queue_depth": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795105963.253, + "0" + ] + } + ] + } + }, + "oldest_job_age": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795105963.262, + "0" + ] + } + ] + } + }, + "jobs": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "outcome": "retry" + }, + "value": [ + 1795105963.273, + "0" + ] + }, + { + "metric": { + "outcome": "success" + }, + "value": [ + 1795105963.273, + "0" + ] + } + ] + } + }, + "alerts": { + "status": "success", + "data": { + "alerts": [] + } + }, + "alert_resolved": true +} diff --git a/evidence/lab6-20261119-193359-database-stop/failure-observation.json b/evidence/lab6-20261119-193359-database-stop/failure-observation.json new file mode 100644 index 0000000..277ab47 --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/failure-observation.json @@ -0,0 +1,43 @@ +{ + "captured_at": "2026-11-19T16:36:47.0833934Z", + "api_5xx": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795106207.063, + "0.5846403015440146" + ] + } + ] + } + }, + "alerts": { + "status": "success", + "data": { + "alerts": [ + { + "labels": { + "alertname": "MessengerApiHighErrorRate", + "severity": "warning", + "slo": "api-availability" + }, + "annotations": { + "description": "Five-minute 5xx ratio is above 5% for two minutes.", + "runbook": "infra/otel/runbooks.md#api-high-error-rate", + "summary": "Messenger API 5xx ratio exceeds 5%" + }, + "state": "firing", + "activeAt": "2026-11-19T16:34:37.684164819Z", + "value": "9.644768265300965e-01" + } + ] + } + }, + "alert_firing": true, + "jdbc_spans_hint": "Grafana Explore -> Tempo: { resource.service.name = \"messenger-api\" && span.db.system.name = \"postgresql\" }", + "correlated_logs_hint": "Grafana Explore -> Loki: {service_name=\"messenger-api\"} | json | level=\"ERROR\"" +} diff --git a/evidence/lab6-20261119-193359-database-stop/http-samples.json b/evidence/lab6-20261119-193359-database-stop/http-samples.json new file mode 100644 index 0000000..75a9981 --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/http-samples.json @@ -0,0 +1,200 @@ +[ + { + "at": "2026-11-19T16:34:00.5253128Z", + "ready_status": 503, + "business_status": 401, + "duration_ms": 4240.2 + }, + { + "at": "2026-11-19T16:34:05.8085003Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4040.7 + }, + { + "at": "2026-11-19T16:34:10.8602421Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4034.5 + }, + { + "at": "2026-11-19T16:34:15.9061557Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4035.0 + }, + { + "at": "2026-11-19T16:34:20.9495283Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4019.5 + }, + { + "at": "2026-11-19T16:34:25.9764025Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4018.2 + }, + { + "at": "2026-11-19T16:34:31.0048332Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4024.0 + }, + { + "at": "2026-11-19T16:34:36.0394294Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4033.6 + }, + { + "at": "2026-11-19T16:34:41.0840646Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4028.6 + }, + { + "at": "2026-11-19T16:34:46.1268110Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4076.1 + }, + { + "at": "2026-11-19T16:34:51.2134687Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4013.2 + }, + { + "at": "2026-11-19T16:34:56.2358974Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4022.9 + }, + { + "at": "2026-11-19T16:35:01.2714634Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4029.5 + }, + { + "at": "2026-11-19T16:35:06.3066502Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4023.7 + }, + { + "at": "2026-11-19T16:35:11.3429193Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4030.3 + }, + { + "at": "2026-11-19T16:35:16.3792975Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4031.5 + }, + { + "at": "2026-11-19T16:35:21.4180181Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4013.5 + }, + { + "at": "2026-11-19T16:35:26.4479913Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4025.9 + }, + { + "at": "2026-11-19T16:35:31.4865276Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4028.5 + }, + { + "at": "2026-11-19T16:35:36.5299328Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4026.3 + }, + { + "at": "2026-11-19T16:35:41.5598609Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4026.4 + }, + { + "at": "2026-11-19T16:35:46.5956509Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4023.2 + }, + { + "at": "2026-11-19T16:35:51.6212244Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4012.7 + }, + { + "at": "2026-11-19T16:35:56.6416286Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4030.0 + }, + { + "at": "2026-11-19T16:36:01.6791804Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4022.7 + }, + { + "at": "2026-11-19T16:36:06.7040114Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4021.4 + }, + { + "at": "2026-11-19T16:36:11.7306456Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4029.9 + }, + { + "at": "2026-11-19T16:36:16.7621800Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4028.0 + }, + { + "at": "2026-11-19T16:36:21.8035195Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4014.1 + }, + { + "at": "2026-11-19T16:36:26.8302112Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4018.8 + }, + { + "at": "2026-11-19T16:36:31.8595958Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4027.3 + }, + { + "at": "2026-11-19T16:36:36.8893717Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4029.3 + }, + { + "at": "2026-11-19T16:36:41.9284299Z", + "ready_status": 503, + "business_status": 503, + "duration_ms": 4032.4 + } +] diff --git a/evidence/lab6-20261119-193359-database-stop/loki-api-logs.json b/evidence/lab6-20261119-193359-database-stop/loki-api-logs.json new file mode 100644 index 0000000..3c905e0 --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/loki-api-logs.json @@ -0,0 +1,2123 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106045094923919", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "b3f2d793c34347c1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "696ba634a1655319ddc27bdcd88edb88" + }, + "values": [ + [ + "1795106045083660948", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106050098079778", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "4def8dfcc0b7c1d1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "8aa2d729f4289a78f5bc063766bb96c8" + }, + "values": [ + [ + "1795106050097281756", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106060132145641", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "0c7b6b91a4211730", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "f6b607ed6b3ba7f26974905700a68e35" + }, + "values": [ + [ + "1795106060130958299", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106065134700854", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "969e2d1a7154490d", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "129282c6ab05d17939dee167e9167f36" + }, + "values": [ + [ + "1795106065133867245", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106075140840454", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "3967041c306256fb", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "ffbcae0757694ae107ff416de59cf035" + }, + "values": [ + [ + "1795106075138792660", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106080145928048", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "0bc2b88d2c4bc425", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "4116ee476129db9d1c5b24b66c78d75d" + }, + "values": [ + [ + "1795106080144395155", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106090190691113", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "694d3060838e0c48", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "bfe5be6ffeda3957c5df018612199dd4" + }, + "values": [ + [ + "1795106090189601889", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106095196634295", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "027ec0258ec3574f", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "06379ee050591e230ce9cdb5ff1a45cd" + }, + "values": [ + [ + "1795106095193670187", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106105201519398", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "f50e8b195a58f578", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "8396a7ae5d6e8b918842c9f7d61b2d5a" + }, + "values": [ + [ + "1795106105200867752", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106110204979230", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "88720a75641d3fca", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "92c26c25b40ecd8451d72d60c12df38f" + }, + "values": [ + [ + "1795106110203652779", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106120246115083", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "b27fea8e7aa18df0", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "f12a923e560e26150f83b27eb3632ee2" + }, + "values": [ + [ + "1795106120245215929", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106125252334226", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "e7caa75a84636a53", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "46ec9d9a4220c636a19fed2130016bd8" + }, + "values": [ + [ + "1795106125250682754", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106135257547408", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "56f8043c2ca8b57c", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "ce8eb97f9cdc5f632a80d7e1637828b9" + }, + "values": [ + [ + "1795106135256388403", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106140261421301", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "ac4446de6b6f354d", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "af456f9971f576e8b747f90bcc86d769" + }, + "values": [ + [ + "1795106140260334827", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106150300710049", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "39c1d3d700ea52ee", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "0f2c3064cc3961cf302bb3ed2d137308" + }, + "values": [ + [ + "1795106150299767309", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106155305074423", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "0bdcddd766b2dd08", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "6e79b3e60f247045daef3682868f3a5e" + }, + "values": [ + [ + "1795106155303611040", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106165313225592", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "059d6002dd63b404", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "c826ae930d3486dafeebf3e7d554aa8b" + }, + "values": [ + [ + "1795106165311016943", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106170318394926", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "db9bf106ecfdf615", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "372ca71e37a1018980b7f601098860b2" + }, + "values": [ + [ + "1795106170317207299", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106180361911299", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "589339bc29a094d7", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "c9595756082de4d69be6b0c5e93fb137" + }, + "values": [ + [ + "1795106180361165993", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106185365322091", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "7968a01473b0db17", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "993ced083a58d59960280738916f995f" + }, + "values": [ + [ + "1795106185364171081", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106195372569046", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "117b71017f2865c4", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "89b7370dc4c0b71921dbde1a86b9add1" + }, + "values": [ + [ + "1795106195371516008", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "error", + "exception_message": "Failed to obtain JDBC Connection", + "exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j", + "exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106200376561573", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "609bdda11c7ec647", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "38292571efad5c6a461c627486b2df20" + }, + "values": [ + [ + "1795106200375660495", + "Unexpected error occurred in scheduled task" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/auth/sessions", + "observed_timestamp": "1795105740817753087", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "c50101d6-c23e-4d3e-a529-15b0883907c8", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "896d4362a5fdd3f1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "6db5a93b0dede22bf80b0698a6f43fb1" + }, + "values": [ + [ + "1795105740817634117", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/auth/sessions", + "observed_timestamp": "1795105741103644683", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "1167feef-d6fd-424e-8aa2-3fbd4fed04ba", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "6f7cbbd3518aa5f9", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "589e96ef21961d4a2f805682cd7ee6d3" + }, + "values": [ + [ + "1795105741103528943", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/auth/sessions", + "observed_timestamp": "1795105741469249661", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "5c84f2a4-edcf-4324-bc16-0befa16cb424", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "a4c85ef5d0bafab5", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "431bbedf0049dfb1b0e90d7356d9f4fd" + }, + "values": [ + [ + "1795105741469036324", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/chats", + "observed_timestamp": "1795105741509480285", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "d9f6ce0a-c4c4-459e-a145-aea18fb6086c", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "e51589473e615db9", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "6eba64d9123928f36e93561a07b30456" + }, + "values": [ + [ + "1795105741509231148", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/users", + "observed_timestamp": "1795105740628228687", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "91dfba82-4880-4510-81db-40ca25aa84de", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "32d7b39bc63c8ebb", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "c80ae61ba12fe1836aee1056fb5548eb" + }, + "values": [ + [ + "1795105740627805518", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/users", + "observed_timestamp": "1795105740983969558", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "65bacf54-6b80-4f48-9995-168a3b2fb238", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "762529225451a985", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "76e33b97c228b1db7e33fe9d34904f5a" + }, + "values": [ + [ + "1795105740983833922", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "201", + "http_route": "/api/v1/users", + "observed_timestamp": "1795105741268478200", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "c842d175-ce1e-4845-b45a-e239f8f5e64a", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "1b9526d92920f2ad", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "9163fbdedfafca68627ba223fe28bba8" + }, + "values": [ + [ + "1795105741268321073", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "202", + "http_route": "/api/v1/chats/{chatId}/attachments", + "observed_timestamp": "1795105741585958459", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "51a177f6-23ee-4465-be64-cfb56c61784d", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "fee1784e4be6972e", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "a804478a5b361c9e08beedc7e407aa53" + }, + "values": [ + [ + "1795105741585814815", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "202", + "http_route": "/api/v1/chats/{chatId}/attachments", + "observed_timestamp": "1795105741823189581", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "15739f10-24f8-4537-93d6-5f5864b3ff8e", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "52a0ba967e3b6d20", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "843414a3f045b980fb646379a56749e1" + }, + "values": [ + [ + "1795105741822746666", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "http_request_completed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "http_request_method": "POST", + "http_response_status_code": "202", + "http_route": "/api/v1/chats/{chatId}/attachments", + "observed_timestamp": "1795105742006064890", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "request_id": "26844edd-f014-41ab-8874-ec65238e5287", + "scope_name": "com.example.mtusi_messenger.web.RequestIdFilter", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "59742523ba6ea233", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "5fc2d445c547400a96b781a13d094a9c" + }, + "values": [ + [ + "1795105742005907963", + "HTTP request completed" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106055099883340", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "9ef94c2dfd5dc700", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "5844ca96255b98a392df5db3a29d40a8" + }, + "values": [ + [ + "1795106055099749190", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106070136446453", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "80a65d9b24f49f18", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "ba42dccdb514a83aad6100ecfcd1fc23" + }, + "values": [ + [ + "1795106070136269146", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106085147437330", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "1f725818cecaa520", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "2beb7a664f0cd4484e283b3e5e49b64e" + }, + "values": [ + [ + "1795106085147219155", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106100199222665", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "658a3ccf3631d804", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "c25edbcb982def02b77df8f920245e27" + }, + "values": [ + [ + "1795106100198970937", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106115208537115", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "712d5883aaff696a", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "51166fb04358dbe027aa1c5d13593dd7" + }, + "values": [ + [ + "1795106115208201009", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106130254892042", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "36df30ebfe4377eb", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "e786a3f5a7e892ca4c1889a4d2aeadac" + }, + "values": [ + [ + "1795106130254737132", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106145263363158", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "0aae595402e202f0", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "07c677e7cd721a6249ffa192c62f6173" + }, + "values": [ + [ + "1795106145263218396", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106160307529059", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "99aef5f47b422128", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "f13e7bea74bee0bf26a0da77ae9d38c7" + }, + "values": [ + [ + "1795106160307413194", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106175320555323", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "284c79726490ba64", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "92546bf73382201def374c3755e00528" + }, + "values": [ + [ + "1795106175320422613", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "event": "queue_metrics_sample_failed", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106190369044151", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "2551a3e9bcf9ed2a", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "afd728d6ccafe1eb28ee831cda9ba166" + }, + "values": [ + [ + "1795106190368620943", + "Queue telemetry sample failed: CannotGetJdbcConnectionException" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "exception_message": "The connection attempt failed.", + "exception_stacktrace": "org.postgresql.util.PSQLException: The connection attempt failed.\n\tat org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:439)\n\tat org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:52)\n\tat or", + "exception_type": "org.postgresql.util.PSQLException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106101349911720", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "b3f2d793c34347c1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "696ba634a1655319ddc27bdcd88edb88" + }, + "values": [ + [ + "1795106101349307682", + "HikariPool-1 - Pool is empty, failed to create/setup connection (f2865ffc-d9e1-4764-b77f-f348b6292b9a)" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "exception_message": "The connection attempt failed.", + "exception_stacktrace": "org.postgresql.util.PSQLException: The connection attempt failed.\n\tat org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:439)\n\tat org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:52)\n\tat or", + "exception_type": "org.postgresql.util.PSQLException", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106168453905293", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "b3f2d793c34347c1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "696ba634a1655319ddc27bdcd88edb88" + }, + "values": [ + [ + "1795106168452959552", + "HikariPool-1 - Pool is empty, failed to create/setup connection (b55754ce-9d70-43fa-b01a-c44b0ddd4339)" + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106040101680090", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "b3f2d793c34347c1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "696ba634a1655319ddc27bdcd88edb88" + }, + "values": [ + [ + "1795106040101112208", + "HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@55afbf50 (This connection has been closed.). Possibly consider using a shorter maxLifetime value." + ] + ] + }, + { + "stream": { + "container_id": "8832436ad82d7ff7c93eca3dde08c214c8ab61f4911adc4c888fb5c3f8945f43", + "deployment_environment_name": "lab", + "detected_level": "warn", + "flags": "3", + "host_arch": "amd64", + "host_name": "8832436ad82d", + "observed_timestamp": "1795106040106586151", + "os_description": "Linux 5.15.167.4-microsoft-standard-WSL2", + "os_type": "linux", + "os_version": "5.15.167.4-microsoft-standard-WSL2", + "process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]", + "process_executable_path": "/opt/java/openjdk/bin/java", + "process_pid": "7", + "process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7", + "process_runtime_name": "OpenJDK Runtime Environment", + "process_runtime_version": "23.0.2+7", + "scope_name": "com.zaxxer.hikari.pool.PoolBase", + "service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484", + "service_name": "messenger-api", + "service_version": "0.0.1", + "severity_number": "13", + "severity_text": "WARN", + "span_id": "b3f2d793c34347c1", + "telemetry_distro_name": "opentelemetry-java-instrumentation", + "telemetry_distro_version": "2.33.1", + "telemetry_sdk_language": "java", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.67.0", + "trace_id": "696ba634a1655319ddc27bdcd88edb88" + }, + "values": [ + [ + "1795106040106457892", + "HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@2bb43502 (This connection has been closed.). Possibly consider using a shorter maxLifetime value." + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 319060, + "linesProcessedPerSecond": 1131, + "totalBytesProcessed": 12974, + "totalLinesProcessed": 46, + "execTime": 0.040663, + "queueTime": 0.000579, + "subqueries": 0, + "totalEntriesReturned": 46, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 46, + "totalStructuredMetadataBytesProcessed": 10624 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "querierExecTime": 0.036878 + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 1, + "totalBatches": 2, + "totalLinesSent": 46, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 12974, + "headChunkLines": 46, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 46, + "headChunkStructuredMetadataBytes": 10624, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 414558, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "recvWaitTime": 0.007136 + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false, + "totalStreams": 0, + "chunkRefsLookupTime": 0, + "bloomFilterTime": 0 + } + } + } +} diff --git a/evidence/lab6-20261119-193359-database-stop/recovery-observation.json b/evidence/lab6-20261119-193359-database-stop/recovery-observation.json new file mode 100644 index 0000000..701e2d1 --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/recovery-observation.json @@ -0,0 +1,26 @@ +{ + "captured_at": "2026-11-19T16:41:55.2729486+00:00", + "readiness_recovered": true, + "alerts": { + "status": "success", + "data": { + "alerts": [] + } + }, + "alert_resolved": true, + "api_5xx_ratio": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1795106515.309, + "0" + ] + } + ] + } + } +} diff --git a/evidence/lab6-20261119-193359-database-stop/tempo-http-errors.json b/evidence/lab6-20261119-193359-database-stop/tempo-http-errors.json new file mode 100644 index 0000000..92e10aa --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/tempo-http-errors.json @@ -0,0 +1,3295 @@ +{ + "traces": [ + { + "traceID": "e7a11a65cfb4123844957f45b485cd8a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106176736455417", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "37a325c476de4f8b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106176736455417", + "durationNanos": "5005258327", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "37a325c476de4f8b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106176736455417", + "durationNanos": "5005258327", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "e069be20481578619848d640c4148da5", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106174522076425", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "09b0d96acbf2ce58", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106174522076425", + "durationNanos": "5004094655", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "09b0d96acbf2ce58", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106174522076425", + "durationNanos": "5004094655", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "dd0d24c0488bb5d36a0882deaba3bba1", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106171722378319", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "acf7fade939569d1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106171722378319", + "durationNanos": "5005295830", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "acf7fade939569d1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106171722378319", + "durationNanos": "5005295830", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "b794c7c901e5b1f2fcae8bf3ac7ba4e8", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106170485345197", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "172ecd9ae93d8ff1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106170485345197", + "durationNanos": "5003315314", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "172ecd9ae93d8ff1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106170485345197", + "durationNanos": "5003315314", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "f79f6f6cdf366dfb8f35457d5a05c5a3", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106166706110114", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "4f2379cfa374060c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106166706110114", + "durationNanos": "5005724417", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "4f2379cfa374060c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106166706110114", + "durationNanos": "5005724417", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "aa6c727d0169f86362bc3f516ae693f5", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106166444704758", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "68ce876828aca735", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106166444704758", + "durationNanos": "5004232559", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "68ce876828aca735", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106166444704758", + "durationNanos": "5004232559", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "a0cad96c720191f2032d3989107ed4ae", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106162395762582", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "42a41e8f1d4816d8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106162395762582", + "durationNanos": "5003688627", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "42a41e8f1d4816d8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106162395762582", + "durationNanos": "5003688627", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "3c7ba1e846da1ec51d2d5e8ebd4a83a0", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106161690959703", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "8234680b16cd920c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106161690959703", + "durationNanos": "5005373008", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "8234680b16cd920c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106161690959703", + "durationNanos": "5005373008", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "dc8f87a5e6351be126fb00b37f6dcdc4", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106158346427601", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "a9c008575c858390", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106158346427601", + "durationNanos": "5004414016", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "a9c008575c858390", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106158346427601", + "durationNanos": "5004414016", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "b911dc312a39ff3afe25f7d4a3916808", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106156658891361", + "durationMs": 5006, + "spanSet": { + "spans": [ + { + "spanID": "b2ba681ab6d57b0f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106156658891361", + "durationNanos": "5006713214", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "b2ba681ab6d57b0f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106156658891361", + "durationNanos": "5006713214", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2dc196b2bb6410d684e24f330c6ac65d", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106154308772679", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "cb035ce55c5cf2a8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106154308772679", + "durationNanos": "5004726077", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "cb035ce55c5cf2a8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106154308772679", + "durationNanos": "5004726077", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "c1214a95d972983c86a83ea124b3e523", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106151630405320", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "ddfb36b3e92a63ac", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106151630405320", + "durationNanos": "5003777498", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ddfb36b3e92a63ac", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106151630405320", + "durationNanos": "5003777498", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "55233d39f125925616fc8172285ba297", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106150254667069", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "e3d3a2dc000298a3", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106150254667069", + "durationNanos": "5004256092", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "e3d3a2dc000298a3", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106150254667069", + "durationNanos": "5004256092", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "e1d9f00ab6fb84db9abc9a395dd1ad4a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106146573837600", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "ecf680c71524257f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106146573837600", + "durationNanos": "5004239512", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ecf680c71524257f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106146573837600", + "durationNanos": "5004239512", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "e54f0dafd05e9910ac572dbb9e57b732", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106146183032415", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "bb5ac28fb6fe8511", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106146183032415", + "durationNanos": "5005861206", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "bb5ac28fb6fe8511", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106146183032415", + "durationNanos": "5005861206", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "1cdc29bcf6f8156c096d7280eeee8426", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106142148720234", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "5bde68ae49dd16d6", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106142148720234", + "durationNanos": "5003785756", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "5bde68ae49dd16d6", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106142148720234", + "durationNanos": "5003785756", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "f274df3f64524c7eae27b86500338f1e", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106141551466389", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "16543d19f9d640e5", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106141551466389", + "durationNanos": "5003445745", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "16543d19f9d640e5", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106141551466389", + "durationNanos": "5003445745", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "e68ecacd1b6bd7e38de1ca786c6d288c", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106138113608920", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "efea3a803da4f400", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106138113608920", + "durationNanos": "5004212573", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "efea3a803da4f400", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106138113608920", + "durationNanos": "5004212573", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "cb9fae08e4d27a35e2f25f0efda7ab2f", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106136537774608", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "1ef9db13d5adae9d", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106136537774608", + "durationNanos": "5004866150", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "1ef9db13d5adae9d", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106136537774608", + "durationNanos": "5004866150", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "8c4f504afb511c28909e976e32357bf7", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106134080435273", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "5d1a7656cf89cab5", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106134080435273", + "durationNanos": "5005063072", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "5d1a7656cf89cab5", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106134080435273", + "durationNanos": "5005063072", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "a0f83ec29dff8fad52ce056fc90d4562", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106131504090227", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "9dec8ef12e35bae8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106131504090227", + "durationNanos": "5004069670", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "9dec8ef12e35bae8", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106131504090227", + "durationNanos": "5004069670", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "21debfd7f51492c02f88c0a0dee32133", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106130038182297", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "283722612107ada9", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106130038182297", + "durationNanos": "5003708438", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "283722612107ada9", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106130038182297", + "durationNanos": "5003708438", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "3483b6e603839f9cdbb6e6aef2d96868", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106126467645944", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "ad3c4af3624ec0e0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106126467645944", + "durationNanos": "5005971237", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ad3c4af3624ec0e0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106126467645944", + "durationNanos": "5005971237", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "7c33de4ec59a12bded0da8b005a5b33b", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106125993800710", + "durationMs": 5007, + "spanSet": { + "spans": [ + { + "spanID": "6d30d04b349e111b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106125993800710", + "durationNanos": "5007079928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "6d30d04b349e111b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106125993800710", + "durationNanos": "5007079928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "931071d64fc7ef52f5147938dc141113", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106121956239812", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "9f897e30a67dd1f2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106121956239812", + "durationNanos": "5004599736", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "9f897e30a67dd1f2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106121956239812", + "durationNanos": "5004599736", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2a8addf7487fa15a8f5f95d14961df86", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106121425333978", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "8f51735ec4714b31", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106121425333978", + "durationNanos": "5004880223", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "8f51735ec4714b31", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106121425333978", + "durationNanos": "5004880223", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "66d34436c9a1168ba560f6c3b95c14dd", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106117872279557", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "d472792741ce6a52", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106117872279557", + "durationNanos": "5004408784", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "d472792741ce6a52", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106117872279557", + "durationNanos": "5004408784", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "9ca7ef9a2db251d041c74214a61ec987", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106116359342288", + "durationMs": 5006, + "spanSet": { + "spans": [ + { + "spanID": "d3d3fe05c5c5a351", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106116359342288", + "durationNanos": "5006616428", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "d3d3fe05c5c5a351", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106116359342288", + "durationNanos": "5006616428", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "fd0de84f0a1d3c4ef6ad15f468ce86fc", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106113832311922", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "0a5efd260bb52b29", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106113832311922", + "durationNanos": "5005074027", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "0a5efd260bb52b29", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106113832311922", + "durationNanos": "5005074027", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "8187c95b13b9e5c7ec0858b25a4e9c8f", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106111334883139", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "a766979195b0532c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106111334883139", + "durationNanos": "5004406216", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "a766979195b0532c", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106111334883139", + "durationNanos": "5004406216", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "f6d540136238639bc25691f9e3f27aa1", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106109800056703", + "durationMs": 5007, + "spanSet": { + "spans": [ + { + "spanID": "c9fe0d7f61f37072", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106109800056703", + "durationNanos": "5007810329", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "c9fe0d7f61f37072", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106109800056703", + "durationNanos": "5007810329", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "a211015095a321e3866bc0f747da7c7a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106106310854143", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "fa95d9000ee9ceeb", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106106310854143", + "durationNanos": "5004768175", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "fa95d9000ee9ceeb", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106106310854143", + "durationNanos": "5004768175", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "4c44d4580dac7d4933168722827b6ddc", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106105746455126", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "20b3bf28d5bf4b22", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106105746455126", + "durationNanos": "5003942409", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "20b3bf28d5bf4b22", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106105746455126", + "durationNanos": "5003942409", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "91712fd57899feb4da3e88e57a56080e", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106101712192758", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "f50c9525a2c1cbf0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106101712192758", + "durationNanos": "5004299624", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "f50c9525a2c1cbf0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106101712192758", + "durationNanos": "5004299624", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2f98e1a58c4433c0c24d0f5dbb58b3cc", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106101283917022", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "32f11794c2d483e6", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106101283917022", + "durationNanos": "5003148913", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "32f11794c2d483e6", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106101283917022", + "durationNanos": "5003148913", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "532cc1e5c8630d0ec7fdf024d0a1ef45", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106097676696312", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "83fc4205c863c870", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106097676696312", + "durationNanos": "5004829928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "83fc4205c863c870", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106097676696312", + "durationNanos": "5004829928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "957b461c9942de69c578daad1dca3fc8", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106096254085907", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "1a9ddd0d1e1698c0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106096254085907", + "durationNanos": "5004801511", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "1a9ddd0d1e1698c0", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106096254085907", + "durationNanos": "5004801511", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "8390167076b47b687a15bd15a742ba99", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106093636592173", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "bbdebb40c99ec555", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106093636592173", + "durationNanos": "5003255321", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "bbdebb40c99ec555", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106093636592173", + "durationNanos": "5003255321", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "8944b195af23df696779d965a54f4275", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106091223933032", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "be34c847690259f2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106091223933032", + "durationNanos": "5004320015", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "be34c847690259f2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106091223933032", + "durationNanos": "5004320015", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "b54edd1d331102cafc309dfa63d4448d", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106089571078515", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "1427b0a833638cef", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106089571078515", + "durationNanos": "5004597845", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "1427b0a833638cef", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106089571078515", + "durationNanos": "5004597845", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "c8f01c5211c5863a4ad1f77478053c35", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106086101416739", + "durationMs": 5007, + "spanSet": { + "spans": [ + { + "spanID": "e42d507121aa4eb2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106086101416739", + "durationNanos": "5007235465", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "e42d507121aa4eb2", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106086101416739", + "durationNanos": "5007235465", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "35de7ce1dc61930491f5c18cd0b599f3", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106085527476586", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "2fcbe67fbc87cd89", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106085527476586", + "durationNanos": "5004586894", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "2fcbe67fbc87cd89", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106085527476586", + "durationNanos": "5004586894", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "eb2c7fe45f545c5152c2a476eb9b056b", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106081494712161", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "ee82cbd254e92871", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106081494712161", + "durationNanos": "5004585539", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ee82cbd254e92871", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106081494712161", + "durationNanos": "5004585539", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "de93e8708657295755397f42e47a66a1", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106081073037817", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "4becdba638df1a68", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106081073037817", + "durationNanos": "5003890025", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "4becdba638df1a68", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106081073037817", + "durationNanos": "5003890025", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2c87be2e96e5b34acf950cd96865e64f", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106077466545152", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "53496e4387bf5dc1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106077466545152", + "durationNanos": "5005037096", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "53496e4387bf5dc1", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106077466545152", + "durationNanos": "5005037096", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "60449c98874f99e2cadf886f2a7f5c06", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106076038740400", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "2e11ae28af2bc97e", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106076038740400", + "durationNanos": "5004265415", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "2e11ae28af2bc97e", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106076038740400", + "durationNanos": "5004265415", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "4c3237ad4631352d021928bd36fb230a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106073418985713", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "22211f273340af68", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106073418985713", + "durationNanos": "5003709903", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "22211f273340af68", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106073418985713", + "durationNanos": "5003709903", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "6d29eedc615212718c00d6312b729bba", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106071015091347", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "5d04190190416a21", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106071015091347", + "durationNanos": "5004997917", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "5d04190190416a21", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106071015091347", + "durationNanos": "5004997917", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "e00f0526f9b9e6f92f272cfe73442238", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106069383706345", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "e585fb7cfc27f143", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106069383706345", + "durationNanos": "5003977885", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "e585fb7cfc27f143", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106069383706345", + "durationNanos": "5003977885", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "998b38ac046bdcaf0ebfab9efc180552", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106065990178355", + "durationMs": 5002, + "spanSet": { + "spans": [ + { + "spanID": "adc73a2f14b4b47b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106065990178355", + "durationNanos": "5002913253", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "adc73a2f14b4b47b", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106065990178355", + "durationNanos": "5002913253", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "119ac59d1d452f6f74c31dd078651c5d", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106065343997266", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "8c6900fe91f31401", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106065343997266", + "durationNanos": "5004397169", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "8c6900fe91f31401", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106065343997266", + "durationNanos": "5004397169", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "6158c454016da140cdaddd5f20469b5a", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106061317773865", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "49c7a878a0b02c60", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106061317773865", + "durationNanos": "5004899095", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "49c7a878a0b02c60", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106061317773865", + "durationNanos": "5004899095", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "9efa210684c501e4a2a99ca741881a36", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106060957346012", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "469d9efa5a40698f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106060957346012", + "durationNanos": "5005329603", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "469d9efa5a40698f", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106060957346012", + "durationNanos": "5005329603", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "7b093d3209e77109bb6e31a37c2b15c8", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106057252422726", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "4a1cee8a2692402a", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106057252422726", + "durationNanos": "5003451588", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "4a1cee8a2692402a", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106057252422726", + "durationNanos": "5003451588", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2defd07fe08ca1fd755bfecc95d6432c", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106055893155959", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "f06a4cf746123e99", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106055893155959", + "durationNanos": "5005425109", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "f06a4cf746123e99", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106055893155959", + "durationNanos": "5005425109", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "7d8bcd2e55b6fe79c6566c453d33ad5f", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106053228899767", + "durationMs": 5005, + "spanSet": { + "spans": [ + { + "spanID": "8d18fc820120f6e9", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106053228899767", + "durationNanos": "5005231001", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "8d18fc820120f6e9", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106053228899767", + "durationNanos": "5005231001", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "cac1963ac5c9c7f414a5b8dc592f0ca8", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106050860522346", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "7f12f400ceb7a440", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106050860522346", + "durationNanos": "5004724928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "7f12f400ceb7a440", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106050860522346", + "durationNanos": "5004724928", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "c13cbf3396b35a9fa945c3fc2319b6bc", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106049172478493", + "durationMs": 5007, + "spanSet": { + "spans": [ + { + "spanID": "0c47fb32f4bdcc35", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106049172478493", + "durationNanos": "5007299574", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "0c47fb32f4bdcc35", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106049172478493", + "durationNanos": "5007299574", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "4c3852f2bce403aadf6219977e7bba55", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106045817966091", + "durationMs": 5004, + "spanSet": { + "spans": [ + { + "spanID": "5542a5f3584e0c08", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106045817966091", + "durationNanos": "5004717644", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "5542a5f3584e0c08", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106045817966091", + "durationNanos": "5004717644", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "d22d46457df40f7a763c2c48a5c34883", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106045101137042", + "durationMs": 5003, + "spanSet": { + "spans": [ + { + "spanID": "865649a9d4da0520", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106045101137042", + "durationNanos": "5003679952", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "865649a9d4da0520", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106045101137042", + "durationNanos": "5003679952", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "3ec6d4803eb529a57a23bc8577644c47", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106041070668438", + "durationMs": 5007, + "spanSet": { + "spans": [ + { + "spanID": "f49c1f8fa3ec6432", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106041070668438", + "durationNanos": "5007107139", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "f49c1f8fa3ec6432", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106041070668438", + "durationNanos": "5007107139", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + }, + { + "traceID": "2b922f1ec502c6c517413367ce734bc7", + "rootServiceName": "messenger-api", + "rootTraceName": "GET /health/ready", + "startTimeUnixNano": "1795106040595941449", + "durationMs": 5012, + "spanSet": { + "spans": [ + { + "spanID": "399a8f091b84cacc", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106040595941449", + "durationNanos": "5012750715", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "399a8f091b84cacc", + "name": "GET /health/ready", + "startTimeUnixNano": "1795106040595941449", + "durationNanos": "5012750715", + "attributes": [ + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 2, + "errorCount": 1 + } + } + } + ], + "metrics": { + "inspectedBytes": "256559", + "completedJobs": 3, + "totalJobs": 3 + } +} diff --git a/evidence/lab6-20261119-193359-database-stop/tempo-postgres-errors.json b/evidence/lab6-20261119-193359-database-stop/tempo-postgres-errors.json new file mode 100644 index 0000000..04ce002 --- /dev/null +++ b/evidence/lab6-20261119-193359-database-stop/tempo-postgres-errors.json @@ -0,0 +1,361 @@ +{ + "traces": [ + { + "traceID": "71d67516856cb51d3fd6187fdf2817f", + "rootServiceName": "messenger-worker", + "rootTraceName": "messenger", + "startTimeUnixNano": "1795106139953505266", + "durationMs": 1, + "spanSet": { + "spans": [ + { + "spanID": "03631e0ae3ce65a9", + "startTimeUnixNano": "1795106139953505266", + "durationNanos": "1393486", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "03631e0ae3ce65a9", + "startTimeUnixNano": "1795106139953505266", + "durationNanos": "1393486", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-worker": { + "spanCount": 1, + "errorCount": 1 + } + } + }, + { + "traceID": "cc2248aee1659a6529b09fd076ff8e02", + "rootServiceName": "messenger-worker", + "rootTraceName": "messenger", + "startTimeUnixNano": "1795106052953980568", + "durationMs": 1, + "spanSet": { + "spans": [ + { + "spanID": "7ffc924466fe3b7f", + "startTimeUnixNano": "1795106052953980568", + "durationNanos": "1763744", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "7ffc924466fe3b7f", + "startTimeUnixNano": "1795106052953980568", + "durationNanos": "1763744", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-worker": { + "spanCount": 1, + "errorCount": 1 + } + } + }, + { + "traceID": "9e389058dfdf137e833785ddbb0d1f6f", + "rootServiceName": "messenger-worker", + "rootTraceName": "messenger", + "startTimeUnixNano": "1795106043478695215", + "spanSet": { + "spans": [ + { + "spanID": "c89fe13bd4a61fa4", + "startTimeUnixNano": "1795106043478695215", + "durationNanos": "953569", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "c89fe13bd4a61fa4", + "startTimeUnixNano": "1795106043478695215", + "durationNanos": "953569", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-worker": { + "spanCount": 1, + "errorCount": 1 + } + } + }, + { + "traceID": "6a35aedc38417630096d551d8d504d6e", + "rootServiceName": "messenger-worker", + "rootTraceName": "messenger", + "startTimeUnixNano": "1795106041627390774", + "durationMs": 18, + "spanSet": { + "spans": [ + { + "spanID": "d68a32afe63bf2f2", + "startTimeUnixNano": "1795106041627390774", + "durationNanos": "18820885", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "d68a32afe63bf2f2", + "startTimeUnixNano": "1795106041627390774", + "durationNanos": "18820885", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 1 + } + ], + "serviceStats": { + "messenger-worker": { + "spanCount": 1, + "errorCount": 1 + } + } + }, + { + "traceID": "696ba634a1655319ddc27bdcd88edb88", + "rootServiceName": "messenger-api", + "rootTraceName": "ExpiredLeaseReaper.recoverExpired", + "startTimeUnixNano": "1795106040066361178", + "durationMs": 5017, + "spanSet": { + "spans": [ + { + "spanID": "bb6cc6ecea134b52", + "startTimeUnixNano": "1795106040066669462", + "durationNanos": "33951674", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + }, + { + "spanID": "27022a80d9f3f062", + "startTimeUnixNano": "1795106040104844852", + "durationNanos": "1526278", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 2 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "bb6cc6ecea134b52", + "startTimeUnixNano": "1795106040066669462", + "durationNanos": "33951674", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + }, + { + "spanID": "27022a80d9f3f062", + "startTimeUnixNano": "1795106040104844852", + "durationNanos": "1526278", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "status", + "value": { + "stringValue": "error" + } + } + ] + } + ], + "matched": 2 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 3, + "errorCount": 3 + } + } + } + ], + "metrics": { + "inspectedBytes": "280709", + "completedJobs": 3, + "totalJobs": 3 + } +} diff --git a/evidence/lab6-20261119-194742-telemetry-backend/collector-samples.json b/evidence/lab6-20261119-194742-telemetry-backend/collector-samples.json new file mode 100644 index 0000000..c6d6f73 --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/collector-samples.json @@ -0,0 +1,142 @@ +[ + { + "captured_at": "2026-11-19T16:47:45.9583664+00:00", + "collector_memory": "105.9MiB", + "collector_memory_bytes": 111044198, + "collector_memory_percent": "20.68%", + "exporter_queue_size": 0.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:01.7530524+00:00", + "collector_memory": "105.9MiB", + "collector_memory_bytes": 111044198, + "collector_memory_percent": "20.69%", + "exporter_queue_size": 2.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:08.4652103+00:00", + "collector_memory": "105.9MiB", + "collector_memory_bytes": 111044198, + "collector_memory_percent": "20.69%", + "exporter_queue_size": 4.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:15.1961009+00:00", + "collector_memory": "106.1MiB", + "collector_memory_bytes": 111253914, + "collector_memory_percent": "20.71%", + "exporter_queue_size": 11.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:21.8707908+00:00", + "collector_memory": "110.7MiB", + "collector_memory_bytes": 116077363, + "collector_memory_percent": "21.61%", + "exporter_queue_size": 11.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:28.5455460+00:00", + "collector_memory": "114.7MiB", + "collector_memory_bytes": 120271667, + "collector_memory_percent": "22.40%", + "exporter_queue_size": 20.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:35.2646871+00:00", + "collector_memory": "120.8MiB", + "collector_memory_bytes": 126667981, + "collector_memory_percent": "23.59%", + "exporter_queue_size": 30.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:42.0621346+00:00", + "collector_memory": "137MiB", + "collector_memory_bytes": 143654912, + "collector_memory_percent": "26.77%", + "exporter_queue_size": 30.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 0.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:48.8023730+00:00", + "collector_memory": "139MiB", + "collector_memory_bytes": 145752064, + "collector_memory_percent": "27.14%", + "exporter_queue_size": 37.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 35.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:48:55.5413710+00:00", + "collector_memory": "139.3MiB", + "collector_memory_bytes": 146066637, + "collector_memory_percent": "27.20%", + "exporter_queue_size": 47.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 35.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:49:02.2702013+00:00", + "collector_memory": "157.4MiB", + "collector_memory_bytes": 165045862, + "collector_memory_percent": "30.74%", + "exporter_queue_size": 53.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 1805.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:49:08.9583898+00:00", + "collector_memory": "162.5MiB", + "collector_memory_bytes": 170393600, + "collector_memory_percent": "31.74%", + "exporter_queue_size": 53.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 1805.0, + "failed_logs_total": 0.0 + }, + { + "captured_at": "2026-11-19T16:49:15.6767766+00:00", + "collector_memory": "162.8MiB", + "collector_memory_bytes": 170708173, + "collector_memory_percent": "31.79%", + "exporter_queue_size": 61.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 2852.0, + "failed_logs_total": 318.0 + }, + { + "captured_at": "2026-11-19T16:49:22.4259651+00:00", + "collector_memory": "165.6MiB", + "collector_memory_bytes": 173644186, + "collector_memory_percent": "32.35%", + "exporter_queue_size": 66.0, + "exporter_queue_capacity": 2048.0, + "failed_spans_total": 3364.0, + "failed_logs_total": 926.0 + } +] diff --git a/evidence/lab6-20261119-194742-telemetry-backend/failure-observation.json b/evidence/lab6-20261119-194742-telemetry-backend/failure-observation.json new file mode 100644 index 0000000..d6f7f4a --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/failure-observation.json @@ -0,0 +1,15 @@ +{ + "captured_at": "2026-11-19T16:49:22.5235902+00:00", + "make_test_passed": true, + "message_load_exit_code": 0, + "successful_message_requests": 3060, + "failed_message_requests": 0, + "collector_memory_limit_bytes": 536870912, + "collector_memory_limiter_bytes": 402653184, + "max_collector_memory_bytes": 173644186, + "max_exporter_queue_size": 66, + "exporter_queue_capacity": 2048, + "failed_spans_total": 3364, + "failed_log_records_total": 926, + "bounded": true +} diff --git a/evidence/lab6-20261119-194742-telemetry-backend/make-test.txt b/evidence/lab6-20261119-194742-telemetry-backend/make-test.txt new file mode 100644 index 0000000..d066a62 --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/make-test.txt @@ -0,0 +1,2 @@ +python tests/smoke.py --base-url "http://127.0.0.1:18080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab6-20261119-194742-telemetry-backend/message-load.json b/evidence/lab6-20261119-194742-telemetry-backend/message-load.json new file mode 100644 index 0000000..897a654 --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/message-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.186, + "concurrency": 4, + "successful_requests": 3060, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 40.699, + "successful_latency_ms": { + "p50": 98.635, + "p95": 183.436, + "p99": 198.298 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-194742-telemetry-backend/recovery-confirmed.json b/evidence/lab6-20261119-194742-telemetry-backend/recovery-confirmed.json new file mode 100644 index 0000000..11bd7cf --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/recovery-confirmed.json @@ -0,0 +1,9 @@ +{ + "captured_at": "2026-11-19T16:50:50.5393078+00:00", + "loki_ready": true, + "loki_http_status": 200, + "tempo_ready": true, + "tempo_http_status": 200, + "exporter_queue_size": 0, + "recovery_confirmed": true +} diff --git a/evidence/lab6-20261119-194742-telemetry-backend/recovery-observation.json b/evidence/lab6-20261119-194742-telemetry-backend/recovery-observation.json new file mode 100644 index 0000000..58bc012 --- /dev/null +++ b/evidence/lab6-20261119-194742-telemetry-backend/recovery-observation.json @@ -0,0 +1,6 @@ +{ + "captured_at": "2026-11-19T16:49:54.2198516+00:00", + "loki_ready": false, + "tempo_ready": true, + "exporter_queue_size": 0.0 +} diff --git a/evidence/lab6-20261119-203024-observability-cost/README.md b/evidence/lab6-20261119-203024-observability-cost/README.md new file mode 100644 index 0000000..cb312c0 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/README.md @@ -0,0 +1,22 @@ +# Observability cost — 19.11.2026 + +Команда: + +```powershell +powershell -File scripts/experiment-observability-cost.ps1 ` + -BaseUrl http://127.0.0.1:18080 -LoadSeconds 45 -Concurrency 4 ` + -WarmupSeconds 10 -StatsIntervalSeconds 5 +``` + +Режимы выполнялись последовательно: telemetry off; full 100%/10s; economy 5%/60s; balanced 25%/30s. Перед каждым режимом API и worker пересоздавались, затем выполнялся одинаковый прогрев. `summary.json` и `comparison.csv` содержат итог; `*-result.json`, `*-load.json` и `*-resource-samples.json` — raw данные. Ingest измерен дельтами Collector/Tempo/Loki/Prometheus counters, CPU/RAM — `docker stats`. + +| mode | RPS | p50/p95/p99 ms | trace retention vs full | backend B/request | Collector lost spans | +| --- | ---: | ---: | ---: | ---: | ---: | +| off | 40.547 | 97.186 / 198.133 / 275.316 | 0% | 0 | 0 | +| full | 22.419 | 185.241 / 315.289 / 405.961 | 100% | 6918.72 | 0 | +| economy | 14.622 | 287.141 / 405.639 / 502.152 | 5.52% | 1108.25 | 0 | +| balanced | 16.934 | 209.980 / 391.490 / 409.089 | 23.48% | 2200.52 | 0 | + +`balanced-head-sampling-probe.json` — 40 synthetic `GET /health/not-found` responses со status 500: 8 sampled, 32 unsampled. В sampled example Tempo вернул 200; в unsampled — 404. Evidence не содержит credentials, request bodies или message text. + +Ограничение: это один closed-loop прогон на локальном Docker Desktop. Он годится для измерения порядка накладных расходов и ingest, но не для статистического ранжирования latency близких sampling-профилей. diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-head-sampling-probe.json b/evidence/lab6-20261119-203024-observability-cost/balanced-head-sampling-probe.json new file mode 100644 index 0000000..d3f5e66 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-head-sampling-probe.json @@ -0,0 +1,20 @@ +{ + "captured_at": "2026-11-19T17:50:37.4885344+00:00", + "request_count": 40, + "response_status_counts": { + "500": 40 + }, + "correlated_log_count": 40, + "sampled_log_count": 8, + "unsampled_log_count": 32, + "sampled_ratio": 0.2, + "sampled_example": { + "trace_id": "00d814f7907510c8066b49c89cefd0ec", + "tempo_http_status": 200 + }, + "unsampled_example": { + "trace_id": "495125068d65af4c622b34ada5196db5", + "tempo_http_status": 404 + }, + "interpretation": "Head sampling is decided before the HTTP outcome: a non-2xx log can exist even when its trace was not emitted to Tempo." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-load.json b/evidence/lab6-20261119-203024-observability-cost/balanced-load.json new file mode 100644 index 0000000..1ef2c66 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.352, + "concurrency": 4, + "successful_requests": 768, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 16.934, + "successful_latency_ms": { + "p50": 209.98, + "p95": 391.49, + "p99": 409.089 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-make-test.txt b/evidence/lab6-20261119-203024-observability-cost/balanced-make-test.txt new file mode 100644 index 0000000..d066a62 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-make-test.txt @@ -0,0 +1,2 @@ +python tests/smoke.py --base-url "http://127.0.0.1:18080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-resource-samples.json b/evidence/lab6-20261119-203024-observability-cost/balanced-resource-samples.json new file mode 100644 index 0000000..3d3049a --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-resource-samples.json @@ -0,0 +1,149 @@ +[ + { + "captured_at": "2026-11-19T17:44:40.0143412+00:00", + "service": "otel-collector", + "cpu_percent": 0.22, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:44:40.0143412+00:00", + "service": "api", + "cpu_percent": 23.47, + "memory_bytes": 396256870, + "memory_display": "377.9MiB" + }, + { + "captured_at": "2026-11-19T17:44:40.0143412+00:00", + "service": "worker", + "cpu_percent": 0.41, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:44:47.6725368+00:00", + "service": "otel-collector", + "cpu_percent": 1.0, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:44:47.6725368+00:00", + "service": "api", + "cpu_percent": 77.02, + "memory_bytes": 411461222, + "memory_display": "392.4MiB" + }, + { + "captured_at": "2026-11-19T17:44:47.6725368+00:00", + "service": "worker", + "cpu_percent": 0.26, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:44:55.3352746+00:00", + "service": "otel-collector", + "cpu_percent": 0.58, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:44:55.3352746+00:00", + "service": "api", + "cpu_percent": 82.94, + "memory_bytes": 416808960, + "memory_display": "397.5MiB" + }, + { + "captured_at": "2026-11-19T17:44:55.3352746+00:00", + "service": "worker", + "cpu_percent": 3.12, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:45:03.0055136+00:00", + "service": "otel-collector", + "cpu_percent": 0.77, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:45:03.0055136+00:00", + "service": "api", + "cpu_percent": 79.98, + "memory_bytes": 420059546, + "memory_display": "400.6MiB" + }, + { + "captured_at": "2026-11-19T17:45:03.0055136+00:00", + "service": "worker", + "cpu_percent": 0.28, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:45:10.7228542+00:00", + "service": "otel-collector", + "cpu_percent": 0.56, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:45:10.7228542+00:00", + "service": "api", + "cpu_percent": 77.02, + "memory_bytes": 420478976, + "memory_display": "401MiB" + }, + { + "captured_at": "2026-11-19T17:45:10.7228542+00:00", + "service": "worker", + "cpu_percent": 0.5, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:45:18.3628330+00:00", + "service": "otel-collector", + "cpu_percent": 0.34, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:45:18.3628330+00:00", + "service": "api", + "cpu_percent": 74.77, + "memory_bytes": 423205274, + "memory_display": "403.6MiB" + }, + { + "captured_at": "2026-11-19T17:45:18.3628330+00:00", + "service": "worker", + "cpu_percent": 0.47, + "memory_bytes": 313524224, + "memory_display": "299MiB" + }, + { + "captured_at": "2026-11-19T17:45:26.0533423+00:00", + "service": "otel-collector", + "cpu_percent": 0.33, + "memory_bytes": 155713536, + "memory_display": "148.5MiB" + }, + { + "captured_at": "2026-11-19T17:45:26.0533423+00:00", + "service": "api", + "cpu_percent": 58.16, + "memory_bytes": 426351002, + "memory_display": "406.6MiB" + }, + { + "captured_at": "2026-11-19T17:45:26.0533423+00:00", + "service": "worker", + "cpu_percent": 0.43, + "memory_bytes": 313524224, + "memory_display": "299MiB" + } +] diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-result.json b/evidence/lab6-20261119-203024-observability-cost/balanced-result.json new file mode 100644 index 0000000..1d56ac0 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-result.json @@ -0,0 +1,103 @@ +{ + "mode": { + "name": "balanced", + "sdk_disabled": false, + "sampler_arg": 0.25, + "metric_interval_ms": 30000 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.352, + "concurrency": 4, + "successful_requests": 768, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 16.934, + "successful_latency_ms": { + "p50": 209.98, + "p95": 391.49, + "p99": 409.089 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "ingest_before": { + "captured_at": "2026-11-19T20:44:35.0412998+03:00", + "collector_accepted_spans": 74001.0, + "collector_accepted_logs": 5524.0, + "collector_accepted_metric_points": 97066.0, + "collector_sent_spans": 70630.0, + "collector_sent_logs": 4389.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 28939480.0, + "tempo_ingest_spans": 56863.0, + "loki_line_bytes": 1491972.0, + "loki_metadata_bytes": 1398836.0, + "loki_lines": 4053.0, + "prometheus_appended_samples": 1157813.0 + }, + "ingest_after": { + "captured_at": "2026-11-19T20:45:43.0724595+03:00", + "collector_accepted_spans": 76201.0, + "collector_accepted_logs": 6299.0, + "collector_accepted_metric_points": 97304.0, + "collector_sent_spans": 72837.0, + "collector_sent_logs": 5164.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 30069342.0, + "tempo_ingest_spans": 59070.0, + "loki_line_bytes": 1780565.0, + "loki_metadata_bytes": 1670379.0, + "loki_lines": 4828.0, + "prometheus_appended_samples": 1164131.0 + }, + "ingest_delta": { + "collector_accepted_spans": 2200.0, + "collector_accepted_logs": 775.0, + "collector_accepted_metric_points": 238.0, + "collector_sent_spans": 2207.0, + "collector_sent_logs": 775.0, + "collector_failed_spans": 0.0, + "collector_failed_logs": 0.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 1129862.0, + "tempo_ingest_spans": 2207.0, + "loki_line_bytes": 288593.0, + "loki_metadata_bytes": 271543.0, + "loki_lines": 775.0, + "prometheus_appended_samples": 6318.0, + "loki_total_bytes": 560136.0, + "backend_total_bytes": 1689998.0 + }, + "resources": { + "api": { + "sample_count": 7, + "cpu_avg_percent": 67.623, + "cpu_peak_percent": 82.94, + "ram_avg_bytes": 416374550, + "ram_peak_bytes": 426351002 + }, + "worker": { + "sample_count": 7, + "cpu_avg_percent": 0.781, + "cpu_peak_percent": 3.12, + "ram_avg_bytes": 313524224, + "ram_peak_bytes": 313524224 + }, + "collector": { + "sample_count": 7, + "cpu_avg_percent": 0.543, + "cpu_peak_percent": 1.0, + "ram_avg_bytes": 155713536, + "ram_peak_bytes": 155713536 + } + } +} diff --git a/evidence/lab6-20261119-203024-observability-cost/balanced-warmup.json b/evidence/lab6-20261119-203024-observability-cost/balanced-warmup.json new file mode 100644 index 0000000..fb57b65 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/balanced-warmup.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.228, + "concurrency": 2, + "successful_requests": 99, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 9.679, + "successful_latency_ms": { + "p50": 188.687, + "p95": 391.506, + "p99": 1458.041 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/comparison.csv b/evidence/lab6-20261119-203024-observability-cost/comparison.csv new file mode 100644 index 0000000..5a546ea --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/comparison.csv @@ -0,0 +1,5 @@ +"mode","sdk_disabled","trace_sample_ratio","metric_interval_ms","successful_requests","failed_requests","successful_rps","rps_change_vs_off_percent","latency_p50_ms","latency_p95_ms","latency_p99_ms","p95_change_vs_off_percent","api_cpu_avg_percent","api_ram_peak_bytes","worker_cpu_avg_percent","worker_ram_peak_bytes","collector_cpu_avg_percent","collector_ram_peak_bytes","accepted_spans","accepted_logs","accepted_metric_points","spans_per_successful_request","trace_retained_percent_of_full_normalized","estimated_spans_not_emitted_by_sdk","collector_transport_lost_spans","tempo_ingest_bytes","loki_ingest_bytes","backend_trace_and_log_ingest_bytes","tempo_bytes_per_successful_request","loki_bytes_per_successful_request","backend_bytes_per_successful_request","prometheus_appended_samples" +"off","True","0","60000","1840","0","40.547","0","97.186","198.133","275.316","0","66.496","380633088","0.33","208876339","0.054","151519232","0","0","0","0","0","22447","0","0","0","0","0","0","0","6643" +"full","False","1","10000","1012","0","22.419","-44.71","185.241","315.289","405.961","59.13","69.176","425407283","0.544","323171123","2.169","157496115","12346","1019","711","12.1996","100","0","0","6286284","715458","7001742","6211.74","706.97","6918.72","7479" +"economy","False","0.05","60000","664","0","14.622","-63.94","287.141","405.639","502.152","104.73","68.564","431803597","0.517","293601280","0.877","159593267","447","671","115","0.6732","5.52","7654","0","241260","494616","735876","363.34","744.9","1108.25","6008" +"balanced","False","0.25","30000","768","0","16.934","-58.24","209.98","391.49","409.089","97.59","67.623","426351002","0.781","313524224","0.543","155713536","2200","775","238","2.8646","23.48","7169","0","1129862","560136","1689998","1471.17","729.34","2200.52","6318" diff --git a/evidence/lab6-20261119-203024-observability-cost/economy-load.json b/evidence/lab6-20261119-203024-observability-cost/economy-load.json new file mode 100644 index 0000000..c08af24 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/economy-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.411, + "concurrency": 4, + "successful_requests": 664, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 14.622, + "successful_latency_ms": { + "p50": 287.141, + "p95": 405.639, + "p99": 502.152 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/economy-resource-samples.json b/evidence/lab6-20261119-203024-observability-cost/economy-resource-samples.json new file mode 100644 index 0000000..d0968d1 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/economy-resource-samples.json @@ -0,0 +1,149 @@ +[ + { + "captured_at": "2026-11-19T17:40:39.6397582+00:00", + "service": "otel-collector", + "cpu_percent": 0.11, + "memory_bytes": 159593267, + "memory_display": "152.2MiB" + }, + { + "captured_at": "2026-11-19T17:40:39.6397582+00:00", + "service": "api", + "cpu_percent": 19.43, + "memory_bytes": 416599245, + "memory_display": "397.3MiB" + }, + { + "captured_at": "2026-11-19T17:40:39.6397582+00:00", + "service": "worker", + "cpu_percent": 0.35, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:40:47.3615591+00:00", + "service": "otel-collector", + "cpu_percent": 0.41, + "memory_bytes": 159593267, + "memory_display": "152.2MiB" + }, + { + "captured_at": "2026-11-19T17:40:47.3615591+00:00", + "service": "api", + "cpu_percent": 80.42, + "memory_bytes": 418801254, + "memory_display": "399.4MiB" + }, + { + "captured_at": "2026-11-19T17:40:47.3615591+00:00", + "service": "worker", + "cpu_percent": 0.57, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:40:55.0298538+00:00", + "service": "otel-collector", + "cpu_percent": 0.03, + "memory_bytes": 159593267, + "memory_display": "152.2MiB" + }, + { + "captured_at": "2026-11-19T17:40:55.0298538+00:00", + "service": "api", + "cpu_percent": 72.47, + "memory_bytes": 427714150, + "memory_display": "407.9MiB" + }, + { + "captured_at": "2026-11-19T17:40:55.0298538+00:00", + "service": "worker", + "cpu_percent": 0.66, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:41:02.7166071+00:00", + "service": "otel-collector", + "cpu_percent": 0.87, + "memory_bytes": 159593267, + "memory_display": "152.2MiB" + }, + { + "captured_at": "2026-11-19T17:41:02.7166071+00:00", + "service": "api", + "cpu_percent": 77.82, + "memory_bytes": 431803597, + "memory_display": "411.8MiB" + }, + { + "captured_at": "2026-11-19T17:41:02.7166071+00:00", + "service": "worker", + "cpu_percent": 0.47, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:41:10.3853303+00:00", + "service": "otel-collector", + "cpu_percent": 0.49, + "memory_bytes": 159593267, + "memory_display": "152.2MiB" + }, + { + "captured_at": "2026-11-19T17:41:10.3853303+00:00", + "service": "api", + "cpu_percent": 76.06, + "memory_bytes": 428343296, + "memory_display": "408.5MiB" + }, + { + "captured_at": "2026-11-19T17:41:10.3853303+00:00", + "service": "worker", + "cpu_percent": 0.5, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:41:18.0435483+00:00", + "service": "otel-collector", + "cpu_percent": 0.52, + "memory_bytes": 157810688, + "memory_display": "150.5MiB" + }, + { + "captured_at": "2026-11-19T17:41:18.0435483+00:00", + "service": "api", + "cpu_percent": 76.23, + "memory_bytes": 428238438, + "memory_display": "408.4MiB" + }, + { + "captured_at": "2026-11-19T17:41:18.0435483+00:00", + "service": "worker", + "cpu_percent": 0.42, + "memory_bytes": 293601280, + "memory_display": "280MiB" + }, + { + "captured_at": "2026-11-19T17:41:25.6861454+00:00", + "service": "otel-collector", + "cpu_percent": 3.71, + "memory_bytes": 157810688, + "memory_display": "150.5MiB" + }, + { + "captured_at": "2026-11-19T17:41:25.6861454+00:00", + "service": "api", + "cpu_percent": 77.52, + "memory_bytes": 431174451, + "memory_display": "411.2MiB" + }, + { + "captured_at": "2026-11-19T17:41:25.6861454+00:00", + "service": "worker", + "cpu_percent": 0.65, + "memory_bytes": 293601280, + "memory_display": "280MiB" + } +] diff --git a/evidence/lab6-20261119-203024-observability-cost/economy-result.json b/evidence/lab6-20261119-203024-observability-cost/economy-result.json new file mode 100644 index 0000000..050eabd --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/economy-result.json @@ -0,0 +1,103 @@ +{ + "mode": { + "name": "economy", + "sdk_disabled": false, + "sampler_arg": 0.05, + "metric_interval_ms": 60000 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.411, + "concurrency": 4, + "successful_requests": 664, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 14.622, + "successful_latency_ms": { + "p50": 287.141, + "p95": 405.639, + "p99": 502.152 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "ingest_before": { + "captured_at": "2026-11-19T20:40:34.9641917+03:00", + "collector_accepted_spans": 73201.0, + "collector_accepted_logs": 4719.0, + "collector_accepted_metric_points": 96428.0, + "collector_sent_spans": 69837.0, + "collector_sent_logs": 3584.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 28514676.0, + "tempo_ingest_spans": 56063.0, + "loki_line_bytes": 1173945.0, + "loki_metadata_bytes": 1099842.0, + "loki_lines": 3248.0, + "prometheus_appended_samples": 1133674.0 + }, + "ingest_after": { + "captured_at": "2026-11-19T20:41:42.7120231+03:00", + "collector_accepted_spans": 73648.0, + "collector_accepted_logs": 5390.0, + "collector_accepted_metric_points": 96543.0, + "collector_sent_spans": 70284.0, + "collector_sent_logs": 4255.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 28755936.0, + "tempo_ingest_spans": 56510.0, + "loki_line_bytes": 1428634.0, + "loki_metadata_bytes": 1339769.0, + "loki_lines": 3919.0, + "prometheus_appended_samples": 1139682.0 + }, + "ingest_delta": { + "collector_accepted_spans": 447.0, + "collector_accepted_logs": 671.0, + "collector_accepted_metric_points": 115.0, + "collector_sent_spans": 447.0, + "collector_sent_logs": 671.0, + "collector_failed_spans": 0.0, + "collector_failed_logs": 0.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 241260.0, + "tempo_ingest_spans": 447.0, + "loki_line_bytes": 254689.0, + "loki_metadata_bytes": 239927.0, + "loki_lines": 671.0, + "prometheus_appended_samples": 6008.0, + "loki_total_bytes": 494616.0, + "backend_total_bytes": 735876.0 + }, + "resources": { + "api": { + "sample_count": 7, + "cpu_avg_percent": 68.564, + "cpu_peak_percent": 80.42, + "ram_avg_bytes": 426096347, + "ram_peak_bytes": 431803597 + }, + "worker": { + "sample_count": 7, + "cpu_avg_percent": 0.517, + "cpu_peak_percent": 0.66, + "ram_avg_bytes": 293601280, + "ram_peak_bytes": 293601280 + }, + "collector": { + "sample_count": 7, + "cpu_avg_percent": 0.877, + "cpu_peak_percent": 3.71, + "ram_avg_bytes": 159083959, + "ram_peak_bytes": 159593267 + } + } +} diff --git a/evidence/lab6-20261119-203024-observability-cost/economy-warmup.json b/evidence/lab6-20261119-203024-observability-cost/economy-warmup.json new file mode 100644 index 0000000..00c7df0 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/economy-warmup.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.398, + "concurrency": 2, + "successful_requests": 77, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 7.405, + "successful_latency_ms": { + "p50": 273.64, + "p95": 396.707, + "p99": 872.673 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/full-load.json b/evidence/lab6-20261119-203024-observability-cost/full-load.json new file mode 100644 index 0000000..d75cbb8 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/full-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.14, + "concurrency": 4, + "successful_requests": 1012, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 22.419, + "successful_latency_ms": { + "p50": 185.241, + "p95": 315.289, + "p99": 405.961 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/full-resource-samples.json b/evidence/lab6-20261119-203024-observability-cost/full-resource-samples.json new file mode 100644 index 0000000..fd4f20f --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/full-resource-samples.json @@ -0,0 +1,149 @@ +[ + { + "captured_at": "2026-11-19T17:36:50.6934872+00:00", + "service": "otel-collector", + "cpu_percent": 0.68, + "memory_bytes": 155084390, + "memory_display": "147.9MiB" + }, + { + "captured_at": "2026-11-19T17:36:50.6934872+00:00", + "service": "api", + "cpu_percent": 43.03, + "memory_bytes": 396781158, + "memory_display": "378.4MiB" + }, + { + "captured_at": "2026-11-19T17:36:50.6934872+00:00", + "service": "worker", + "cpu_percent": 0.34, + "memory_bytes": 323066266, + "memory_display": "308.1MiB" + }, + { + "captured_at": "2026-11-19T17:36:58.3607881+00:00", + "service": "otel-collector", + "cpu_percent": 0.6, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:36:58.3607881+00:00", + "service": "api", + "cpu_percent": 81.27, + "memory_bytes": 409573786, + "memory_display": "390.6MiB" + }, + { + "captured_at": "2026-11-19T17:36:58.3607881+00:00", + "service": "worker", + "cpu_percent": 2.09, + "memory_bytes": 323066266, + "memory_display": "308.1MiB" + }, + { + "captured_at": "2026-11-19T17:37:06.0106951+00:00", + "service": "otel-collector", + "cpu_percent": 6.64, + "memory_bytes": 154874675, + "memory_display": "147.7MiB" + }, + { + "captured_at": "2026-11-19T17:37:06.0106951+00:00", + "service": "api", + "cpu_percent": 79.67, + "memory_bytes": 409783501, + "memory_display": "390.8MiB" + }, + { + "captured_at": "2026-11-19T17:37:06.0106951+00:00", + "service": "worker", + "cpu_percent": 0.31, + "memory_bytes": 323066266, + "memory_display": "308.1MiB" + }, + { + "captured_at": "2026-11-19T17:37:13.6552262+00:00", + "service": "otel-collector", + "cpu_percent": 0.27, + "memory_bytes": 155294106, + "memory_display": "148.1MiB" + }, + { + "captured_at": "2026-11-19T17:37:13.6552262+00:00", + "service": "api", + "cpu_percent": 74.69, + "memory_bytes": 418801254, + "memory_display": "399.4MiB" + }, + { + "captured_at": "2026-11-19T17:37:13.6552262+00:00", + "service": "worker", + "cpu_percent": 0.28, + "memory_bytes": 323171123, + "memory_display": "308.2MiB" + }, + { + "captured_at": "2026-11-19T17:37:21.2593950+00:00", + "service": "otel-collector", + "cpu_percent": 0.26, + "memory_bytes": 155294106, + "memory_display": "148.1MiB" + }, + { + "captured_at": "2026-11-19T17:37:21.2593950+00:00", + "service": "api", + "cpu_percent": 77.01, + "memory_bytes": 421632410, + "memory_display": "402.1MiB" + }, + { + "captured_at": "2026-11-19T17:37:21.2593950+00:00", + "service": "worker", + "cpu_percent": 0.24, + "memory_bytes": 323171123, + "memory_display": "308.2MiB" + }, + { + "captured_at": "2026-11-19T17:37:28.9314370+00:00", + "service": "otel-collector", + "cpu_percent": 6.11, + "memory_bytes": 157076685, + "memory_display": "149.8MiB" + }, + { + "captured_at": "2026-11-19T17:37:28.9314370+00:00", + "service": "api", + "cpu_percent": 80.22, + "memory_bytes": 424358707, + "memory_display": "404.7MiB" + }, + { + "captured_at": "2026-11-19T17:37:28.9314370+00:00", + "service": "worker", + "cpu_percent": 0.33, + "memory_bytes": 323171123, + "memory_display": "308.2MiB" + }, + { + "captured_at": "2026-11-19T17:37:36.5264368+00:00", + "service": "otel-collector", + "cpu_percent": 0.62, + "memory_bytes": 157496115, + "memory_display": "150.2MiB" + }, + { + "captured_at": "2026-11-19T17:37:36.5264368+00:00", + "service": "api", + "cpu_percent": 48.34, + "memory_bytes": 425407283, + "memory_display": "405.7MiB" + }, + { + "captured_at": "2026-11-19T17:37:36.5264368+00:00", + "service": "worker", + "cpu_percent": 0.22, + "memory_bytes": 323171123, + "memory_display": "308.2MiB" + } +] diff --git a/evidence/lab6-20261119-203024-observability-cost/full-result.json b/evidence/lab6-20261119-203024-observability-cost/full-result.json new file mode 100644 index 0000000..8542339 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/full-result.json @@ -0,0 +1,103 @@ +{ + "mode": { + "name": "full", + "sdk_disabled": false, + "sampler_arg": 1.0, + "metric_interval_ms": 10000 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.14, + "concurrency": 4, + "successful_requests": 1012, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 22.419, + "successful_latency_ms": { + "p50": 185.241, + "p95": 315.289, + "p99": 405.961 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "ingest_before": { + "captured_at": "2026-11-19T20:36:45.972962+03:00", + "collector_accepted_spans": 60758.0, + "collector_accepted_logs": 3588.0, + "collector_accepted_metric_points": 95359.0, + "collector_sent_spans": 57378.0, + "collector_sent_logs": 2453.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 22180655.0, + "tempo_ingest_spans": 43604.0, + "loki_line_bytes": 747186.0, + "loki_metadata_bytes": 699288.0, + "loki_lines": 2117.0, + "prometheus_appended_samples": 1109213.0 + }, + "ingest_after": { + "captured_at": "2026-11-19T20:37:53.5546074+03:00", + "collector_accepted_spans": 73104.0, + "collector_accepted_logs": 4607.0, + "collector_accepted_metric_points": 96070.0, + "collector_sent_spans": 69432.0, + "collector_sent_logs": 3472.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 28466939.0, + "tempo_ingest_spans": 55981.0, + "loki_line_bytes": 1116124.0, + "loki_metadata_bytes": 1045808.0, + "loki_lines": 3136.0, + "prometheus_appended_samples": 1116692.0 + }, + "ingest_delta": { + "collector_accepted_spans": 12346.0, + "collector_accepted_logs": 1019.0, + "collector_accepted_metric_points": 711.0, + "collector_sent_spans": 12054.0, + "collector_sent_logs": 1019.0, + "collector_failed_spans": 0.0, + "collector_failed_logs": 0.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 6286284.0, + "tempo_ingest_spans": 12377.0, + "loki_line_bytes": 368938.0, + "loki_metadata_bytes": 346520.0, + "loki_lines": 1019.0, + "prometheus_appended_samples": 7479.0, + "loki_total_bytes": 715458.0, + "backend_total_bytes": 7001742.0 + }, + "resources": { + "api": { + "sample_count": 7, + "cpu_avg_percent": 69.176, + "cpu_peak_percent": 81.27, + "ram_avg_bytes": 415191157, + "ram_peak_bytes": 425407283 + }, + "worker": { + "sample_count": 7, + "cpu_avg_percent": 0.544, + "cpu_peak_percent": 2.09, + "ram_avg_bytes": 323126184, + "ram_peak_bytes": 323171123 + }, + "collector": { + "sample_count": 7, + "cpu_avg_percent": 2.169, + "cpu_peak_percent": 6.64, + "ram_avg_bytes": 155234187, + "ram_peak_bytes": 157496115 + } + } +} diff --git a/evidence/lab6-20261119-203024-observability-cost/full-warmup.json b/evidence/lab6-20261119-203024-observability-cost/full-warmup.json new file mode 100644 index 0000000..11c7cb7 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/full-warmup.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.178, + "concurrency": 2, + "successful_requests": 83, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 8.155, + "successful_latency_ms": { + "p50": 210.318, + "p95": 382.103, + "p99": 1033.558 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/off-load.json b/evidence/lab6-20261119-203024-observability-cost/off-load.json new file mode 100644 index 0000000..7ae5225 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/off-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.379, + "concurrency": 4, + "successful_requests": 1840, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 40.547, + "successful_latency_ms": { + "p50": 97.186, + "p95": 198.133, + "p99": 275.316 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/off-resource-samples.json b/evidence/lab6-20261119-203024-observability-cost/off-resource-samples.json new file mode 100644 index 0000000..4784d98 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/off-resource-samples.json @@ -0,0 +1,149 @@ +[ + { + "captured_at": "2026-11-19T17:32:57.3534028+00:00", + "service": "otel-collector", + "cpu_percent": 0.05, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:32:57.3534028+00:00", + "service": "api", + "cpu_percent": 7.72, + "memory_bytes": 304296755, + "memory_display": "290.2MiB" + }, + { + "captured_at": "2026-11-19T17:32:57.3534028+00:00", + "service": "worker", + "cpu_percent": 0.5, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:05.0940138+00:00", + "service": "otel-collector", + "cpu_percent": 0.05, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:05.0940138+00:00", + "service": "api", + "cpu_percent": 75.89, + "memory_bytes": 369203610, + "memory_display": "352.1MiB" + }, + { + "captured_at": "2026-11-19T17:33:05.0940138+00:00", + "service": "worker", + "cpu_percent": 0.27, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:12.7649354+00:00", + "service": "otel-collector", + "cpu_percent": 0.04, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:12.7649354+00:00", + "service": "api", + "cpu_percent": 76.82, + "memory_bytes": 368259891, + "memory_display": "351.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:12.7649354+00:00", + "service": "worker", + "cpu_percent": 0.3, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:20.3812382+00:00", + "service": "otel-collector", + "cpu_percent": 0.04, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:20.3812382+00:00", + "service": "api", + "cpu_percent": 78.72, + "memory_bytes": 374446490, + "memory_display": "357.1MiB" + }, + { + "captured_at": "2026-11-19T17:33:20.3812382+00:00", + "service": "worker", + "cpu_percent": 0.24, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:28.0386363+00:00", + "service": "otel-collector", + "cpu_percent": 0.1, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:28.0386363+00:00", + "service": "api", + "cpu_percent": 77.11, + "memory_bytes": 377067930, + "memory_display": "359.6MiB" + }, + { + "captured_at": "2026-11-19T17:33:28.0386363+00:00", + "service": "worker", + "cpu_percent": 0.42, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:35.6857265+00:00", + "service": "otel-collector", + "cpu_percent": 0.05, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:35.6857265+00:00", + "service": "api", + "cpu_percent": 75.04, + "memory_bytes": 380633088, + "memory_display": "363MiB" + }, + { + "captured_at": "2026-11-19T17:33:35.6857265+00:00", + "service": "worker", + "cpu_percent": 0.24, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:43.3529739+00:00", + "service": "otel-collector", + "cpu_percent": 0.05, + "memory_bytes": 151519232, + "memory_display": "144.5MiB" + }, + { + "captured_at": "2026-11-19T17:33:43.3529739+00:00", + "service": "api", + "cpu_percent": 74.17, + "memory_bytes": 379794227, + "memory_display": "362.2MiB" + }, + { + "captured_at": "2026-11-19T17:33:43.3529739+00:00", + "service": "worker", + "cpu_percent": 0.34, + "memory_bytes": 208876339, + "memory_display": "199.2MiB" + } +] diff --git a/evidence/lab6-20261119-203024-observability-cost/off-result.json b/evidence/lab6-20261119-203024-observability-cost/off-result.json new file mode 100644 index 0000000..5364fe8 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/off-result.json @@ -0,0 +1,103 @@ +{ + "mode": { + "name": "off", + "sdk_disabled": true, + "sampler_arg": 0.0, + "metric_interval_ms": 60000 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 45.0, + "elapsed_seconds": 45.379, + "concurrency": 4, + "successful_requests": 1840, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 40.547, + "successful_latency_ms": { + "p50": 97.186, + "p95": 198.133, + "p99": 275.316 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "ingest_before": { + "captured_at": "2026-11-19T20:32:50.4901468+03:00", + "collector_accepted_spans": 59600.0, + "collector_accepted_logs": 3470.0, + "collector_accepted_metric_points": 94238.0, + "collector_sent_spans": 56236.0, + "collector_sent_logs": 2335.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 21604219.0, + "tempo_ingest_spans": 42462.0, + "loki_line_bytes": 691475.0, + "loki_metadata_bytes": 647495.0, + "loki_lines": 1999.0, + "prometheus_appended_samples": 1084710.0 + }, + "ingest_after": { + "captured_at": "2026-11-19T20:34:00.39937+03:00", + "collector_accepted_spans": 59600.0, + "collector_accepted_logs": 3470.0, + "collector_accepted_metric_points": 94238.0, + "collector_sent_spans": 56236.0, + "collector_sent_logs": 2335.0, + "collector_failed_spans": 3364.0, + "collector_failed_logs": 1135.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 21604219.0, + "tempo_ingest_spans": 42462.0, + "loki_line_bytes": 691475.0, + "loki_metadata_bytes": 647495.0, + "loki_lines": 1999.0, + "prometheus_appended_samples": 1091353.0 + }, + "ingest_delta": { + "collector_accepted_spans": 0.0, + "collector_accepted_logs": 0.0, + "collector_accepted_metric_points": 0.0, + "collector_sent_spans": 0.0, + "collector_sent_logs": 0.0, + "collector_failed_spans": 0.0, + "collector_failed_logs": 0.0, + "collector_enqueue_failed_spans": 0.0, + "collector_refused_spans": 0.0, + "tempo_ingest_bytes": 0.0, + "tempo_ingest_spans": 0.0, + "loki_line_bytes": 0.0, + "loki_metadata_bytes": 0.0, + "loki_lines": 0.0, + "prometheus_appended_samples": 6643.0, + "loki_total_bytes": 0.0, + "backend_total_bytes": 0.0 + }, + "resources": { + "api": { + "sample_count": 7, + "cpu_avg_percent": 66.496, + "cpu_peak_percent": 78.72, + "ram_avg_bytes": 364814570, + "ram_peak_bytes": 380633088 + }, + "worker": { + "sample_count": 7, + "cpu_avg_percent": 0.33, + "cpu_peak_percent": 0.5, + "ram_avg_bytes": 208876339, + "ram_peak_bytes": 208876339 + }, + "collector": { + "sample_count": 7, + "cpu_avg_percent": 0.054, + "cpu_peak_percent": 0.1, + "ram_avg_bytes": 151519232, + "ram_peak_bytes": 151519232 + } + } +} diff --git a/evidence/lab6-20261119-203024-observability-cost/off-warmup.json b/evidence/lab6-20261119-203024-observability-cost/off-warmup.json new file mode 100644 index 0000000..01f98a5 --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/off-warmup.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.225, + "concurrency": 2, + "successful_requests": 126, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 12.323, + "successful_latency_ms": { + "p50": 122.017, + "p95": 277.867, + "p99": 964.446 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-20261119-203024-observability-cost/summary.json b/evidence/lab6-20261119-203024-observability-cost/summary.json new file mode 100644 index 0000000..225a29b --- /dev/null +++ b/evidence/lab6-20261119-203024-observability-cost/summary.json @@ -0,0 +1,150 @@ +{ + "captured_at": "2026-11-19T20:45:43.8506063+03:00", + "methodology": { + "load_seconds": 45, + "concurrency": 4, + "warmup_seconds": 10, + "stats_interval_seconds": 5, + "workload": "closed-loop POST messages; identical profile per mode; API and worker recreated before each mode", + "sampling_loss_definition": "estimated against full-mode accepted spans per successful request; it is SDK non-emission, not Collector transport loss", + "ingest_units": "Collector items; Tempo/Loki uncompressed backend bytes; Prometheus appended samples" + }, + "comparison": [ + { + "mode": "off", + "sdk_disabled": true, + "trace_sample_ratio": 0.0, + "metric_interval_ms": 60000, + "successful_requests": 1840, + "failed_requests": 0, + "successful_rps": 40.547, + "rps_change_vs_off_percent": 0.0, + "latency_p50_ms": 97.186, + "latency_p95_ms": 198.133, + "latency_p99_ms": 275.316, + "p95_change_vs_off_percent": 0.0, + "api_cpu_avg_percent": 66.496, + "api_ram_peak_bytes": 380633088, + "worker_cpu_avg_percent": 0.33, + "worker_ram_peak_bytes": 208876339, + "collector_cpu_avg_percent": 0.054, + "collector_ram_peak_bytes": 151519232, + "accepted_spans": 0.0, + "accepted_logs": 0.0, + "accepted_metric_points": 0.0, + "spans_per_successful_request": 0.0, + "trace_retained_percent_of_full_normalized": 0.0, + "estimated_spans_not_emitted_by_sdk": 22447.0, + "collector_transport_lost_spans": 0.0, + "tempo_ingest_bytes": 0.0, + "loki_ingest_bytes": 0.0, + "backend_trace_and_log_ingest_bytes": 0.0, + "tempo_bytes_per_successful_request": 0.0, + "loki_bytes_per_successful_request": 0.0, + "backend_bytes_per_successful_request": 0.0, + "prometheus_appended_samples": 6643.0 + }, + { + "mode": "full", + "sdk_disabled": false, + "trace_sample_ratio": 1.0, + "metric_interval_ms": 10000, + "successful_requests": 1012, + "failed_requests": 0, + "successful_rps": 22.419, + "rps_change_vs_off_percent": -44.71, + "latency_p50_ms": 185.241, + "latency_p95_ms": 315.289, + "latency_p99_ms": 405.961, + "p95_change_vs_off_percent": 59.13, + "api_cpu_avg_percent": 69.176, + "api_ram_peak_bytes": 425407283, + "worker_cpu_avg_percent": 0.544, + "worker_ram_peak_bytes": 323171123, + "collector_cpu_avg_percent": 2.169, + "collector_ram_peak_bytes": 157496115, + "accepted_spans": 12346.0, + "accepted_logs": 1019.0, + "accepted_metric_points": 711.0, + "spans_per_successful_request": 12.1996, + "trace_retained_percent_of_full_normalized": 100.0, + "estimated_spans_not_emitted_by_sdk": 0.0, + "collector_transport_lost_spans": 0.0, + "tempo_ingest_bytes": 6286284.0, + "loki_ingest_bytes": 715458.0, + "backend_trace_and_log_ingest_bytes": 7001742.0, + "tempo_bytes_per_successful_request": 6211.74, + "loki_bytes_per_successful_request": 706.97, + "backend_bytes_per_successful_request": 6918.72, + "prometheus_appended_samples": 7479.0 + }, + { + "mode": "economy", + "sdk_disabled": false, + "trace_sample_ratio": 0.05, + "metric_interval_ms": 60000, + "successful_requests": 664, + "failed_requests": 0, + "successful_rps": 14.622, + "rps_change_vs_off_percent": -63.94, + "latency_p50_ms": 287.141, + "latency_p95_ms": 405.639, + "latency_p99_ms": 502.152, + "p95_change_vs_off_percent": 104.73, + "api_cpu_avg_percent": 68.564, + "api_ram_peak_bytes": 431803597, + "worker_cpu_avg_percent": 0.517, + "worker_ram_peak_bytes": 293601280, + "collector_cpu_avg_percent": 0.877, + "collector_ram_peak_bytes": 159593267, + "accepted_spans": 447.0, + "accepted_logs": 671.0, + "accepted_metric_points": 115.0, + "spans_per_successful_request": 0.6732, + "trace_retained_percent_of_full_normalized": 5.52, + "estimated_spans_not_emitted_by_sdk": 7654.0, + "collector_transport_lost_spans": 0.0, + "tempo_ingest_bytes": 241260.0, + "loki_ingest_bytes": 494616.0, + "backend_trace_and_log_ingest_bytes": 735876.0, + "tempo_bytes_per_successful_request": 363.34, + "loki_bytes_per_successful_request": 744.9, + "backend_bytes_per_successful_request": 1108.25, + "prometheus_appended_samples": 6008.0 + }, + { + "mode": "balanced", + "sdk_disabled": false, + "trace_sample_ratio": 0.25, + "metric_interval_ms": 30000, + "successful_requests": 768, + "failed_requests": 0, + "successful_rps": 16.934, + "rps_change_vs_off_percent": -58.24, + "latency_p50_ms": 209.98, + "latency_p95_ms": 391.49, + "latency_p99_ms": 409.089, + "p95_change_vs_off_percent": 97.59, + "api_cpu_avg_percent": 67.623, + "api_ram_peak_bytes": 426351002, + "worker_cpu_avg_percent": 0.781, + "worker_ram_peak_bytes": 313524224, + "collector_cpu_avg_percent": 0.543, + "collector_ram_peak_bytes": 155713536, + "accepted_spans": 2200.0, + "accepted_logs": 775.0, + "accepted_metric_points": 238.0, + "spans_per_successful_request": 2.8646, + "trace_retained_percent_of_full_normalized": 23.48, + "estimated_spans_not_emitted_by_sdk": 7169.0, + "collector_transport_lost_spans": 0.0, + "tempo_ingest_bytes": 1129862.0, + "loki_ingest_bytes": 560136.0, + "backend_trace_and_log_ingest_bytes": 1689998.0, + "tempo_bytes_per_successful_request": 1471.17, + "loki_bytes_per_successful_request": 729.34, + "backend_bytes_per_successful_request": 2200.52, + "prometheus_appended_samples": 6318.0 + } + ] +} diff --git a/evidence/rolling-update-2026-11-19.json b/evidence/rolling-update-2026-11-19.json new file mode 100644 index 0000000..6362bc4 --- /dev/null +++ b/evidence/rolling-update-2026-11-19.json @@ -0,0 +1,23 @@ +{ + "scenario": "sequential API replacement under closed-loop writes", + "requested_duration_seconds": 180.0, + "elapsed_seconds": 182.022, + "rolling_action_seconds": 125.275, + "concurrency": 4, + "observed_instance_ids": [ + "431acab3a50c", + "8e081627dfc3", + "d67820ebd09d", + "fdff9a67741c" + ], + "successful_requests": 335, + "failed_requests": 3, + "error_ratio": 0.008876, + "successful_rps": 1.84, + "successful_latency_ms": { + "p50": 2092.164, + "p95": 2252.301, + "p99": 3064.667 + }, + "note": "Closed-loop result on one host; success latency excludes failures." +} diff --git a/evidence/upload-memory-2026-11-11.json b/evidence/upload-memory-2026-11-11.json new file mode 100644 index 0000000..6ee7149 --- /dev/null +++ b/evidence/upload-memory-2026-11-11.json @@ -0,0 +1,14 @@ +{ + "concurrency": 12, + "successful_uploads": 12, + "payload_bytes_each": 6835833, + "duration_seconds": 10.156, + "api_replicas": 2, + "memory_samples": 5, + "baseline_api_memory_bytes": 668467200, + "peak_api_memory_bytes": 839909375, + "peak_increase_bytes": 171442175, + "configured_max_concurrent_uploads_per_replica": 3, + "configured_max_upload_bytes": 10485760, + "configured_max_image_pixels": 40000000 +} diff --git a/evidence/upload-memory-2026-11-19.json b/evidence/upload-memory-2026-11-19.json new file mode 100644 index 0000000..252469d --- /dev/null +++ b/evidence/upload-memory-2026-11-19.json @@ -0,0 +1,14 @@ +{ + "concurrency": 12, + "successful_uploads": 12, + "payload_bytes_each": 6835833, + "duration_seconds": 31.241, + "api_replicas": 2, + "memory_samples": 10, + "baseline_api_memory_bytes": 665321471, + "peak_api_memory_bytes": 833198489, + "peak_increase_bytes": 167877018, + "configured_max_concurrent_uploads_per_replica": 3, + "configured_max_upload_bytes": 10485760, + "configured_max_image_pixels": 40000000 +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..eddabd2 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..ad7845b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip +networkTimeout=10000 +retries=0 +retryBackOffMs=500 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..249efbb --- /dev/null +++ b/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# gradlew start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh gradlew +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..a51ec4f --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,82 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem gradlew startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:execute +@rem Setup the command line + + + +@rem Execute gradlew +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel + +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 0000000..0ee222d --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,27 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::__S3_BUCKET__" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::__S3_BUCKET__/attachments/*", + "arn:aws:s3:::__S3_BUCKET__/staging/*" + ] + } + ] +} diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..7aabc52 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,103 @@ +worker_processes auto; +worker_shutdown_timeout 10s; + +events { + worker_connections 1024; +} + +http { + # Let the application enforce the exact 10 MiB contract and return JSON. + client_max_body_size 11m; + client_body_timeout 10s; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log warn; + + # Docker embedded DNS returns current Compose replicas. Nginx OSS 1.27.3+ + # refreshes these addresses without a configuration reload. + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 1s; + + upstream api_backend { + zone api_backend 64k; + least_conn; + server api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 32; + } + + server { + listen 80; + server_name _; + proxy_intercept_errors on; + error_page 502 503 504 = @no_ready_api; + + location = /nginx-health { + access_log off; + default_type application/json; + return 200 '{"status":"ok"}'; + } + + # After both peers have refused traffic or become unreachable, Nginx + # would normally emit its own HTML 502. Keep the public API contract: + # no ready backend is a JSON 503 with the required tracing headers. + location @no_ready_api { + internal; + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id nginx always; + add_header Cache-Control "no-store" always; + return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}'; + } + + # Message POST is the only non-idempotent HTTP method that is safe to + # replay here: the API requires Idempotency-Key and commits the message + # together with its shared PostgreSQL idempotency record. + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 2; + } + + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + + proxy_connect_timeout 500ms; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + # Upload is not replayed: it has no idempotency key. The client may retry explicitly. + proxy_next_upstream off; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + + # No sticky sessions. Failed/unready peers are retried elsewhere. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..13f1f84 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,30 @@ -# Конфигурация наблюдаемости — реализовать +# Стек наблюдаемости -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +Поток данных: -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +`messenger-api / messenger-worker -> OTLP/HTTP -> otel-collector -> Prometheus / Loki / Tempo -> Grafana`. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Версии закреплены в `compose.yaml`: Java agent 2.33.1 (SDK 1.67.0), Collector Contrib 0.165.0, Prometheus 3.14.0, Loki 3.7.8, Tempo 3.1.1 и Grafana 13.3.0. + +## Файлы + +- `collector.yaml` — OTLP receivers, memory limiter, redaction, batch и bounded retry queues; +- `prometheus.yaml` и `prometheus-alerts.yaml` — scrape, SLO/queue/worker/export alerts; +- `loki.yaml` — локальное хранилище структурных OTLP logs; +- `tempo.yaml` — локальное хранилище traces; +- `grafana/provisioning` — три datasources и dashboard provisioning; +- `grafana/dashboards/messenger-overview.json` — импортируемый dashboard; +- `runbooks.md` — локальные действия для каждого alert. + +Grafana доступна только на `127.0.0.1:3000`, Prometheus — на `127.0.0.1:9090`. Для автоматизированных incident snapshots Loki и Tempo доступны только на loopback-портах 3100 и 3200. OTLP Collector не публикуется на интерфейсы хоста. + +## Проверка корреляции + +1. Выполнить `python tests/smoke.py --base-url http://localhost:8080`. +2. В Tempo найти server span upload и перейти к RabbitMQ producer/consumer span. +3. Скопировать `trace_id` и выполнить в Loki `{service_name=~"messenger-(api|worker)"} | json | trace_id="..."`. +4. Убедиться, что у API и worker разные `service.name`, но общий trace context. + +Для durable retry `traceparent/tracestate` сохраняются рядом с job в PostgreSQL и восстанавливаются перед outbox republish. Проверяемый сценарий: `scripts/incident-retry-trace.ps1`; он требует минимум два producer/consumer span в одной trace и сохраняет их parent IDs. + +Collector удаляет потенциально чувствительные HTTP/message/body/DB parameter attributes. Java agent не настроен на захват request/response headers или bodies. Метрики используют только route template, status/method, job type/outcome и queue name. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..2824aae --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,87 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 384 + spike_limit_mib: 96 + attributes/redact: + actions: + - key: http.request.header.authorization + action: delete + - key: http.request.body + action: delete + - key: http.response.body + action: delete + - key: messaging.message.body + action: delete + - key: db.statement.parameters + action: delete + - key: url.query + action: delete + batch: + send_batch_size: 512 + timeout: 5s + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + send_timestamps: true + resource_constant_labels: + included: ["service.*", "deployment.environment.name"] + otlp_http/tempo: + endpoint: http://tempo:4318 + sending_queue: + enabled: true + queue_size: 2048 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + queue_size: 2048 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: false + without_units: false + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlp_http/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlp_http/loki] diff --git a/infra/otel/docker-entrypoint.sh b/infra/otel/docker-entrypoint.sh new file mode 100644 index 0000000..9f13a7c --- /dev/null +++ b/infra/otel/docker-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +if [ "${OTEL_SDK_DISABLED:-false}" = "true" ]; then + exec java -jar /app/app.jar "$@" +fi + +exec java -javaagent:/opt/opentelemetry-javaagent.jar -jar /app/app.jar "$@" diff --git a/infra/otel/grafana/dashboards/messenger-overview.json b/infra/otel/grafana/dashboards/messenger-overview.json new file mode 100644 index 0000000..efdf757 --- /dev/null +++ b/infra/otel/grafana/dashboards/messenger-overview.json @@ -0,0 +1,141 @@ +{ + "annotations": {"list": []}, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "title": "API requests / second", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "sum by (http_request_method) (rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_request_method}}"}], + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "API 5xx ratio", + "description": "5xx business responses divided by all business responses over 5 minutes; health routes are excluded. Displayed as percent (0..100%).", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_response_status_code=~\"5..\",http_route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])), 0.001)", "legendFormat": "5xx / all"}], + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 0} + }, + { + "id": 3, + "title": "API duration p50 / p95 / p99", + "description": "Seconds. histogram_quantile is calculated from the summed Prometheus histogram buckets of all API replicas; replica percentiles are never averaged.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99"} + ], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 0} + }, + { + "id": 4, + "title": "Job outcomes / second", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[5m]))", "legendFormat": "{{outcome}}"}], + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8} + }, + { + "id": 5, + "title": "Job duration p95", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[5m])))", "legendFormat": "{{outcome}}"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8} + }, + { + "id": 6, + "title": "Time to ready p95", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[10m])))", "legendFormat": "p95"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8} + }, + { + "id": 7, + "title": "Queue depth and oldest age", + "description": "Left axis: durable queued jobs (count). Right axis: age of the oldest unfinished job (seconds). Backlog age is not processing latency.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"refId": "A", "expr": "max(messenger_queue_depth{service_name=\"messenger-api\"})", "legendFormat": "depth"}, + {"refId": "B", "expr": "max(messenger_queue_oldest_job_age_seconds{service_name=\"messenger-api\"})", "legendFormat": "oldest age (s)"} + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + {"matcher": {"id": "byName", "options": "depth"}, "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "left"}]}, + {"matcher": {"id": "byName", "options": "oldest age (s)"}, "properties": [{"id": "unit", "value": "s"}, {"id": "custom.axisPlacement", "value": "right"}]} + ] + }, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 16} + }, + { + "id": 8, + "title": "Worker saturation", + "description": "busy workers / configured worker capacity. Dimensionless ratio displayed as percent.", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{"refId": "A", "expr": "messenger_worker_busy{service_name=\"messenger-worker\"} / clamp_min(messenger_worker_capacity{service_name=\"messenger-worker\"}, 1)", "legendFormat": "{{service_instance_id}}"}], + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 16} + }, + { + "id": 9, + "title": "DB pool used / waiting", + "description": "Number of HikariCP connections by state plus pending acquisition requests. Unit: connections/requests (count).", + "type": "timeseries", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + {"refId": "A", "expr": "sum by (service_name, state) (db_client_connections_usage{service_name=~\"messenger-(api|worker)\"})", "legendFormat": "{{service_name}} {{state}}"}, + {"refId": "B", "expr": "sum by (service_name) (db_client_connections_pending_requests{service_name=~\"messenger-(api|worker)\"})", "legendFormat": "{{service_name}} waiting"} + ], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 16} + }, + { + "id": 10, + "title": "Correlated application logs", + "type": "logs", + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [{"refId": "A", "expr": "{service_name=~\"messenger-(api|worker)\"} | json", "queryType": "range"}], + "options": {"showTime": true, "showLabels": false, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "sortOrder": "Descending", "dedupStrategy": "none"}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 24} + }, + { + "id": 11, + "title": "Recent traces", + "type": "traces", + "datasource": {"type": "tempo", "uid": "tempo"}, + "targets": [{"refId": "A", "queryType": "search", "serviceName": "messenger-api", "limit": 20}], + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 24} + } + ], + "refresh": "10s", + "schemaVersion": 41, + "tags": ["hla", "lab6", "opentelemetry"], + "templating": {"list": []}, + "time": {"from": "now-30m", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Messenger observability", + "uid": "messenger-observability", + "version": 3, + "weekStart": "" +} diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yaml b/infra/otel/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..ce50d27 --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: Messenger Lab 6 + orgId: 1 + folder: HLA Lab 6 + type: file + disableDeletion: true + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yaml b/infra/otel/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..492d3a7 --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,59 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 10s + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([a-f0-9]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + httpMethod: GET + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -1m + spanEndTimeShift: 1m + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + search: + hide: false diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..54a82ce --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,37 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 72h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/prometheus-alerts.yaml b/infra/otel/prometheus-alerts.yaml new file mode 100644 index 0000000..47c23ac --- /dev/null +++ b/infra/otel/prometheus-alerts.yaml @@ -0,0 +1,51 @@ +groups: + - name: messenger-slo + rules: + - alert: MessengerApiHighErrorRate + expr: | + ( + (sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5..",http_route!~"/health/.*"}[5m])) or vector(0)) + / + clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m])), 0.001) + ) > 0.05 + for: 2m + labels: + severity: warning + slo: api-availability + annotations: + summary: Messenger API 5xx ratio exceeds 5% + description: Five-minute 5xx ratio is above 5% for two minutes. + runbook: infra/otel/runbooks.md#api-high-error-rate + + - alert: AttachmentQueueLagHigh + expr: messenger_queue_oldest_job_age_seconds{service_name="messenger-api"} > 60 + for: 1m + labels: + severity: warning + slo: attachment-time-to-ready + annotations: + summary: Oldest attachment job is over 60 seconds old + description: The durable worker queue has not drained within the SLO threshold. + runbook: infra/otel/runbooks.md#queue-lag + + - alert: AttachmentWorkerSaturated + expr: | + messenger_worker_busy{service_name="messenger-worker"} + / clamp_min(messenger_worker_capacity{service_name="messenger-worker"}, 1) >= 1 + for: 2m + labels: + severity: warning + annotations: + summary: All attachment workers are busy + description: Worker busy/capacity has remained at 100% for two minutes. + runbook: infra/otel/runbooks.md#worker-saturation + + - alert: TelemetryExportFailures + expr: sum(rate(otelcol_exporter_send_failed_spans_total[5m])) + sum(rate(otelcol_exporter_send_failed_log_records_total[5m])) > 0 + for: 1m + labels: + severity: info + annotations: + summary: Collector is dropping traces or logs + description: A telemetry backend is unavailable or rejecting data. + runbook: infra/otel/runbooks.md#telemetry-backend diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..9098d10 --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,19 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/prometheus-alerts.yaml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["prometheus:9090"] + + - job_name: otel-collector-exported-metrics + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: otel-collector-internal + static_configs: + - targets: ["otel-collector:8888"] diff --git a/infra/otel/runbooks.md b/infra/otel/runbooks.md new file mode 100644 index 0000000..079db87 --- /dev/null +++ b/infra/otel/runbooks.md @@ -0,0 +1,27 @@ +# Runbooks + +## API high error rate + +1. In Grafana, compare the 5xx ratio with `db_client_*` pool metrics. +2. Open a failed server span in Tempo and inspect child JDBC/Redis/S3 spans. +3. Search Loki by the trace ID. Do not search by a user, chat or message ID. +4. Restore the failing dependency and confirm the alert resolves after the five-minute window. + +## Queue lag + +1. Check `messenger_queue_depth` and `messenger_queue_oldest_job_age_seconds`. +2. Check worker busy/capacity and the rate of `messenger_jobs_processed_total` by outcome. +3. Search worker errors in Loki, then open the linked consumer trace in Tempo. +4. Restore or scale the worker and confirm depth and age return to zero. + +## Worker saturation + +1. Compare busy/capacity with job duration p95 and container CPU/RAM. +2. Inspect slow consumer spans and their S3/JDBC children. +3. Remove the bottleneck before increasing concurrency; memory is explicitly bounded. + +## Telemetry backend + +1. Inspect Collector exporter failure and queue metrics on port 8888 through Prometheus. +2. Restore Loki or Tempo. Collector retries for at most 60 seconds with a 2048-item queue. +3. Confirm the application remained available; telemetry loss is allowed and bounded. diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..cc753b0 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,31 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +live_store: + max_trace_idle: 10s + max_block_duration: 5m + wal: + path: /var/tempo/live-store/traces + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + +overrides: + defaults: + compaction: + block_retention: 72h + +usage_report: + reporting_enabled: false diff --git a/scripts/check-private-bucket.cmd b/scripts/check-private-bucket.cmd new file mode 100644 index 0000000..1065a92 --- /dev/null +++ b/scripts/check-private-bucket.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." +if not defined ENV_FILE set "ENV_FILE=.env" + +docker compose --env-file "%ENV_FILE%" --profile verification run --rm private-bucket-check +exit /b %ERRORLEVEL% diff --git a/scripts/check-private-bucket.sh b/scripts/check-private-bucket.sh new file mode 100644 index 0000000..eab9468 --- /dev/null +++ b/scripts/check-private-bucket.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" +: "${ENV_FILE:=.env}" + +docker compose --env-file "$ENV_FILE" --profile verification run --rm private-bucket-check diff --git a/scripts/experiment-observability-cost.ps1 b/scripts/experiment-observability-cost.ps1 new file mode 100644 index 0000000..0d6c82f --- /dev/null +++ b/scripts/experiment-observability-cost.ps1 @@ -0,0 +1,278 @@ +param( + [string]$BaseUrl = "http://127.0.0.1:18080", + [string]$EnvFile = ".env", + [int]$LoadSeconds = 45, + [int]$Concurrency = 4, + [int]$WarmupSeconds = 10, + [int]$StatsIntervalSeconds = 5 +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidenceDir = Join-Path $project "evidence\lab6-$stamp-observability-cost" +New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null + +$modes = @( + [ordered]@{ name = "off"; sdk_disabled = $true; sampler_arg = 0.0; metric_interval_ms = 60000 }, + [ordered]@{ name = "full"; sdk_disabled = $false; sampler_arg = 1.0; metric_interval_ms = 10000 }, + [ordered]@{ name = "economy"; sdk_disabled = $false; sampler_arg = 0.05; metric_interval_ms = 60000 }, + [ordered]@{ name = "balanced"; sdk_disabled = $false; sampler_arg = 0.25; metric_interval_ms = 30000 } +) + +function Query-PrometheusScalar([string]$expression) { + $encoded = [uri]::EscapeDataString($expression) + $response = Invoke-RestMethod "http://127.0.0.1:9090/api/v1/query?query=$encoded" + if (@($response.data.result).Count -eq 0) { return 0.0 } + return [double]$response.data.result[0].value[1] +} + +function Query-TextMetric([string]$url, [string]$metricName) { + $content = (Invoke-WebRequest -UseBasicParsing $url).Content + $sum = 0.0 + foreach ($line in ($content -split "`n")) { + if ($line -match ('^' + [regex]::Escape($metricName) + '(?:\{[^}]*\})?\s+([0-9.eE+\-]+)\s*$')) { + $sum += [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) + } + } + return $sum +} + +function Capture-IngestCounters { + return [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + collector_accepted_spans = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_spans_total) or vector(0)' + collector_accepted_logs = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_log_records_total) or vector(0)' + collector_accepted_metric_points = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_metric_points_total) or vector(0)' + collector_sent_spans = Query-PrometheusScalar 'sum(otelcol_exporter_sent_spans_total) or vector(0)' + collector_sent_logs = Query-PrometheusScalar 'sum(otelcol_exporter_sent_log_records_total) or vector(0)' + collector_failed_spans = Query-PrometheusScalar 'sum(otelcol_exporter_send_failed_spans_total) or vector(0)' + collector_failed_logs = Query-PrometheusScalar 'sum(otelcol_exporter_send_failed_log_records_total) or vector(0)' + collector_enqueue_failed_spans = Query-PrometheusScalar 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)' + collector_refused_spans = Query-PrometheusScalar 'sum(otelcol_receiver_refused_spans_total) or vector(0)' + tempo_ingest_bytes = Query-TextMetric 'http://127.0.0.1:3200/metrics' 'tempo_distributor_bytes_received_total' + tempo_ingest_spans = Query-TextMetric 'http://127.0.0.1:3200/metrics' 'tempo_distributor_spans_received_total' + loki_line_bytes = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_bytes_received_total' + loki_metadata_bytes = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_structured_metadata_bytes_received_total' + loki_lines = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_lines_received_total' + prometheus_appended_samples = Query-PrometheusScalar 'sum(prometheus_tsdb_head_samples_appended_total) or vector(0)' + } +} + +function Convert-MemoryToBytes([string]$value) { + if ($value -notmatch '^([0-9.]+)\s*([KMG]iB)$') { return 0L } + $number = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) + $factor = switch ($Matches[2]) { + 'KiB' { 1KB } + 'MiB' { 1MB } + 'GiB' { 1GB } + } + return [long]($number * $factor) +} + +function Wait-ApiReady { + $deadline = (Get-Date).AddMinutes(4) + do { + try { + $response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 3 "$BaseUrl/health/ready" + if ($response.StatusCode -eq 200) { return } + } + catch { Start-Sleep -Seconds 2 } + } while ((Get-Date) -lt $deadline) + throw "API did not become ready at $BaseUrl" +} + +function Capture-ContainerStats([hashtable]$containerNames) { + $names = @($containerNames.Values) + $rows = @(docker stats --no-stream --format '{{json .}}' $names | ForEach-Object { $_ | ConvertFrom-Json }) + $captured = [DateTimeOffset]::UtcNow.ToString("o") + $result = @() + foreach ($service in $containerNames.Keys) { + $name = $containerNames[$service] + $row = @($rows | Where-Object { $_.Name -eq $name })[0] + if ($null -eq $row) { continue } + $used = ($row.MemUsage -split '/')[0].Trim() + $cpu = [double]::Parse(($row.CPUPerc -replace '%','').Trim(), [Globalization.CultureInfo]::InvariantCulture) + $result += [pscustomobject][ordered]@{ + captured_at = $captured + service = $service + cpu_percent = $cpu + memory_bytes = Convert-MemoryToBytes $used + memory_display = $used + } + } + return $result +} + +function Delta([object]$before, [object]$after, [string]$property) { + return [double]$after.$property - [double]$before.$property +} + +function Summarize-Stats([object[]]$samples, [string]$service) { + $selected = @($samples | Where-Object { $_.service -eq $service }) + if ($selected.Count -eq 0) { + return [ordered]@{ sample_count = 0; cpu_avg_percent = 0; cpu_peak_percent = 0; ram_avg_bytes = 0; ram_peak_bytes = 0 } + } + $cpu = $selected | Measure-Object -Property cpu_percent -Average -Maximum + $ram = $selected | Measure-Object -Property memory_bytes -Average -Maximum + return [ordered]@{ + sample_count = $selected.Count + cpu_avg_percent = [Math]::Round($cpu.Average, 3) + cpu_peak_percent = [Math]::Round($cpu.Maximum, 3) + ram_avg_bytes = [long]$ram.Average + ram_peak_bytes = [long]$ram.Maximum + } +} + +$modeResults = @() +$loadJob = $null +Push-Location $project +try { + foreach ($mode in $modes) { + Write-Host "=== mode=$($mode.name) sdk_disabled=$($mode.sdk_disabled) sampler=$($mode.sampler_arg) metrics_ms=$($mode.metric_interval_ms) ===" + $env:OTEL_SDK_DISABLED = $mode.sdk_disabled.ToString().ToLowerInvariant() + $env:OTEL_TRACES_SAMPLER = 'parentbased_traceidratio' + $env:OTEL_TRACES_SAMPLER_ARG = [string]$mode.sampler_arg + $env:OTEL_METRIC_EXPORT_INTERVAL = [string]$mode.metric_interval_ms + + docker compose --env-file $EnvFile up -d --no-deps --force-recreate api worker | Out-Host + if ($LASTEXITCODE -ne 0) { throw "compose recreate failed for $($mode.name)" } + Wait-ApiReady + + $warmupFile = Join-Path $evidenceDir "$($mode.name)-warmup.json" + & python tests/load.py --base-url $BaseUrl --duration $WarmupSeconds --concurrency 2 --output $warmupFile | Out-Host + if ($LASTEXITCODE -ne 0) { throw "warm-up failed for $($mode.name)" } + Start-Sleep -Seconds 12 + + $before = Capture-IngestCounters + $containerNames = @{} + foreach ($service in @('api','worker','otel-collector')) { + $id = docker compose --env-file $EnvFile ps -q $service + if (-not $id) { throw "container is missing for $service" } + $containerNames[$service] = (docker inspect --format '{{.Name}}' $id).TrimStart('/') + } + + $loadFile = Join-Path $evidenceDir "$($mode.name)-load.json" + $loadJob = Start-Job -ArgumentList $project,$BaseUrl,$LoadSeconds,$Concurrency,$loadFile -ScriptBlock { + param($projectPath,$url,$duration,$workers,$outputPath) + Set-Location $projectPath + $env:PYTHONUTF8 = "1" + & python tests/load.py --base-url $url --duration $duration --concurrency $workers --output $outputPath + [pscustomobject]@{ exit_code = $LASTEXITCODE } + } + + $samples = @() + while ($loadJob.State -in @('NotStarted','Running')) { + $samples += Capture-ContainerStats $containerNames + Start-Sleep -Seconds $StatsIntervalSeconds + } + $jobOutput = @(Receive-Job $loadJob) + $exitRecord = @($jobOutput | Where-Object { $_.PSObject.Properties.Name -contains 'exit_code' })[-1] + if ($null -eq $exitRecord -or $exitRecord.exit_code -ne 0) { throw "load failed for $($mode.name)" } + Remove-Job $loadJob + $loadJob = $null + + Start-Sleep -Seconds 12 + $after = Capture-IngestCounters + $load = Get-Content $loadFile -Raw | ConvertFrom-Json + $delta = [ordered]@{} + foreach ($property in @( + 'collector_accepted_spans','collector_accepted_logs','collector_accepted_metric_points', + 'collector_sent_spans','collector_sent_logs','collector_failed_spans','collector_failed_logs', + 'collector_enqueue_failed_spans','collector_refused_spans','tempo_ingest_bytes','tempo_ingest_spans', + 'loki_line_bytes','loki_metadata_bytes','loki_lines','prometheus_appended_samples')) { + $delta[$property] = Delta $before $after $property + } + $delta['loki_total_bytes'] = $delta.loki_line_bytes + $delta.loki_metadata_bytes + $delta['backend_total_bytes'] = $delta.tempo_ingest_bytes + $delta.loki_total_bytes + + $modeResult = [ordered]@{ + mode = $mode + load = $load + ingest_before = $before + ingest_after = $after + ingest_delta = $delta + resources = [ordered]@{ + api = Summarize-Stats $samples 'api' + worker = Summarize-Stats $samples 'worker' + collector = Summarize-Stats $samples 'otel-collector' + } + } + $modeResult | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "$($mode.name)-result.json") + $samples | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "$($mode.name)-resource-samples.json") + $modeResults += $modeResult + } + + $full = @($modeResults | Where-Object { $_.mode.name -eq 'full' })[0] + $off = @($modeResults | Where-Object { $_.mode.name -eq 'off' })[0] + $fullSpansPerRequest = [double]$full.ingest_delta.collector_accepted_spans / [Math]::Max(1, [double]$full.load.successful_requests) + $comparison = @() + foreach ($result in $modeResults) { + $requests = [double]$result.load.successful_requests + $spansPerRequest = [double]$result.ingest_delta.collector_accepted_spans / [Math]::Max(1, $requests) + $expectedAtFull = $fullSpansPerRequest * $requests + $notEmitted = [Math]::Max(0, $expectedAtFull - [double]$result.ingest_delta.collector_accepted_spans) + $transportLost = [double]$result.ingest_delta.collector_failed_spans + [double]$result.ingest_delta.collector_enqueue_failed_spans + [double]$result.ingest_delta.collector_refused_spans + $comparison += [pscustomobject][ordered]@{ + mode = $result.mode.name + sdk_disabled = $result.mode.sdk_disabled + trace_sample_ratio = $result.mode.sampler_arg + metric_interval_ms = $result.mode.metric_interval_ms + successful_requests = $result.load.successful_requests + failed_requests = $result.load.failed_requests + successful_rps = $result.load.successful_rps + rps_change_vs_off_percent = [Math]::Round(100 * ([double]$result.load.successful_rps / [double]$off.load.successful_rps - 1), 2) + latency_p50_ms = $result.load.successful_latency_ms.p50 + latency_p95_ms = $result.load.successful_latency_ms.p95 + latency_p99_ms = $result.load.successful_latency_ms.p99 + p95_change_vs_off_percent = [Math]::Round(100 * ([double]$result.load.successful_latency_ms.p95 / [double]$off.load.successful_latency_ms.p95 - 1), 2) + api_cpu_avg_percent = $result.resources.api.cpu_avg_percent + api_ram_peak_bytes = $result.resources.api.ram_peak_bytes + worker_cpu_avg_percent = $result.resources.worker.cpu_avg_percent + worker_ram_peak_bytes = $result.resources.worker.ram_peak_bytes + collector_cpu_avg_percent = $result.resources.collector.cpu_avg_percent + collector_ram_peak_bytes = $result.resources.collector.ram_peak_bytes + accepted_spans = $result.ingest_delta.collector_accepted_spans + accepted_logs = $result.ingest_delta.collector_accepted_logs + accepted_metric_points = $result.ingest_delta.collector_accepted_metric_points + spans_per_successful_request = [Math]::Round($spansPerRequest, 4) + trace_retained_percent_of_full_normalized = if ($fullSpansPerRequest -gt 0) { [Math]::Round(100 * $spansPerRequest / $fullSpansPerRequest, 2) } else { 0 } + estimated_spans_not_emitted_by_sdk = [Math]::Round($notEmitted, 0) + collector_transport_lost_spans = $transportLost + tempo_ingest_bytes = $result.ingest_delta.tempo_ingest_bytes + loki_ingest_bytes = $result.ingest_delta.loki_total_bytes + backend_trace_and_log_ingest_bytes = $result.ingest_delta.backend_total_bytes + tempo_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.tempo_ingest_bytes / [Math]::Max(1, $requests), 2) + loki_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.loki_total_bytes / [Math]::Max(1, $requests), 2) + backend_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.backend_total_bytes / [Math]::Max(1, $requests), 2) + prometheus_appended_samples = $result.ingest_delta.prometheus_appended_samples + } + } + + $comparison | Export-Csv -NoTypeInformation -Encoding utf8 (Join-Path $evidenceDir 'comparison.csv') + [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + methodology = [ordered]@{ + load_seconds = $LoadSeconds + concurrency = $Concurrency + warmup_seconds = $WarmupSeconds + stats_interval_seconds = $StatsIntervalSeconds + workload = 'closed-loop POST messages; identical profile per mode; API and worker recreated before each mode' + sampling_loss_definition = 'estimated against full-mode accepted spans per successful request; it is SDK non-emission, not Collector transport loss' + ingest_units = 'Collector items; Tempo/Loki uncompressed backend bytes; Prometheus appended samples' + } + comparison = $comparison + } | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 (Join-Path $evidenceDir 'summary.json') + + & make test BASE_URL=$BaseUrl PYTHON=python 2>&1 | Tee-Object -FilePath (Join-Path $evidenceDir 'balanced-make-test.txt') | Out-Host + if ($LASTEXITCODE -ne 0) { throw 'make test failed with balanced telemetry' } + Write-Host "Evidence: $evidenceDir" +} +finally { + if ($null -ne $loadJob) { Stop-Job $loadJob -ErrorAction SilentlyContinue; Remove-Job $loadJob -Force -ErrorAction SilentlyContinue } + Remove-Item Env:OTEL_SDK_DISABLED -ErrorAction SilentlyContinue + Remove-Item Env:OTEL_TRACES_SAMPLER -ErrorAction SilentlyContinue + Remove-Item Env:OTEL_TRACES_SAMPLER_ARG -ErrorAction SilentlyContinue + Remove-Item Env:OTEL_METRIC_EXPORT_INTERVAL -ErrorAction SilentlyContinue + Pop-Location +} diff --git a/scripts/incident-database-stop.ps1 b/scripts/incident-database-stop.ps1 new file mode 100644 index 0000000..a82fd73 --- /dev/null +++ b/scripts/incident-database-stop.ps1 @@ -0,0 +1,120 @@ +param( + [int]$DurationSeconds = 165, + [string]$BaseUrl = "http://localhost:8080", + [string]$EnvFile = ".env" +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidenceDir = Join-Path $project "evidence\lab6-$stamp-database-stop" +New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null + +Push-Location $project +try { + docker compose --env-file $EnvFile stop postgres + $samples = @() + $deadline = (Get-Date).AddSeconds($DurationSeconds) + while ((Get-Date) -lt $deadline) { + $started = Get-Date + try { + $response = Invoke-WebRequest -Uri "$BaseUrl/health/ready" ` + -SkipHttpErrorCheck -TimeoutSec 5 + $readyStatus = [int]$response.StatusCode + } + catch { + $readyStatus = 0 + } + try { + $response = Invoke-WebRequest -Uri "$BaseUrl/api/v1/users/me" ` + -Headers @{ Authorization = "Bearer incident-synthetic-token" } ` + -SkipHttpErrorCheck -TimeoutSec 5 + $businessStatus = [int]$response.StatusCode + } + catch { + $businessStatus = 0 + } + $samples += [ordered]@{ + at = $started.ToUniversalTime().ToString("o") + ready_status = $readyStatus + business_status = $businessStatus + duration_ms = [math]::Round(((Get-Date) - $started).TotalMilliseconds, 1) + } + Start-Sleep -Seconds 1 + } + $samples | ConvertTo-Json -Depth 5 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "http-samples.json") + + $alertsSnapshot = Invoke-RestMethod "http://localhost:9090/api/v1/alerts" + $highErrorAlert = @($alertsSnapshot.data.alerts | Where-Object { + $_.labels.alertname -eq "MessengerApiHighErrorRate" -and $_.state -eq "firing" + }) + $observations = [ordered]@{ + captured_at = (Get-Date).ToUniversalTime().ToString("o") + api_5xx = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=sum(rate(http_server_request_duration_seconds_count%7Bhttp_response_status_code%3D~%225..%22%7D%5B2m%5D))" + alerts = $alertsSnapshot + alert_firing = ($highErrorAlert.Count -gt 0) + jdbc_spans_hint = "Grafana Explore -> Tempo: { resource.service.name = `"messenger-api`" && span.db.system.name = `"postgresql`" }" + correlated_logs_hint = "Grafana Explore -> Loki: {service_name=`"messenger-api`"} | json | level=`"ERROR`"" + } + $observations | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "failure-observation.json") + if ($highErrorAlert.Count -eq 0) { + throw "MessengerApiHighErrorRate did not reach firing state" + } + + $end = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() * 1000000 + $start = [DateTimeOffset]::UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds() * 1000000 + $logQuery = [uri]::EscapeDataString('{service_name="messenger-api"}') + Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$logQuery&start=$start&end=$end&limit=100&direction=backward" | + ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "loki-api-logs.json") + $dbTraceQuery = [uri]::EscapeDataString('{ span.db.system = "postgresql" && status = error }') + Invoke-RestMethod "http://localhost:3200/api/search?q=$dbTraceQuery&limit=100" | + ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-postgres-errors.json") + $httpTraceQuery = [uri]::EscapeDataString('{ name = "GET /health/ready" && status = error }') + Invoke-RestMethod "http://localhost:3200/api/search?q=$httpTraceQuery&limit=100" | + ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-http-errors.json") +} +finally { + docker compose --env-file $EnvFile start postgres + try { + $readyDeadline = (Get-Date).AddMinutes(2) + do { + Start-Sleep -Seconds 5 + try { + $readyResponse = Invoke-WebRequest -Uri "$BaseUrl/health/ready" ` + -SkipHttpErrorCheck -TimeoutSec 10 + $ready = ([int]$readyResponse.StatusCode -eq 200) + } + catch { + $ready = $false + } + } while (-not $ready -and (Get-Date) -lt $readyDeadline) + + $alertDeadline = (Get-Date).AddMinutes(6) + do { + Start-Sleep -Seconds 10 + $recoveryAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts" + $activeHighError = @($recoveryAlerts.data.alerts | Where-Object { + $_.labels.alertname -eq "MessengerApiHighErrorRate" + }) + } while ($activeHighError.Count -gt 0 -and (Get-Date) -lt $alertDeadline) + + [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + readiness_recovered = $ready + alerts = $recoveryAlerts + alert_resolved = ($activeHighError.Count -eq 0) + api_5xx_ratio = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_response_status_code%3D~%225..%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%20or%20vector%280%29%29%20%2F%20clamp_min%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%2C%200.001%29" + } | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "recovery-observation.json") + if (-not $ready -or $activeHighError.Count -gt 0) { + throw "database or MessengerApiHighErrorRate did not recover before the deadline" + } + } + finally { + Pop-Location + } +} + +Write-Host "Evidence saved to $evidenceDir" diff --git a/scripts/incident-retry-trace.ps1 b/scripts/incident-retry-trace.ps1 new file mode 100644 index 0000000..5d923f7 --- /dev/null +++ b/scripts/incident-retry-trace.ps1 @@ -0,0 +1,117 @@ +param( + [string]$BaseUrl = "http://localhost:8080", + [string]$EnvFile = ".env", + [int]$RecoveryWaitSeconds = 60 +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidenceDir = Join-Path $project "evidence\lab6-$stamp-retry-trace" +New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null + +function Ensure-Running([string]$container) { + $state = docker inspect --format '{{.State.Status}}' $container + if ($state -ne 'running') { + docker start $container | Out-Null + } +} + +Push-Location $project +try { + $worker = docker compose --env-file $EnvFile ps -q worker + $minio = docker compose --env-file $EnvFile ps -q minio + if (-not $worker -or -not $minio) { + throw "worker/minio containers do not exist; start the stack first" + } + + docker stop $worker | Out-Null + $env:PYTHONUTF8 = "1" + python tests/attachment_load.py --base-url $BaseUrl --count 1 --interval 0 ` + --output (Join-Path $evidenceDir "load.json") + if ($LASTEXITCODE -ne 0) { + throw "synthetic upload failed" + } + + docker stop $minio | Out-Null + $startedAt = [DateTimeOffset]::UtcNow + $dockerSince = $startedAt.ToString("yyyy-MM-ddTHH:mm:ssZ") + docker start $worker | Out-Null + + $retrySeen = $false + $deadline = (Get-Date).AddMinutes(4) + do { + Start-Sleep -Seconds 1 + $workerLogs = docker logs --since $dockerSince $worker 2>&1 | Out-String + $retrySeen = $workerLogs.Contains("attachment_job_retry_scheduled") + } while (-not $retrySeen -and (Get-Date) -lt $deadline) + + if (-not $retrySeen) { + throw "worker did not schedule a retry before the deadline" + } + Ensure-Running $minio + Start-Sleep -Seconds $RecoveryWaitSeconds + + $startNanos = $startedAt.ToUnixTimeMilliseconds() * 1000000 + $retryQuery = [uri]::EscapeDataString('{service_name="messenger-worker"} | event="attachment_job_retry_scheduled"') + $retryResult = Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$retryQuery&start=$startNanos&limit=20&direction=forward" + $retryStream = $retryResult.data.result | Where-Object { $_.stream.trace_id } | Select-Object -First 1 + if (-not $retryStream) { + throw "retry event was not found in Loki" + } + $traceId = $retryStream.stream.trace_id + + $successQueryText = '{service_name="messenger-worker"} | trace_id="' + $traceId + '" | event="attachment_job_completed"' + $successQuery = [uri]::EscapeDataString($successQueryText) + $successResult = Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$successQuery&start=$startNanos&limit=20&direction=forward" + if (-not $successResult.data.result) { + throw "successful retry completion was not found in Loki" + } + + $trace = Invoke-RestMethod "http://localhost:3200/api/traces/$traceId" + $spans = @() + foreach ($batch in $trace.batches) { + $service = ($batch.resource.attributes | Where-Object key -eq 'service.name').value.stringValue + foreach ($scope in $batch.scopeSpans) { + foreach ($span in $scope.spans) { + if ($span.name -match 'attachments|publish|process|PUT|DELETE|INSERT|UPDATE|SELECT') { + $spans += [ordered]@{ + service = $service + name = $span.name + kind = $span.kind + span_id = $span.spanId + parent_span_id = $span.parentSpanId + } + } + } + } + } + + $producerCount = @($spans | Where-Object kind -eq 'SPAN_KIND_PRODUCER').Count + $consumerCount = @($spans | Where-Object kind -eq 'SPAN_KIND_CONSUMER').Count + if ($producerCount -lt 2 -or $consumerCount -lt 2) { + throw "trace does not contain both initial and retry producer/consumer spans" + } + + [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + trace_id = $traceId + retry_event = $retryStream.stream.event + retry_attempt = $retryStream.stream.attempt + retry_error_code = $retryStream.stream.error_code + completion_event = "attachment_job_completed" + producer_spans = $producerCount + consumer_spans = $consumerCount + same_trace = $true + synthetic_data_only = $true + spans = $spans + } | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "retry-trace-summary.json") +} +finally { + if ($minio) { Ensure-Running $minio } + if ($worker) { Ensure-Running $worker } + Pop-Location +} + +Write-Host "Evidence saved to $evidenceDir" diff --git a/scripts/incident-telemetry-backend.ps1 b/scripts/incident-telemetry-backend.ps1 new file mode 100644 index 0000000..2d90cc9 --- /dev/null +++ b/scripts/incident-telemetry-backend.ps1 @@ -0,0 +1,149 @@ +param( + [string]$BaseUrl = "http://localhost:8080", + [string]$EnvFile = ".env", + [int]$LoadSeconds = 75, + [int]$Concurrency = 4 +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidenceDir = Join-Path $project "evidence\lab6-$stamp-telemetry-backend" +New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null + +function Query-Scalar([string]$expression) { + try { + $encoded = [uri]::EscapeDataString($expression) + $response = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=$encoded" + if (@($response.data.result).Count -eq 0) { return 0.0 } + return [double]$response.data.result[0].value[1] + } + catch { return 0.0 } +} + +function Convert-MemoryToBytes([string]$value) { + if ($value -notmatch '^([0-9.]+)([KMG]iB)$') { return 0 } + $number = [double]$Matches[1] + $factor = switch ($Matches[2]) { + 'KiB' { 1KB } + 'MiB' { 1MB } + 'GiB' { 1GB } + } + return [long]($number * $factor) +} + +function Capture-Sample { + $stats = docker stats --no-stream --format '{{json .}}' $collector | ConvertFrom-Json + $used = ($stats.MemUsage -split '/')[0].Trim() + return [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + collector_memory = $used + collector_memory_bytes = Convert-MemoryToBytes $used + collector_memory_percent = $stats.MemPerc + exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)' + exporter_queue_capacity = Query-Scalar 'max(otelcol_exporter_queue_capacity)' + failed_spans_total = Query-Scalar 'sum(otelcol_exporter_send_failed_spans_total) or vector(0)' + failed_logs_total = Query-Scalar 'sum(otelcol_exporter_send_failed_log_records_total) or vector(0)' + } +} + +function Ensure-Running([string]$container) { + if ($container -and (docker inspect --format '{{.State.Status}}' $container) -ne 'running') { + docker start $container | Out-Null + } +} + +$samples = @() +$loadJob = $null +Push-Location $project +try { + $collector = docker compose --env-file $EnvFile ps -q otel-collector + $loki = docker compose --env-file $EnvFile ps -q loki + $tempo = docker compose --env-file $EnvFile ps -q tempo + if (-not $collector -or -not $loki -or -not $tempo) { + throw "collector/loki/tempo containers do not exist; start the stack first" + } + + $samples += Capture-Sample + docker stop $loki $tempo | Out-Null + + $makeOutput = & make test BASE_URL=$BaseUrl PYTHON=python 2>&1 + $makeExit = $LASTEXITCODE + $makeOutput | Set-Content -Encoding utf8 (Join-Path $evidenceDir "make-test.txt") + + $loadOutput = Join-Path $evidenceDir "message-load.json" + $loadJob = Start-Job -ArgumentList $project,$BaseUrl,$LoadSeconds,$Concurrency,$loadOutput -ScriptBlock { + param($projectPath,$url,$duration,$workers,$outputPath) + Set-Location $projectPath + $env:PYTHONUTF8 = "1" + & python tests/load.py --base-url $url --duration $duration ` + --concurrency $workers --output $outputPath + [pscustomobject]@{ exit_code = $LASTEXITCODE } + } + + while ($loadJob.State -in @('NotStarted','Running')) { + $samples += Capture-Sample + Start-Sleep -Seconds 5 + } + $loadJobResult = Receive-Job $loadJob + $loadExit = @($loadJobResult | Where-Object { $_.PSObject.Properties.Name -contains 'exit_code' })[-1].exit_code + $samples += Capture-Sample + + $maxMemory = (($samples | ForEach-Object { [long]$_['collector_memory_bytes'] }) | + Measure-Object -Maximum).Maximum + $maxQueue = (($samples | ForEach-Object { [double]$_['exporter_queue_size'] }) | + Measure-Object -Maximum).Maximum + $maxFailedSpans = (($samples | ForEach-Object { [double]$_['failed_spans_total'] }) | + Measure-Object -Maximum).Maximum + $maxFailedLogs = (($samples | ForEach-Object { [double]$_['failed_logs_total'] }) | + Measure-Object -Maximum).Maximum + $loadReport = Get-Content $loadOutput -Raw | ConvertFrom-Json + + $samples | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "collector-samples.json") + [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + make_test_passed = ($makeExit -eq 0 -and ($makeOutput -join "`n") -match 'PASS:') + message_load_exit_code = $loadExit + successful_message_requests = $loadReport.successful_requests + failed_message_requests = $loadReport.failed_requests + collector_memory_limit_bytes = 536870912 + collector_memory_limiter_bytes = 402653184 + max_collector_memory_bytes = $maxMemory + max_exporter_queue_size = $maxQueue + exporter_queue_capacity = 2048 + failed_spans_total = $maxFailedSpans + failed_log_records_total = $maxFailedLogs + bounded = ($maxMemory -lt 536870912 -and $maxQueue -le 2048) + } | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "failure-observation.json") + + if ($makeExit -ne 0 -or $loadExit -ne 0 -or $loadReport.successful_requests -lt 1) { + throw "business traffic failed while telemetry backends were unavailable" + } +} +finally { + if ($loadJob) { Remove-Job $loadJob -Force -ErrorAction SilentlyContinue } + if ($loki) { Ensure-Running $loki } + if ($tempo) { Ensure-Running $tempo } + Pop-Location +} + +$recoveryDeadline = (Get-Date).AddMinutes(2) +$recovery = [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString("o") + loki_ready = $false + tempo_ready = $false + exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)' +} +do { + Start-Sleep -Seconds 5 + try { $recovery.loki_ready = ((Invoke-WebRequest http://localhost:3100/ready -TimeoutSec 5).StatusCode -eq 200) } catch {} + try { $recovery.tempo_ready = ((Invoke-WebRequest http://localhost:3200/ready -TimeoutSec 5).StatusCode -eq 200) } catch {} +} while ((-not $recovery.loki_ready -or -not $recovery.tempo_ready) -and + (Get-Date) -lt $recoveryDeadline) +$recovery.exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)' +$recovery | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "recovery-observation.json") + +Write-Host "Evidence saved to $evidenceDir" diff --git a/scripts/incident-worker-stop.ps1 b/scripts/incident-worker-stop.ps1 new file mode 100644 index 0000000..fd129c2 --- /dev/null +++ b/scripts/incident-worker-stop.ps1 @@ -0,0 +1,85 @@ +param( + [int]$DurationSeconds = 150, + [int]$JobCount = 10, + [string]$BaseUrl = "http://localhost:8080", + [string]$EnvFile = ".env" +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidenceDir = Join-Path $project "evidence\lab6-$stamp-worker-stop" +New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null + +Push-Location $project +try { + $workerContainer = docker compose --env-file $EnvFile ps -q worker + if (-not $workerContainer) { + throw "worker container does not exist; start the stack first" + } + docker stop $workerContainer + python tests/attachment_load.py --base-url $BaseUrl --count $JobCount ` + --output (Join-Path $evidenceDir "load.json") + Start-Sleep -Seconds $DurationSeconds + + $failureAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts" + $firingQueueAlert = @($failureAlerts.data.alerts | Where-Object { + $_.labels.alertname -eq "AttachmentQueueLagHigh" -and $_.state -eq "firing" + }) + $queries = [ordered]@{ + captured_at = (Get-Date).ToUniversalTime().ToString("o") + queue_depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)" + oldest_job_age = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_oldest_job_age_seconds)" + alerts = $failureAlerts + alert_firing = ($firingQueueAlert.Count -gt 0) + } + $queries | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "failure-observation.json") + if ($firingQueueAlert.Count -eq 0) { + throw "AttachmentQueueLagHigh did not reach firing state" + } +} +finally { + if ($workerContainer) { + docker start $workerContainer + } + $deadline = (Get-Date).AddMinutes(4) + do { + Start-Sleep -Seconds 5 + try { + $depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)" + $remaining = [double]$depth.data.result[0].value[1] + } + catch { + $remaining = [double]::PositiveInfinity + } + } while ($remaining -gt 0 -and (Get-Date) -lt $deadline) + $alertDeadline = (Get-Date).AddMinutes(2) + do { + Start-Sleep -Seconds 5 + $recoveryAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts" + $queueAlert = @($recoveryAlerts.data.alerts | Where-Object { + $_.labels.alertname -eq "AttachmentQueueLagHigh" + }) + } while ($queueAlert.Count -gt 0 -and (Get-Date) -lt $alertDeadline) + try { + $recovery = [ordered]@{ + captured_at = (Get-Date).ToUniversalTime().ToString("o") + queue_depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)" + oldest_job_age = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_oldest_job_age_seconds)" + jobs = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=sum%20by%20(outcome)%20(increase(messenger_jobs_processed_total%5B10m%5D))" + alerts = $recoveryAlerts + alert_resolved = ($queueAlert.Count -eq 0) + } + $recovery | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 ` + (Join-Path $evidenceDir "recovery-observation.json") + if ($queueAlert.Count -gt 0) { + throw "AttachmentQueueLagHigh did not resolve before the deadline" + } + } + finally { + Pop-Location + } +} + +Write-Host "Evidence saved to $evidenceDir" diff --git a/scripts/probe-head-sampling.ps1 b/scripts/probe-head-sampling.ps1 new file mode 100644 index 0000000..1fbc214 --- /dev/null +++ b/scripts/probe-head-sampling.ps1 @@ -0,0 +1,92 @@ +param( + [string]$BaseUrl = "http://127.0.0.1:18080", + [string]$EnvFile = ".env", + [int]$RequestCount = 40, + [string]$OutputFile = "" +) + +$ErrorActionPreference = "Stop" +$project = Split-Path -Parent $PSScriptRoot +if (-not $OutputFile) { + $OutputFile = Join-Path $project "evidence\head-sampling-probe.json" +} + +function Get-TempoStatus([string]$traceId) { + try { + $response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 "http://127.0.0.1:3200/api/traces/$traceId" + return [int]$response.StatusCode + } + catch { + if ($_.Exception.Response) { return [int]$_.Exception.Response.StatusCode } + return 0 + } +} + +Push-Location $project +try { + $apiId = docker compose --env-file $EnvFile ps -q api + if (-not $apiId) { throw 'API container is not running' } + $since = [DateTimeOffset]::UtcNow.AddSeconds(-2) + $statuses = @{} + for ($i = 0; $i -lt $RequestCount; $i++) { + try { + $response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 "$BaseUrl/health/not-found" + $status = [int]$response.StatusCode + } + catch { + $status = if ($_.Exception.Response) { [int]$_.Exception.Response.StatusCode } else { 0 } + } + $key = [string]$status + $statuses[$key] = 1 + [int]$statuses[$key] + } + + Start-Sleep -Seconds 15 + $rawLogs = @(docker logs --since $since.ToString('o') $apiId 2>&1) + $events = @() + foreach ($line in $rawLogs) { + $text = [string]$line + $jsonStart = $text.IndexOf('{') + if ($jsonStart -lt 0) { continue } + try { $entry = $text.Substring($jsonStart) | ConvertFrom-Json } catch { continue } + if ($entry.event -eq 'http_request_completed' -and [int]$entry.'http.response.status_code' -eq 500) { + $flags = [Convert]::ToInt32([string]$entry.trace_flags, 16) + $events += [pscustomobject]@{ + trace_id = [string]$entry.trace_id + sampled = (($flags -band 1) -eq 1) + } + } + } + + $sampled = @($events | Where-Object sampled) + $unsampled = @($events | Where-Object { -not $_.sampled }) + if ($sampled.Count -eq 0 -or $unsampled.Count -eq 0) { + throw "Expected both sampled and unsampled 500 events; sampled=$($sampled.Count), unsampled=$($unsampled.Count)" + } + $sampledId = $sampled[0].trace_id + $unsampledId = $unsampled[0].trace_id + $result = [ordered]@{ + captured_at = [DateTimeOffset]::UtcNow.ToString('o') + request_count = $RequestCount + response_status_counts = $statuses + correlated_log_count = $events.Count + sampled_log_count = $sampled.Count + unsampled_log_count = $unsampled.Count + sampled_ratio = [Math]::Round($sampled.Count / [double]$events.Count, 4) + sampled_example = [ordered]@{ + trace_id = $sampledId + tempo_http_status = Get-TempoStatus $sampledId + } + unsampled_example = [ordered]@{ + trace_id = $unsampledId + tempo_http_status = Get-TempoStatus $unsampledId + } + interpretation = 'Head sampling is decided before the HTTP outcome: a non-2xx log can exist even when its trace was not emitted to Tempo.' + } + $parent = Split-Path -Parent $OutputFile + New-Item -ItemType Directory -Force -Path $parent | Out-Null + $result | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 $OutputFile + $result | ConvertTo-Json -Depth 10 +} +finally { + Pop-Location +} diff --git a/scripts/promote-pipeline.ps1 b/scripts/promote-pipeline.ps1 new file mode 100644 index 0000000..6eae43d --- /dev/null +++ b/scripts/promote-pipeline.ps1 @@ -0,0 +1,73 @@ +param( + [Parameter(Mandatory = $true)] + [string]$AttachmentId, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2)] + [int]$PipelineVersion, + [string]$RequestedBy = $env:USERNAME, + [string]$EnvFile = ".env" +) + +$parsedAttachmentId = [guid]::Empty +if (-not [guid]::TryParse($AttachmentId, [ref]$parsedAttachmentId)) { + throw "AttachmentId must be a UUID" +} +if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') { + throw "RequestedBy contains unsupported characters" +} + +$sql = @" +WITH candidate AS ( + SELECT id, pipeline_version + FROM attachments + WHERE id = '$parsedAttachmentId'::uuid AND pipeline_version < $PipelineVersion + FOR UPDATE +), superseded AS ( + UPDATE attachment_jobs j + SET status = 'superseded', published = true, + last_error_code = 'superseded_by_v$PipelineVersion', + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c + WHERE j.attachment_id = c.id AND j.pipeline_version < $PipelineVersion + AND j.status IN ('queued', 'processing') + RETURNING j.id +), created AS ( + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, + next_attempt_at, created_at, updated_at) + SELECT gen_random_uuid(), id, $PipelineVersion, 1, 'queued', false, + clock_timestamp(), clock_timestamp(), clock_timestamp() + FROM candidate + ON CONFLICT (attachment_id, pipeline_version) DO NOTHING + RETURNING id, attachment_id +), promoted AS ( + UPDATE attachments a + SET pipeline_version = $PipelineVersion, status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + FROM candidate c, created n + WHERE a.id = c.id AND n.attachment_id = c.id + RETURNING a.id +), audited AS ( + INSERT INTO attachment_pipeline_promotions + (attachment_id, from_version, to_version, job_id, requested_by) + SELECT c.id, c.pipeline_version, $PipelineVersion, n.id, '$RequestedBy' + FROM candidate c + JOIN created n ON n.attachment_id = c.id + JOIN promoted p ON p.id = c.id + RETURNING job_id +) +SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op'); +"@ + +$result = docker compose --env-file $EnvFile exec -T postgres ` + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql +if ($LASTEXITCODE -ne 0) { + throw "Pipeline promotion transaction failed" +} + +$last = ($result | Select-Object -Last 1).Trim() +if ($last -eq "no-op") { + Write-Output "No-op: attachment is missing, already at v$PipelineVersion, or that version already exists" +} else { + Write-Output "Promoted attachment $parsedAttachmentId to pipeline v$PipelineVersion with job $last" +} diff --git a/scripts/promote-pipeline.sh b/scripts/promote-pipeline.sh new file mode 100644 index 0000000..dbed0ac --- /dev/null +++ b/scripts/promote-pipeline.sh @@ -0,0 +1,59 @@ +#!/bin/sh +set -eu + +attachment_id="${1:?usage: promote-pipeline.sh ATTACHMENT_UUID PIPELINE_VERSION [REQUESTED_BY] [ENV_FILE]}" +pipeline_version="${2:?pipeline version is required}" +requested_by="${3:-operator}" +env_file="${4:-.env}" + +case "$attachment_id" in + ????????-????-????-????-????????????) ;; + *) echo "AttachmentId must be a UUID" >&2; exit 2 ;; +esac +case "$pipeline_version" in + 1|2) ;; + *) echo "PipelineVersion must be 1 or 2" >&2; exit 2 ;; +esac +case "$requested_by" in + *[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;; +esac + +sql="WITH candidate AS ( + SELECT id, pipeline_version FROM attachments + WHERE id = '$attachment_id'::uuid AND pipeline_version < $pipeline_version FOR UPDATE +), superseded AS ( + UPDATE attachment_jobs j SET status = 'superseded', published = true, + last_error_code = 'superseded_by_v$pipeline_version', + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c WHERE j.attachment_id = c.id + AND j.pipeline_version < $pipeline_version AND j.status IN ('queued', 'processing') + RETURNING j.id +), created AS ( + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, + next_attempt_at, created_at, updated_at) + SELECT gen_random_uuid(), id, $pipeline_version, 1, 'queued', false, + clock_timestamp(), clock_timestamp(), clock_timestamp() FROM candidate + ON CONFLICT (attachment_id, pipeline_version) DO NOTHING + RETURNING id, attachment_id +), promoted AS ( + UPDATE attachments a SET pipeline_version = $pipeline_version, + status = 'queued', error_code = NULL, updated_at = clock_timestamp() + FROM candidate c, created n WHERE a.id = c.id AND n.attachment_id = c.id + RETURNING a.id +), audited AS ( + INSERT INTO attachment_pipeline_promotions + (attachment_id, from_version, to_version, job_id, requested_by) + SELECT c.id, c.pipeline_version, $pipeline_version, n.id, '$requested_by' + FROM candidate c JOIN created n ON n.attachment_id = c.id + JOIN promoted p ON p.id = c.id RETURNING job_id +) +SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');" + +result="$(docker compose --env-file "$env_file" exec -T postgres \ + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")" +if [ "$result" = "no-op" ]; then + echo "No-op: attachment is missing, already at v$pipeline_version, or that version already exists" +else + echo "Promoted attachment $attachment_id to pipeline v$pipeline_version with job $result" +fi diff --git a/scripts/replay-failed.ps1 b/scripts/replay-failed.ps1 new file mode 100644 index 0000000..ea9c4bd --- /dev/null +++ b/scripts/replay-failed.ps1 @@ -0,0 +1,58 @@ +param( + [Parameter(Mandatory = $true)] + [string]$JobId, + [string]$RequestedBy = $env:USERNAME, + [string]$EnvFile = ".env" +) + +$parsedJobId = [guid]::Empty +if (-not [guid]::TryParse($JobId, [ref]$parsedJobId)) { + throw "JobId must be a UUID" +} +if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') { + throw "RequestedBy contains unsupported characters" +} + +$sql = @" +WITH candidate AS ( + SELECT j.id, j.attachment_id, j.attempt + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = '$parsedJobId'::uuid + AND j.status = 'failed' AND a.status = 'failed' + AND j.pipeline_version = a.pipeline_version + FOR UPDATE OF j, a +), audited AS ( + INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) + SELECT id, '$RequestedBy', attempt FROM candidate + RETURNING job_id +), reset_job AS ( + UPDATE attachment_jobs j + SET status = 'queued', attempt = 1, published = false, + last_error_code = NULL, next_attempt_at = clock_timestamp(), + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c, audited a + WHERE j.id = c.id AND a.job_id = c.id + RETURNING j.attachment_id +), reset_attachment AS ( + UPDATE attachments a + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + FROM reset_job r + WHERE a.id = r.attachment_id AND a.status = 'failed' + RETURNING a.id +) +SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment) + THEN 'replayed' ELSE 'no-op' END; +"@ + +$result = docker compose --env-file $EnvFile exec -T postgres ` + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql +if ($LASTEXITCODE -ne 0) { + throw "Replay transaction failed" +} + +if (($result | Select-Object -Last 1).Trim() -eq "replayed") { + Write-Output "Replay requested for failed job $parsedJobId by $RequestedBy" +} else { + Write-Output "No-op: job $parsedJobId is missing, not failed, or already recovered" +} diff --git a/scripts/replay-failed.sh b/scripts/replay-failed.sh new file mode 100644 index 0000000..4bfe11b --- /dev/null +++ b/scripts/replay-failed.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +job_id="${1:?usage: replay-failed.sh JOB_UUID [REQUESTED_BY] [ENV_FILE]}" +requested_by="${2:-operator}" +env_file="${3:-.env}" + +case "$job_id" in + ????????-????-????-????-????????????) ;; + *) echo "JobId must be a UUID" >&2; exit 2 ;; +esac +case "$requested_by" in + *[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;; +esac + +sql="WITH candidate AS ( + SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed' + AND j.pipeline_version = a.pipeline_version + FOR UPDATE OF j, a +), audited AS ( + INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt) + SELECT id, '$requested_by', attempt FROM candidate RETURNING job_id +), reset_job AS ( + UPDATE attachment_jobs j SET status = 'queued', attempt = 1, published = false, + last_error_code = NULL, next_attempt_at = clock_timestamp(), + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + FROM candidate c, audited a WHERE j.id = c.id AND a.job_id = c.id + RETURNING j.attachment_id +), reset_attachment AS ( + UPDATE attachments a SET status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + FROM reset_job r WHERE a.id = r.attachment_id AND a.status = 'failed' + RETURNING a.id +) +SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment) + THEN 'replayed' ELSE 'no-op' END;" + +result="$(docker compose --env-file "$env_file" exec -T postgres \ + psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")" +if [ "$result" = "replayed" ]; then + echo "Replay requested for failed job $job_id by $requested_by" +else + echo "No-op: job $job_id is missing, not failed, or already recovered" +fi diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..ddc596a --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,10 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." +if not defined ENV_FILE set "ENV_FILE=.env" + +docker compose --env-file "%ENV_FILE%" stop nginx api redis postgres minio || exit /b 1 +docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 postgres redis minio || exit /b 1 +docker compose --env-file "%ENV_FILE%" run --rm minio-init || exit /b 1 +docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api || exit /b 1 +docker compose --env-file "%ENV_FILE%" up -d --no-deps nginx || exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..d8a5cec --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" +: "${ENV_FILE:=.env}" + +# Preserve named volumes and the network. The migration job is intentionally +# not re-run during a same-version restart. +docker compose --env-file "$ENV_FILE" stop nginx api redis postgres minio +docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 postgres redis minio +docker compose --env-file "$ENV_FILE" run --rm minio-init +docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api +docker compose --env-file "$ENV_FILE" up -d --no-deps nginx diff --git a/scripts/restore-api.cmd b/scripts/restore-api.cmd new file mode 100644 index 0000000..adebf59 --- /dev/null +++ b/scripts/restore-api.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." +if not defined ENV_FILE set "ENV_FILE=.env" + +docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api +exit /b %ERRORLEVEL% diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100644 index 0000000..7518473 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" +: "${ENV_FILE:=.env}" + +docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 \ + --no-deps --scale api=2 api diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 0000000..a881740 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,16 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." + +set /a REPLICA_COUNT=0 +for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do set /a REPLICA_COUNT+=1 +if %REPLICA_COUNT% LSS 2 exit /b 2 + +for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do call :replace_one %%C +exit /b 0 + +:replace_one +docker stop --time 10 %1 >nul || exit /b 1 +docker rm %1 >nul || exit /b 1 +docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api >nul || exit /b 1 +exit /b 0 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 0000000..17a6c19 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" + +original_ids=$(docker compose --env-file .env ps -q api) +count=$(printf '%s\n' "$original_ids" | sed '/^$/d' | wc -l | tr -d ' ') +if [ "$count" -lt 2 ]; then + printf '%s\n' 'Rolling update requires at least two API replicas' >&2 + exit 2 +fi + +for container_id in $original_ids; do + # Replace exactly one original replica and wait for the replacement before + # touching the next one. Named volumes and state stores are not changed. + docker stop --time 10 "$container_id" >/dev/null + docker rm "$container_id" >/dev/null + docker compose --env-file .env up -d --wait --wait-timeout 90 \ + --no-deps --scale api=2 api >/dev/null +done diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..1137fb8 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,26 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +cd /d "%~dp0\.." +if not defined ENV_FILE set "ENV_FILE=.env" + +set "TARGET_CONTAINER=" +set /a MATCHES=0 +for /f "usebackq delims=" %%C in (`docker compose --env-file "%ENV_FILE%" ps -q api`) do ( + set "CONTAINER_ID=%%C" + set "SHORT_ID=!CONTAINER_ID:~0,12!" + set "HOSTNAME_VALUE=" + for /f "usebackq delims=" %%H in (`docker inspect --format "{{.Config.Hostname}}" %%C`) do set "HOSTNAME_VALUE=%%H" + if "!HOSTNAME_VALUE!"=="%TARGET_INSTANCE_ID%" ( + set "TARGET_CONTAINER=%%C" + set /a MATCHES+=1 + ) else if "!SHORT_ID!"=="%TARGET_INSTANCE_ID%" ( + set "TARGET_CONTAINER=%%C" + set /a MATCHES+=1 + ) +) + +if not "!MATCHES!"=="1" exit /b 2 +docker stop --time 5 "!TARGET_CONTAINER!" >nul +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..19370e1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" +: "${ENV_FILE:=.env}" + +target="" +matches=0 +for container_id in $(docker compose --env-file "$ENV_FILE" ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + short_id=$(printf '%s' "$container_id" | cut -c1-12) + if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$short_id" ]; then + target=$container_id + matches=$((matches + 1)) + fi +done + +if [ "$matches" -ne 1 ]; then + printf '%s\n' "Expected one API replica for X-Instance-Id, found $matches" >&2 + exit 2 +fi + +# Five seconds keeps the operator hook below the test's ten-second budget while +# still delivering SIGTERM and allowing Spring's graceful shutdown phase. +docker stop --time 5 "$target" >/dev/null diff --git a/scripts/swarm-lab-down.ps1 b/scripts/swarm-lab-down.ps1 new file mode 100644 index 0000000..e894742 --- /dev/null +++ b/scripts/swarm-lab-down.ps1 @@ -0,0 +1,19 @@ +$ErrorActionPreference = 'Continue' +if (Test-Path variable:PSNativeCommandUseErrorActionPreference) { + $PSNativeCommandUseErrorActionPreference = $false +} + +$nodes = @('hla-swarm-manager', 'hla-swarm-worker1', 'hla-swarm-worker2', 'hla-swarm-registry') +foreach ($node in $nodes) { + $existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}') + if ($existing -eq $node) { + docker rm --force $node | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Cannot remove $node" } + } +} +$existingNetwork = (& docker network ls --filter 'name=^hla-swarm-underlay$' --format '{{.Name}}') +if ($existingNetwork -eq 'hla-swarm-underlay') { + docker network rm hla-swarm-underlay | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Cannot remove hla-swarm-underlay' } +} +Write-Output 'Removed the disposable three-node Swarm lab and its nested volumes.' diff --git a/scripts/swarm-lab-up.ps1 b/scripts/swarm-lab-up.ps1 new file mode 100644 index 0000000..69069a2 --- /dev/null +++ b/scripts/swarm-lab-up.ps1 @@ -0,0 +1,171 @@ +$ErrorActionPreference = 'Continue' +if (Test-Path variable:PSNativeCommandUseErrorActionPreference) { + $PSNativeCommandUseErrorActionPreference = $false +} + +$projectDir = Split-Path -Parent $PSScriptRoot +Set-Location $projectDir + +$manager = 'hla-swarm-manager' +$nodes = @($manager, 'hla-swarm-worker1', 'hla-swarm-worker2') +$network = 'hla-swarm-underlay' +$registry = 'hla-swarm-registry' + +function Invoke-Checked { + param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Command) + $executable = $Command[0] + $arguments = if ($Command.Length -gt 1) { + $Command[1..($Command.Length - 1)] + } else { + @() + } + & $executable @arguments + if ($LASTEXITCODE -ne 0) { + throw "Command failed: $($Command -join ' ')" + } +} + +function Wait-InnerDocker([string]$Node) { + $deadline = (Get-Date).AddSeconds(90) + do { + & docker exec $Node docker info *> $null + if ($LASTEXITCODE -eq 0) { return } + Start-Sleep -Seconds 2 + } while ((Get-Date) -lt $deadline) + throw "Docker daemon did not start in $Node" +} + +function Wait-Service([string]$Service, [string]$Expected, [int]$Seconds = 180) { + $deadline = (Get-Date).AddSeconds($Seconds) + do { + $replicas = (& docker exec $manager docker service ls ` + --filter "name=$Service" --format '{{.Replicas}}' 2>$null | Select-Object -First 1) + if ($replicas -like "$Expected*") { return } + Start-Sleep -Seconds 2 + } while ((Get-Date) -lt $deadline) + throw "Service $Service did not reach $Expected replicas" +} + +foreach ($node in $nodes) { + $existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}') + if ($existing -eq $node) { + throw "Container $node already exists. Run scripts/swarm-lab-down.ps1 first." + } +} +$existingRegistry = (& docker ps --all --filter "name=^/${registry}$" --format '{{.Names}}') +if ($existingRegistry -eq $registry) { + throw "Container $registry already exists. Run scripts/swarm-lab-down.ps1 first." +} + +$envValues = @{} +Get-Content -LiteralPath '.env' | ForEach-Object { + if ($_ -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$') { + $envValues[$matches[1]] = $matches[2].Trim('"', "'") + } +} +$password = $envValues['DATABASE_PASSWORD'] +if ([string]::IsNullOrWhiteSpace($password)) { throw 'DATABASE_PASSWORD is missing from .env' } +$database = if ($envValues['DATABASE_NAME']) { $envValues['DATABASE_NAME'] } else { 'messenger' } +$user = if ($envValues['DATABASE_USER']) { $envValues['DATABASE_USER'] } else { 'messenger' } +$minioRootUser = $envValues['MINIO_ROOT_USER'] +$minioRootPassword = $envValues['MINIO_ROOT_PASSWORD'] +$s3AccessKey = $envValues['S3_ACCESS_KEY_ID'] +$s3SecretKey = $envValues['S3_SECRET_ACCESS_KEY'] +$s3Bucket = if ($envValues['S3_BUCKET']) { $envValues['S3_BUCKET'] } else { 'messenger-attachments' } +$s3Region = if ($envValues['S3_REGION']) { $envValues['S3_REGION'] } else { 'us-east-1' } +foreach ($required in @{ + MINIO_ROOT_USER = $minioRootUser + MINIO_ROOT_PASSWORD = $minioRootPassword + S3_ACCESS_KEY_ID = $s3AccessKey + S3_SECRET_ACCESS_KEY = $s3SecretKey +}.GetEnumerator()) { + if ([string]::IsNullOrWhiteSpace($required.Value)) { throw "$($required.Key) is missing from .env" } +} + +$existingNetwork = (& docker network ls --filter "name=^${network}$" --format '{{.Name}}') +if ($existingNetwork -ne $network) { Invoke-Checked docker network create $network } + +Invoke-Checked docker run --detach --name $registry --hostname $registry ` + --network $network --publish 127.0.0.1:15000:5000 registry:2 + +Invoke-Checked docker run --privileged --detach --name $manager --hostname $manager ` + --network $network --publish 127.0.0.1:18080:8080 ` + --volume "${projectDir}:/project:ro" --env DOCKER_TLS_CERTDIR= docker:28-dind ` + --storage-driver=overlay2 --insecure-registry=${registry}:5000 +foreach ($worker in $nodes[1..2]) { + Invoke-Checked docker run --privileged --detach --name $worker --hostname $worker ` + --network $network --env DOCKER_TLS_CERTDIR= docker:28-dind ` + --storage-driver=overlay2 --insecure-registry=${registry}:5000 +} +foreach ($node in $nodes) { Wait-InnerDocker $node } + +$managerIp = (& docker inspect --format ` + '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $manager).Trim() +Invoke-Checked docker exec $manager docker swarm init --advertise-addr $managerIp +$token = (& docker exec $manager docker swarm join-token --quiet worker).Trim() +foreach ($worker in $nodes[1..2]) { + Invoke-Checked docker exec $worker docker swarm join --token $token "$managerIp`:2377" +} +Invoke-Checked docker exec $manager docker node update ` + --label-add hla.state=true --label-add hla.lb=true $manager + +Invoke-Checked docker compose --env-file .env build api +Invoke-Checked docker tag hla-lab6-app:local localhost:15000/hla-lab6-api:swarm +Invoke-Checked docker build --file swarm/nginx.Dockerfile ` + --tag localhost:15000/hla-lab6-nginx:swarm . +Invoke-Checked docker push localhost:15000/hla-lab6-api:swarm +Invoke-Checked docker push localhost:15000/hla-lab6-nginx:swarm + +Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" ` + --env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" ` + --env "MINIO_ROOT_USER=$minioRootUser" --env "MINIO_ROOT_PASSWORD=$minioRootPassword" ` + --env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" ` + --env "S3_BUCKET=$s3Bucket" ` + $manager docker stack deploy --compose-file /project/swarm/stack.state.yaml hla-state +Wait-Service 'hla-state_postgres' '1/1' +Wait-Service 'hla-state_redis' '1/1' +Wait-Service 'hla-state_minio' '1/1' + +$deadline = (Get-Date).AddSeconds(180) +do { + $minioInitState = (& docker exec $manager docker service ps hla-state_minio-init ` + --no-trunc --format '{{.CurrentState}}' | Select-Object -First 1) + if ($minioInitState -like 'Complete*') { break } + Start-Sleep -Seconds 2 +} while ((Get-Date) -lt $deadline) +if ($minioInitState -notlike 'Complete*') { throw 'MinIO bucket bootstrap did not complete' } + +Invoke-Checked docker exec $manager docker service create --name hla-migrate ` + --mode replicated-job --replicas 1 --max-concurrent 1 ` + --network hla-state_backend --constraint node.labels.hla.state==true ` + --restart-condition on-failure --restart-delay 3s ` + --limit-cpu 0.75 --limit-memory 768M ` + --env "DATABASE_URL=jdbc:postgresql://hla-state_postgres:5432/$database" ` + --env "DATABASE_USER=$user" --env "DATABASE_PASSWORD=$password" ` + --env FLYWAY_ENABLED=true --env MIGRATE_ONLY=true ` + --env SPRING_MAIN_WEB_APPLICATION_TYPE=none ` + --env "S3_ENDPOINT=http://hla-state_minio:9000" ` + --env "S3_BUCKET=$s3Bucket" --env "S3_REGION=$s3Region" ` + --env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" ` + hla-swarm-registry:5000/hla-lab6-api:swarm +$deadline = (Get-Date).AddSeconds(180) +do { + $state = (& docker exec $manager docker service ps hla-migrate ` + --no-trunc --format '{{.CurrentState}}' | Select-Object -First 1) + if ($state -like 'Complete*') { break } + Start-Sleep -Seconds 2 +} while ((Get-Date) -lt $deadline) +if ($state -notlike 'Complete*') { throw 'Migration job did not complete' } +Invoke-Checked docker exec $manager docker service rm hla-migrate + +Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" ` + --env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" ` + --env "S3_BUCKET=$s3Bucket" --env "S3_REGION=$s3Region" ` + --env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" ` + $manager docker stack deploy --compose-file /project/swarm/stack.app.yaml hla-app +Wait-Service 'hla-app_api' '2/2' +Wait-Service 'hla-app_nginx' '1/1' + +Write-Output 'Swarm lab is ready at http://localhost:18080' +Invoke-Checked docker exec $manager docker node ls +Invoke-Checked docker exec $manager docker service ps hla-app_api diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..d8f971a --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'mtusi-messenger' diff --git a/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java new file mode 100644 index 0000000..4e0eb36 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java @@ -0,0 +1,20 @@ +package com.example.mtusi_messenger; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication +@EnableScheduling +public class MtusiMessengerApplication { + + public static void main(String[] args) { + ConfigurableApplicationContext context = SpringApplication.run( + MtusiMessengerApplication.class, args); + if (Boolean.parseBoolean(System.getenv("MIGRATE_ONLY"))) { + System.exit(SpringApplication.exit(context)); + } + } + +} diff --git a/src/main/java/com/example/mtusi_messenger/cache/UserCache.java b/src/main/java/com/example/mtusi_messenger/cache/UserCache.java new file mode 100644 index 0000000..04ad803 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/cache/UserCache.java @@ -0,0 +1,83 @@ +package com.example.mtusi_messenger.cache; + +import com.example.mtusi_messenger.model.User; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; + +@Component +public class UserCache { + private static final Logger log = LoggerFactory.getLogger(UserCache.class); + private static final String KEY_PREFIX = "messenger:user:"; + + private final StringRedisTemplate redis; + private final ObjectMapper mapper; + private final Duration ttl; + + public UserCache( + StringRedisTemplate redis, + ObjectMapper mapper, + @Value("${messenger.cache.user-ttl-seconds:300}") long ttlSeconds + ) { + if (ttlSeconds <= 0) { + throw new IllegalArgumentException("User cache TTL must be positive"); + } + this.redis = redis; + this.mapper = mapper; + this.ttl = Duration.ofSeconds(ttlSeconds); + } + + public Optional find(UUID userId) { + try { + String value = redis.opsForValue().get(key(userId)); + if (value == null) { + return Optional.empty(); + } + User cachedUser = mapper.readValue(value, User.class); + if (cachedUser == null) { + log.warn("Ignoring null Redis user cache entry for {}", userId); + evict(userId); + return Optional.empty(); + } + return Optional.of(cachedUser); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; using PostgreSQL fallback"); + return Optional.empty(); + } catch (JsonProcessingException e) { + log.warn("Ignoring malformed Redis user cache entry for {}", userId); + evict(userId); + return Optional.empty(); + } + } + + public void put(User user) { + try { + redis.opsForValue().set(key(user.id()), mapper.writeValueAsString(user), ttl); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; user was not cached"); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize user cache entry", e); + } + } + + public void evict(UUID userId) { + try { + redis.delete(key(userId)); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; cache eviction was skipped"); + } + } + + private String key(UUID userId) { + return KEY_PREFIX + userId; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java b/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java new file mode 100644 index 0000000..568a65e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java @@ -0,0 +1,28 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer; + +import tools.jackson.databind.cfg.CoercionAction; +import tools.jackson.databind.cfg.CoercionInputShape; +import tools.jackson.databind.type.LogicalType; + +@Configuration +public class JacksonConfig { + + @Bean + JsonMapperBuilderCustomizer strictStringTypes() { + return builder -> builder.withCoercionConfig( + LogicalType.Textual, + config -> { + config.setCoercion( + CoercionInputShape.Integer, CoercionAction.Fail); + config.setCoercion( + CoercionInputShape.Float, CoercionAction.Fail); + config.setCoercion( + CoercionInputShape.Boolean, CoercionAction.Fail); + } + ); + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java b/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java new file mode 100644 index 0000000..14cc24f --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.config; + + +import io.swagger.v3.oas.annotations.enums.SecuritySchemeType; +import io.swagger.v3.oas.annotations.security.SecurityScheme; +import org.springframework.context.annotation.Configuration; + +@SecurityScheme( + name = "basicAuth", + type = SecuritySchemeType.HTTP, + scheme = "basic" +) +@SecurityScheme( + name = "bearerAuth", + type = SecuritySchemeType.HTTP, + scheme = "bearer", + bearerFormat = "opaque session token" +) +@Configuration +public class OpenApiConfig { +} diff --git a/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java b/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java new file mode 100644 index 0000000..98436f1 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class PasswordConfig { + @Bean + PasswordEncoder passwordEncoder() { + return Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java b/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java new file mode 100644 index 0000000..ec334a9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/RabbitJobConfig.java @@ -0,0 +1,57 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RabbitJobConfig { + @Bean + DirectExchange attachmentExchange(@Value("${messenger.jobs.exchange}") String name) { + return new DirectExchange(name, true, false); + } + + @Bean + Queue attachmentQueue( + @Value("${messenger.jobs.queue}") String name, + @Value("${messenger.jobs.dead-letter-exchange}") String deadLetterExchange, + @Value("${messenger.jobs.dead-letter-routing-key}") String deadLetterRoutingKey) { + return QueueBuilder.durable(name) + .withArgument("x-dead-letter-exchange", deadLetterExchange) + .withArgument("x-dead-letter-routing-key", deadLetterRoutingKey) + .build(); + } + + @Bean + DirectExchange attachmentDeadLetterExchange( + @Value("${messenger.jobs.dead-letter-exchange}") String name) { + return new DirectExchange(name, true, false); + } + + @Bean + Queue attachmentDeadLetterQueue(@Value("${messenger.jobs.dead-letter-queue}") String name) { + return QueueBuilder.durable(name).build(); + } + + @Bean + Binding attachmentBinding( + @Qualifier("attachmentQueue") Queue attachmentQueue, + @Qualifier("attachmentExchange") DirectExchange attachmentExchange, + @Value("${messenger.jobs.routing-key}") String routingKey) { + return BindingBuilder.bind(attachmentQueue).to(attachmentExchange).with(routingKey); + } + + @Bean + Binding attachmentDeadLetterBinding( + @Qualifier("attachmentDeadLetterQueue") Queue queue, + @Qualifier("attachmentDeadLetterExchange") DirectExchange exchange, + @Value("${messenger.jobs.dead-letter-routing-key}") String routingKey) { + return BindingBuilder.bind(queue).to(exchange).with(routingKey); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java new file mode 100644 index 0000000..f2b9ae4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java @@ -0,0 +1,107 @@ +package com.example.mtusi_messenger.config; + +import com.example.mtusi_messenger.security.ApiAuthenticationFilter; +import com.example.mtusi_messenger.security.DatabaseAuthenticationProvider; +import com.example.mtusi_messenger.security.RestAuthenticationEntryPoint; +import com.example.mtusi_messenger.service.AuthenticationService; +import com.example.mtusi_messenger.web.ReadinessGuardFilter; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.ProviderManager; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.AnonymousAuthenticationFilter; + +@Configuration +@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) +public class SecurityConfig { + @Bean + AuthenticationManager authenticationManager(DatabaseAuthenticationProvider provider) { + return new ProviderManager(provider); + } + + @Bean + RestAuthenticationEntryPoint basicAuthenticationEntryPoint(ObjectMapper mapper) { + return new RestAuthenticationEntryPoint( + mapper, + "Basic realm=\"messenger\", charset=\"UTF-8\""); + } + + @Bean + RestAuthenticationEntryPoint bearerAuthenticationEntryPoint(ObjectMapper mapper) { + return new RestAuthenticationEntryPoint(mapper, "Bearer realm=\"messenger\""); + } + + @Bean + ApiAuthenticationFilter apiAuthenticationFilter( + AuthenticationManager authenticationManager, + AuthenticationService authenticationService, + @Qualifier("basicAuthenticationEntryPoint") RestAuthenticationEntryPoint basicAuthenticationEntryPoint, + @Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint + ) { + return new ApiAuthenticationFilter( + authenticationManager, + authenticationService, + basicAuthenticationEntryPoint, + bearerAuthenticationEntryPoint); + } + + @Bean + FilterRegistrationBean disableServletFilterRegistration( + ApiAuthenticationFilter filter + ) { + var registration = new FilterRegistrationBean<>(filter); + registration.setEnabled(false); + return registration; + } + + @Bean + FilterRegistrationBean disableReadinessGuardServletRegistration( + ReadinessGuardFilter filter + ) { + var registration = new FilterRegistrationBean<>(filter); + registration.setEnabled(false); + return registration; + } + + @Bean + SecurityFilterChain securityFilterChain( + HttpSecurity http, + ApiAuthenticationFilter apiAuthenticationFilter, + ReadinessGuardFilter readinessGuardFilter, + @Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint + ) throws Exception { + http + .csrf(csrf -> csrf.disable()) + .formLogin(form -> form.disable()) + .httpBasic(basic -> basic.disable()) + .logout(logout -> logout.disable()) + .requestCache(cache -> cache.disable()) + .sessionManagement(session -> session + .sessionCreationPolicy(SessionCreationPolicy.STATELESS)) + .exceptionHandling(errors -> errors + .authenticationEntryPoint(bearerAuthenticationEntryPoint)) + .authorizeHttpRequests(authorize -> authorize + .requestMatchers(HttpMethod.GET, "/health/**").permitAll() + .requestMatchers( + "/swagger-ui.html", + "/swagger-ui/**", + "/v3/api-docs/**" + ).permitAll() + .requestMatchers(HttpMethod.POST, "/api/v1/users").permitAll() + .anyRequest().authenticated()) + // Refuse business traffic before authentication touches storage. + // Nginx retries this 503 on another peer and passively excludes + // the refusing replica according to max_fails/fail_timeout. + .addFilterBefore(readinessGuardFilter, AnonymousAuthenticationFilter.class) + .addFilterAfter(apiAuthenticationFilter, ReadinessGuardFilter.class); + return http.build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/WebConfig.java b/src/main/java/com/example/mtusi_messenger/config/WebConfig.java new file mode 100644 index 0000000..ac26bb3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/WebConfig.java @@ -0,0 +1,19 @@ +package com.example.mtusi_messenger.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class WebConfig { + @Bean + ObjectMapper objectMapper() { + return JsonMapper.builder().findAndAddModules() + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java b/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java new file mode 100644 index 0000000..72350f5 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/AttachmentController.java @@ -0,0 +1,75 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.dto.AttachmentResponse; +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.security.SessionPrincipal; +import com.example.mtusi_messenger.service.AttachmentService; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.servlet.http.HttpServletRequest; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static com.example.mtusi_messenger.service.UuidValidator.parse; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/api/v1") +@SecurityRequirement(name = "bearerAuth") +public class AttachmentController { + private final AttachmentService attachments; + + @PostMapping(value = "/chats/{chatId}/attachments", + consumes = {"image/png", "image/jpeg"}, + produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity upload( + @AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId, + @RequestParam String filename, + @RequestHeader(HttpHeaders.CONTENT_TYPE) String contentType, + HttpServletRequest request + ) throws IOException { + Attachment attachment = attachments.upload(parse(chatId), principal.user().id(), filename, + contentType, request.getContentLengthLong(), request.getInputStream()); + return ResponseEntity.status(HttpStatus.ACCEPTED).body(AttachmentResponse.from(attachment)); + } + + @GetMapping("/attachments/{attachmentId}") + public AttachmentResponse metadata( + @AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String attachmentId + ) { + return AttachmentResponse.from(attachments.getAccessible(parse(attachmentId), principal.user().id())); + } + + @GetMapping("/attachments/{attachmentId}/content") + public ResponseEntity download( + @AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String attachmentId, + @RequestParam(defaultValue = "original") String variant + ) { + AttachmentService.Download download = attachments.download( + parse(attachmentId), principal.user().id(), variant); + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(download.contentType())) + .contentLength(download.bytes().length) + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.inline() + .filename(download.filename(), StandardCharsets.UTF_8).build().toString()) + .body(download.bytes()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/ChatController.java b/src/main/java/com/example/mtusi_messenger/controller/ChatController.java new file mode 100644 index 0000000..ff02c9b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/ChatController.java @@ -0,0 +1,83 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.exception.ApiExceptions; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.model.Chat; +import com.example.mtusi_messenger.service.ChatService; +import com.example.mtusi_messenger.security.SessionPrincipal; +import java.util.List; +import java.util.UUID; + +import static com.example.mtusi_messenger.service.UuidValidator.parse; + +@RestController +@RequestMapping("/api/v1/chats") +@SecurityRequirement(name = "bearerAuth") +@RequiredArgsConstructor +public class ChatController { + + private final ChatService chats; + + @PostMapping + public ResponseEntity create(@AuthenticationPrincipal SessionPrincipal principal, + @Valid @RequestBody CreateChatRequest body + ) { + UUID userId = principal.user().id(); + Chat c = chats.create(body, userId); + return ResponseEntity.status(HttpStatus.CREATED).body(ChatResponse.from(c)); + } + + @GetMapping + public ChatPageResponse list(@AuthenticationPrincipal SessionPrincipal principal, + @RequestParam(required = false) String limit, + @RequestParam(required = false) String cursor + ) { + UUID userId = principal.user().id(); + int parsedLimit = validateLimit(limit); + validateCursorPresence(cursor); + List page = chats.page(userId, parsedLimit, cursor); + boolean hasNext = page.size() > parsedLimit; + if (hasNext) page = page.subList(0, parsedLimit); + return new ChatPageResponse(page.stream().map(ChatResponse::from).toList(), page.isEmpty() ? null : + chats.nextCursor(userId, page.get(page.size() - 1), hasNext)); + } + + @GetMapping("/{chatId}") + public ChatResponse get(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId + ) { + UUID userId = principal.user().id(); + return ChatResponse.from(chats.getAccessible(parse(chatId), userId)); + } + + private static int validateLimit(String limit) { + if (limit == null) { + return 50; + } + if (!limit.matches("[0-9]+")) { + throw ApiExceptions.invalid("Invalid limit"); + } + try { + int value = Integer.parseInt(limit); + + if (value < 1 || value > 100) { + throw ApiExceptions.invalid("Invalid limit"); + } + + return value; + } catch (NumberFormatException ex) { + throw ApiExceptions.invalid("Invalid limit"); + } + } + + private static void validateCursorPresence(String c) { + if (c != null && c.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/HealthController.java b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java new file mode 100644 index 0000000..e4bfec3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java @@ -0,0 +1,32 @@ +package com.example.mtusi_messenger.controller; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import com.example.mtusi_messenger.dto.HealthResponse; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.service.ReadinessService; + +@RestController +public class HealthController { + private final ReadinessService readiness; + + public HealthController(ReadinessService readiness) { + this.readiness = readiness; + } + + @GetMapping("/health/live") + public ResponseEntity live() { + return ResponseEntity.ok(new HealthResponse("ok")); + } + + @GetMapping("/health/ready") + public ResponseEntity ready() { + // Always probe dependencies, even after a failure, so a restored schema + // can move this process back to ACCEPTING_TRAFFIC without a restart. + if (!readiness.probeDependencies() || !readiness.acceptsBusinessTraffic()) { + throw ApiExceptions.unavailable("Application is not accepting traffic"); + } + return ResponseEntity.ok(new HealthResponse("ok")); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/MessageController.java b/src/main/java/com/example/mtusi_messenger/controller/MessageController.java new file mode 100644 index 0000000..e26c4cb --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/MessageController.java @@ -0,0 +1,79 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.exception.ApiExceptions; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.service.MessageService; +import com.example.mtusi_messenger.security.SessionPrincipal; +import java.util.List; +import java.util.UUID; + +import static com.example.mtusi_messenger.service.UuidValidator.parse; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/api/v1/chats/{chatId}/messages") +@SecurityRequirement(name = "bearerAuth") +public class MessageController { + private static final String IDEMPOTENCY_KEY_PATTERN = "[A-Za-z0-9._:-]{1,128}"; + private final MessageService messages; + + @PostMapping + public ResponseEntity create(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId, + @RequestHeader(name = "Idempotency-Key", required = false) String idempotencyKey, + @Valid @RequestBody CreateMessageRequest body + ) { + if (idempotencyKey == null || !idempotencyKey.matches(IDEMPOTENCY_KEY_PATTERN)) { + throw ApiExceptions.invalid("Invalid or missing Idempotency-Key"); + } + UUID userId = principal.user().id(); + UUID parsedChatId = parse(chatId); + Message m = messages.create(parsedChatId, userId, body, idempotencyKey); + return ResponseEntity.status(HttpStatus.CREATED).body(MessageResponse.from(m)); + } + + @GetMapping + public MessagePageResponse list(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId, + @RequestParam(required = false) String limit, + @RequestParam(required=false) String cursor + ) { + UUID userId = principal.user().id(); + UUID parsedChatId = parse(chatId); + int parsedLimit = validateLimit(limit); + if (cursor != null && cursor.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + List page = messages.page(parsedChatId, userId, parsedLimit, cursor); + boolean hasNext = page.size() > parsedLimit; if (hasNext) page = page.subList(0, parsedLimit); + return new MessagePageResponse(page.stream().map(MessageResponse::from).toList(), + page.isEmpty() ? null : messages.nextCursor(parsedChatId, page.get(page.size()-1), hasNext)); + } + + private static int validateLimit(String limit) { + if (limit == null) { + return 50; + } + if (!limit.matches("[0-9]+")) { + throw ApiExceptions.invalid("Invalid limit"); + } + try { + int value = Integer.parseInt(limit); + + if (value < 1 || value > 100) { + throw ApiExceptions.invalid("Invalid limit"); + } + + return value; + } catch (NumberFormatException ex) { + throw ApiExceptions.invalid("Invalid limit"); + } + } + +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/SessionController.java b/src/main/java/com/example/mtusi_messenger/controller/SessionController.java new file mode 100644 index 0000000..f79528f --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/SessionController.java @@ -0,0 +1,41 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.dto.SessionResponse; +import com.example.mtusi_messenger.dto.UserResponse; +import com.example.mtusi_messenger.service.AuthenticationService; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.security.SessionPrincipal; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/auth/sessions") +@RequiredArgsConstructor +public class SessionController { + private final AuthenticationService authentication; + + @PostMapping + @SecurityRequirement(name = "basicAuth") + public ResponseEntity create(@AuthenticationPrincipal User user) { + var issued = authentication.issueSession(user); + return ResponseEntity.status(HttpStatus.CREATED).body(new SessionResponse( + issued.token(), + "Bearer", + issued.expiresAt(), + UserResponse.from(issued.user()))); + } + + @DeleteMapping("/current") + @SecurityRequirement(name = "bearerAuth") + public ResponseEntity revoke(@AuthenticationPrincipal SessionPrincipal principal) { + authentication.revoke(principal); + return ResponseEntity.noContent().build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/UserController.java b/src/main/java/com/example/mtusi_messenger/controller/UserController.java new file mode 100644 index 0000000..0d2d9cf --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/UserController.java @@ -0,0 +1,32 @@ +package com.example.mtusi_messenger.controller; + +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.service.UserService; +import com.example.mtusi_messenger.security.SessionPrincipal; + +@RestController +@RequestMapping("/api/v1/users") +@RequiredArgsConstructor +public class UserController { + + private final UserService users; + + @PostMapping + public ResponseEntity create(@Valid @RequestBody CreateUserRequest request) { + var user = users.create(request); + return ResponseEntity.status(HttpStatus.CREATED).body(UserResponse.from(user)); + } + + @GetMapping("/me") + @SecurityRequirement(name = "bearerAuth") + public UserResponse me(@AuthenticationPrincipal SessionPrincipal principal) { + return UserResponse.from(principal.user()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java b/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java new file mode 100644 index 0000000..7d7d2fa --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/AttachmentResponse.java @@ -0,0 +1,36 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.Attachment; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record AttachmentResponse(UUID id, + UUID chat_id, + UUID owner_id, + String filename, + String content_type, + long size_bytes, + String sha256, + String status, + List variants, + String error_code, + Instant created_at, + Instant updated_at) { + public static AttachmentResponse from(Attachment attachment) { + return new AttachmentResponse( + attachment.id(), + attachment.chatId(), + attachment.ownerId(), + attachment.filename(), + attachment.contentType(), + attachment.sizeBytes(), + attachment.sha256(), + attachment.status(), + attachment.variants().stream().map(AttachmentVariantResponse::from).toList(), + attachment.errorCode(), + attachment.createdAt(), + attachment.updatedAt()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java b/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java new file mode 100644 index 0000000..7bbdc3c --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/AttachmentVariantResponse.java @@ -0,0 +1,16 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.AttachmentVariant; + +public record AttachmentVariantResponse(String name, + String content_type, + long size_bytes, + int width, + int height, + String sha256) { + public static AttachmentVariantResponse from(AttachmentVariant variant) { + return new AttachmentVariantResponse( + variant.name(), variant.contentType(), variant.sizeBytes(), + variant.width(), variant.height(), variant.sha256()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java new file mode 100644 index 0000000..e1d54bf --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +import java.util.List; + +public record ChatPageResponse( + List items, + String next_cursor +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java new file mode 100644 index 0000000..96b2abb --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.Chat; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record ChatResponse(UUID id, + String title, + List member_ids, + Instant created_at) { + public static ChatResponse from(Chat c) { + return new ChatResponse( + c.id(), + c.title(), + c.memberIds(), + c.createdAt()); } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java new file mode 100644 index 0000000..3f79234 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java @@ -0,0 +1,19 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.NotUnicodeBlank; +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.Size; +import java.util.List; +import java.util.UUID; + +public record CreateChatRequest( + @NotUnicodeBlank + @UnicodeSize(min = 1, max = 100) + String title, + + @NotEmpty + @Size(min = 1, max = 100) + List member_ids +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java new file mode 100644 index 0000000..202f911 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java @@ -0,0 +1,31 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public record CreateMessageRequest( + @NotNull @UnicodeSize(max = 4000) String text, + @Size(max = 10) List attachment_ids +) { + public CreateMessageRequest { + attachment_ids = attachment_ids == null ? List.of() : List.copyOf(attachment_ids); + } + + @AssertTrue(message = "text or attachment_ids must be present") + public boolean isContentPresent() { + return text != null && (!text.isBlank() || !attachment_ids.isEmpty()); + } + + @AssertTrue(message = "attachment_ids must be unique and non-null") + public boolean isAttachmentIdsValid() { + return attachment_ids != null + && attachment_ids.stream().noneMatch(java.util.Objects::isNull) + && new HashSet<>(attachment_ids).size() == attachment_ids.size(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java new file mode 100644 index 0000000..6d6e5a9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java @@ -0,0 +1,20 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.NotUnicodeBlank; +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; + +public record CreateUserRequest( + @NotBlank + @Pattern(regexp = "[a-z0-9_]{3,32}") + String username, + + @NotUnicodeBlank + @UnicodeSize(min = 1, max = 100) + String display_name, + + @NotUnicodeBlank + @UnicodeSize(min = 12, max = 128) + String password +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java new file mode 100644 index 0000000..4b1a2a9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +public record ErrorResponse(ErrorBody error) { + public record ErrorBody(String code, + String message, + String request_id) + {} +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java b/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java new file mode 100644 index 0000000..9630b10 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java @@ -0,0 +1,3 @@ +package com.example.mtusi_messenger.dto; + +public record HealthResponse(String status) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java new file mode 100644 index 0000000..db72eee --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +import java.util.List; + +public record MessagePageResponse( + List items, + String next_cursor +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java new file mode 100644 index 0000000..2f59187 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java @@ -0,0 +1,22 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.Message; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record MessageResponse(UUID id, + UUID chat_id, + UUID sender_id, + String text, + Instant created_at, + List attachment_ids) { + public static MessageResponse from(Message m) { + return new MessageResponse(m.id(), + m.chatId(), + m.senderId(), + m.text(), + m.createdAt(), + m.attachmentIds()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java b/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java new file mode 100644 index 0000000..64b54f9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java @@ -0,0 +1,10 @@ +package com.example.mtusi_messenger.dto; + +import java.time.Instant; + +public record SessionResponse( + String token, + String token_type, + Instant expires_at, + UserResponse user +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java b/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java new file mode 100644 index 0000000..e84bcb7 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.User; +import java.time.Instant; +import java.util.UUID; + +public record UserResponse(UUID id, + String username, + String display_name, + Instant created_at) { + public static UserResponse from(User u) { + return new UserResponse( + u.id(), + u.username(), + u.displayName(), + u.createdAt()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/exception/ApiException.java b/src/main/java/com/example/mtusi_messenger/exception/ApiException.java new file mode 100644 index 0000000..2c07f22 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/exception/ApiException.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.exception; + +import org.springframework.http.HttpStatus; + +public class ApiException extends RuntimeException { + private final String code; private final HttpStatus status; private final String authenticate; + public ApiException(String code, HttpStatus status, String message) { this(code, status, message, null); } + public ApiException(String code, HttpStatus status, String message, String authenticate) { + super(message); this.code = code; this.status = status; this.authenticate = authenticate; + } + public String code() { return code; } + public HttpStatus status() { return status; } + public String authenticate() { return authenticate; } +} diff --git a/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java new file mode 100644 index 0000000..d7cecee --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java @@ -0,0 +1,17 @@ +package com.example.mtusi_messenger.exception; + +import org.springframework.http.HttpStatus; + +public final class ApiExceptions { + private ApiExceptions() {} + public static ApiException invalid(String m) { return new ApiException("invalid_request", HttpStatus.BAD_REQUEST, m); } + public static ApiException unauthorized(String m) { return unauthorizedBearer(m); } + public static ApiException unauthorizedBasic(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Basic realm=\"messenger\", charset=\"UTF-8\""); } + public static ApiException unauthorizedBearer(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Bearer realm=\"messenger\""); } + public static ApiException notFound(String m) { return new ApiException("not_found", HttpStatus.NOT_FOUND, m); } + public static ApiException conflict(String m) { return new ApiException("conflict", HttpStatus.CONFLICT, m); } + public static ApiException payloadTooLarge(String m) { return new ApiException("payload_too_large", HttpStatus.PAYLOAD_TOO_LARGE, m); } + public static ApiException unsupportedMediaType(String m) { return new ApiException("unsupported_media_type", HttpStatus.UNSUPPORTED_MEDIA_TYPE, m); } + public static ApiException tooManyRequests(String m) { return new ApiException("too_many_requests", HttpStatus.TOO_MANY_REQUESTS, m); } + public static ApiException unavailable(String m) { return new ApiException("unavailable", HttpStatus.SERVICE_UNAVAILABLE, m); } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/Attachment.java b/src/main/java/com/example/mtusi_messenger/model/Attachment.java new file mode 100644 index 0000000..acb2515 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Attachment.java @@ -0,0 +1,38 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record Attachment(UUID id, + UUID chatId, + UUID ownerId, + String filename, + String contentType, + long sizeBytes, + String sha256, + String objectKey, + String stagingKey, + String status, + List variants, + String errorCode, + Instant createdAt, + Instant updatedAt) { + public Attachment { + variants = variants == null ? List.of() : List.copyOf(variants); + } + + public Attachment(UUID id, + UUID chatId, + UUID ownerId, + String filename, + String contentType, + long sizeBytes, + String sha256, + String objectKey, + Instant createdAt, + Instant updatedAt) { + this(id, chatId, ownerId, filename, contentType, sizeBytes, sha256, objectKey, + null, "ready", List.of(), null, createdAt, updatedAt); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java new file mode 100644 index 0000000..a189403 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/AttachmentJob.java @@ -0,0 +1,40 @@ +package com.example.mtusi_messenger.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.UUID; + +public record AttachmentJob( + @JsonProperty("schema_version") int schemaVersion, + @JsonProperty("job_id") UUID jobId, + String type, + @JsonProperty("attachment_id") UUID attachmentId, + @JsonProperty("pipeline_version") int pipelineVersion, + int attempt, + @JsonProperty("created_at") Instant createdAt) { + + public static final int SCHEMA_VERSION = 1; + public static final int MIN_PIPELINE_VERSION = 1; + public static final int CURRENT_PIPELINE_VERSION = 2; + public static final String TYPE = "attachment.process.v1"; + + public AttachmentJob { + if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type) + || pipelineVersion < MIN_PIPELINE_VERSION + || pipelineVersion > CURRENT_PIPELINE_VERSION || attempt < 1) { + throw new IllegalArgumentException("Unsupported attachment job message"); + } + } + + public static AttachmentJob first( + UUID jobId, UUID attachmentId, int pipelineVersion, Instant createdAt) { + return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId, + pipelineVersion, 1, createdAt); + } + + public AttachmentJob nextAttempt(int nextAttempt) { + return new AttachmentJob(schemaVersion, jobId, type, attachmentId, + pipelineVersion, nextAttempt, createdAt); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java b/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java new file mode 100644 index 0000000..ff4e809 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/AttachmentVariant.java @@ -0,0 +1,10 @@ +package com.example.mtusi_messenger.model; + +public record AttachmentVariant(String name, + String contentType, + long sizeBytes, + int width, + int height, + String sha256, + String objectKey) { +} diff --git a/src/main/java/com/example/mtusi_messenger/model/Chat.java b/src/main/java/com/example/mtusi_messenger/model/Chat.java new file mode 100644 index 0000000..829d99e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Chat.java @@ -0,0 +1,11 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record Chat(UUID id, String title, List memberIds, Instant createdAt) { + public Chat { + memberIds = List.copyOf(memberIds); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java b/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java new file mode 100644 index 0000000..424d7a3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java @@ -0,0 +1,3 @@ +package com.example.mtusi_messenger.model; + +public record IdempotentMessage(byte[] requestHash, Message message) {} diff --git a/src/main/java/com/example/mtusi_messenger/model/Message.java b/src/main/java/com/example/mtusi_messenger/model/Message.java new file mode 100644 index 0000000..2539faa --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Message.java @@ -0,0 +1,16 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record Message(UUID id, + UUID chatId, + UUID senderId, + String text, + Instant createdAt, + List attachmentIds) { + public Message { + attachmentIds = attachmentIds == null ? List.of() : List.copyOf(attachmentIds); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/MessageKey.java b/src/main/java/com/example/mtusi_messenger/model/MessageKey.java new file mode 100644 index 0000000..545fd60 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/MessageKey.java @@ -0,0 +1,35 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record MessageKey(Instant createdAt, UUID id) + implements Comparable { + + private static int compareUuidV7(UUID a, UUID b) { + int result = Long.compareUnsigned( + a.getMostSignificantBits(), + b.getMostSignificantBits() + ); + + if (result != 0) { + return result; + } + + return Long.compareUnsigned( + a.getLeastSignificantBits(), + b.getLeastSignificantBits() + ); + } + + @Override + public int compareTo(MessageKey other) { + int result = createdAt.compareTo(other.createdAt); + + if (result != 0) { + return result; + } + + return compareUuidV7(id, other.id); + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/model/Session.java b/src/main/java/com/example/mtusi_messenger/model/Session.java new file mode 100644 index 0000000..27aa2fb --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Session.java @@ -0,0 +1,15 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record Session(UUID id, UUID userId, byte[] tokenHash, Instant createdAt, Instant expiresAt) { + public Session { + tokenHash = tokenHash.clone(); + } + + @Override + public byte[] tokenHash() { + return tokenHash.clone(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/User.java b/src/main/java/com/example/mtusi_messenger/model/User.java new file mode 100644 index 0000000..1fefb17 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/User.java @@ -0,0 +1,6 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record User(UUID id, String username, String displayName, Instant createdAt) {} diff --git a/src/main/java/com/example/mtusi_messenger/observability/DurableTraceContext.java b/src/main/java/com/example/mtusi_messenger/observability/DurableTraceContext.java new file mode 100644 index 0000000..021a29e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/observability/DurableTraceContext.java @@ -0,0 +1,45 @@ +package com.example.mtusi_messenger.observability; + +import com.example.mtusi_messenger.repository.AttachmentRepository.TraceContext; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.TextMapGetter; +import io.opentelemetry.context.propagation.TextMapPropagator; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Component +public class DurableTraceContext { + private static final TextMapGetter> GETTER = new TextMapGetter<>() { + @Override + public Iterable keys(Map carrier) { + return carrier.keySet(); + } + + @Override + public String get(Map carrier, String key) { + return carrier.get(key); + } + }; + + private final TextMapPropagator propagator = GlobalOpenTelemetry.getPropagators() + .getTextMapPropagator(); + + public TraceContext capture() { + Map carrier = new HashMap<>(); + propagator.inject(Context.current(), carrier, Map::put); + return new TraceContext(carrier.get("traceparent"), carrier.get("tracestate")); + } + + public Scope restore(TraceContext traceContext) { + Map carrier = new HashMap<>(); + carrier.put("traceparent", traceContext.traceparent()); + if (traceContext.tracestate() != null && !traceContext.tracestate().isBlank()) { + carrier.put("tracestate", traceContext.tracestate()); + } + return propagator.extract(Context.root(), carrier, GETTER).makeCurrent(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/observability/JobTelemetry.java b/src/main/java/com/example/mtusi_messenger/observability/JobTelemetry.java new file mode 100644 index 0000000..8001cdc --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/observability/JobTelemetry.java @@ -0,0 +1,77 @@ +package com.example.mtusi_messenger.observability; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; + +/** Low-cardinality business metrics required by contracts/observability.md. */ +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "worker") +public class JobTelemetry { + private static final AttributeKey JOB_TYPE = AttributeKey.stringKey("job.type"); + private static final AttributeKey OUTCOME = AttributeKey.stringKey("outcome"); + private static final String ATTACHMENT_JOB = "attachment_variant"; + + private final LongCounter processed; + private final DoubleHistogram processingDuration; + private final DoubleHistogram timeToReady; + private final AtomicLong busy = new AtomicLong(); + + public JobTelemetry(@Value("${spring.rabbitmq.listener.simple.concurrency:2}") long capacity) { + Meter meter = GlobalOpenTelemetry.get().meterBuilder("messenger-jobs") + .setInstrumentationVersion("1.0.0") + .build(); + processed = meter.counterBuilder("messenger.jobs.processed") + .setDescription("Completed attachment job attempts") + .build(); + processingDuration = meter.histogramBuilder("messenger.job.processing.duration") + .setDescription("Attachment job attempt duration") + .setUnit("s") + .build(); + timeToReady = meter.histogramBuilder("messenger.attachment.time_to_ready") + .setDescription("Time from durable acceptance to ready state") + .setUnit("s") + .build(); + meter.gaugeBuilder("messenger.worker.busy") + .ofLongs() + .setDescription("Workers currently processing an attachment job") + .buildWithCallback(measurement -> measurement.record(busy.get())); + meter.gaugeBuilder("messenger.worker.capacity") + .ofLongs() + .setDescription("Configured attachment worker capacity") + .buildWithCallback(measurement -> measurement.record(capacity)); + } + + public long started() { + busy.incrementAndGet(); + return System.nanoTime(); + } + + public void finished(long startedNanos, Instant acceptedAt, String outcome) { + busy.updateAndGet(current -> Math.max(0, current - 1)); + Attributes attributes = Attributes.of(JOB_TYPE, ATTACHMENT_JOB, OUTCOME, outcome); + processed.add(1, attributes); + processingDuration.record( + Math.max(0, System.nanoTime() - startedNanos) / 1_000_000_000.0, + attributes); + if ("success".equals(outcome)) { + timeToReady.record( + Math.max(0, Duration.between(acceptedAt, Instant.now()).toMillis()) / 1000.0, + Attributes.of(JOB_TYPE, ATTACHMENT_JOB, OUTCOME, outcome)); + } + } + + public void malformedMessage() { + processed.add(1, Attributes.of(JOB_TYPE, ATTACHMENT_JOB, OUTCOME, "invalid_message")); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/observability/QueueTelemetry.java b/src/main/java/com/example/mtusi_messenger/observability/QueueTelemetry.java new file mode 100644 index 0000000..4aa2f30 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/observability/QueueTelemetry.java @@ -0,0 +1,71 @@ +package com.example.mtusi_messenger.observability; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.concurrent.atomic.AtomicLong; + +/** Samples the durable job queue without placing job or user identifiers in labels. */ +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class QueueTelemetry { + private static final Logger log = LoggerFactory.getLogger(QueueTelemetry.class); + private static final AttributeKey QUEUE_NAME = AttributeKey.stringKey("messaging.destination.name"); + + private final JdbcTemplate jdbc; + private final AtomicLong depth = new AtomicLong(); + private final AtomicLong oldestAgeSeconds = new AtomicLong(); + private final Attributes queueAttributes; + + public QueueTelemetry( + JdbcTemplate jdbc, + @Value("${messenger.jobs.queue}") String queueName) { + this.jdbc = jdbc; + this.queueAttributes = Attributes.of(QUEUE_NAME, queueName); + Meter meter = GlobalOpenTelemetry.get().meterBuilder("messenger-queue") + .setInstrumentationVersion("1.0.0") + .build(); + meter.gaugeBuilder("messenger.queue.depth") + .ofLongs() + .setDescription("Durable queued or processing attachment jobs") + .buildWithCallback(measurement -> measurement.record(depth.get(), queueAttributes)); + meter.gaugeBuilder("messenger.queue.oldest_job.age") + .ofLongs() + .setDescription("Age of the oldest non-terminal attachment job") + .setUnit("s") + .buildWithCallback(measurement -> measurement.record(oldestAgeSeconds.get(), queueAttributes)); + } + + @Scheduled(fixedDelayString = "${messenger.observability.queue-poll-interval:5s}") + public void sample() { + try { + QueueSnapshot snapshot = jdbc.queryForObject(""" + SELECT count(*) AS depth, + COALESCE(EXTRACT(EPOCH FROM + (clock_timestamp() - min(created_at))), 0)::bigint AS oldest_age + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """, (rs, row) -> new QueueSnapshot(rs.getLong("depth"), rs.getLong("oldest_age"))); + if (snapshot != null) { + depth.set(snapshot.depth()); + oldestAgeSeconds.set(Math.max(0, snapshot.oldestAgeSeconds())); + } + } catch (RuntimeException error) { + log.atWarn() + .addKeyValue("event", "queue_metrics_sample_failed") + .log("Queue telemetry sample failed: {}", error.getClass().getSimpleName()); + } + } + + private record QueueSnapshot(long depth, long oldestAgeSeconds) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java b/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java new file mode 100644 index 0000000..ce87b42 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/AttachmentJobPublisher.java @@ -0,0 +1,65 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.model.AttachmentJob; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; + +@Component +public class AttachmentJobPublisher { + private final RabbitTemplate rabbit; + private final ObjectMapper mapper; + private final String exchange; + private final String routingKey; + private final long confirmTimeoutMillis; + + public AttachmentJobPublisher( + RabbitTemplate rabbit, + ObjectMapper mapper, + @Value("${messenger.jobs.exchange}") String exchange, + @Value("${messenger.jobs.routing-key}") String routingKey, + @Value("${messenger.jobs.publish-confirm-timeout:5s}") Duration confirmTimeout) { + this.rabbit = rabbit; + this.mapper = mapper; + this.exchange = exchange; + this.routingKey = routingKey; + this.confirmTimeoutMillis = confirmTimeout.toMillis(); + this.rabbit.setMandatory(true); + } + + public void publish(AttachmentJob job) { + try { + byte[] body = mapper.writeValueAsBytes(job); + MessageProperties properties = new MessageProperties(); + properties.setContentType(MessageProperties.CONTENT_TYPE_JSON); + properties.setContentEncoding("UTF-8"); + properties.setDeliveryMode(MessageDeliveryMode.PERSISTENT); + properties.setMessageId(job.jobId().toString()); + properties.setHeader("schema_version", job.schemaVersion()); + Message message = new Message(body, properties); + + rabbit.invoke(operations -> { + operations.send(exchange, routingKey, message); + operations.waitForConfirmsOrDie(confirmTimeoutMillis); + return null; + }); + } catch (JsonProcessingException error) { + throw new JobPublishException("Cannot encode attachment job", error); + } catch (RuntimeException error) { + throw new JobPublishException("RabbitMQ did not confirm attachment job", error); + } + } + + public static final class JobPublishException extends RuntimeException { + public JobPublishException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java new file mode 100644 index 0000000..4ddeb98 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/ExpiredLeaseReaper.java @@ -0,0 +1,57 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.Clock; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class ExpiredLeaseReaper { + private static final Logger log = LoggerFactory.getLogger(ExpiredLeaseReaper.class); + private final AttachmentRepository attachments; + private final RetryPolicy retryPolicy; + private final TransactionTemplate transaction; + private final int maxAttempts; + private final Clock clock = Clock.systemUTC(); + + public ExpiredLeaseReaper( + AttachmentRepository attachments, + RetryPolicy retryPolicy, + TransactionTemplate transaction, + @Value("${messenger.jobs.max-attempts:5}") int maxAttempts) { + this.attachments = attachments; + this.retryPolicy = retryPolicy; + this.transaction = transaction; + this.maxAttempts = maxAttempts; + } + + @Scheduled(fixedDelayString = "${messenger.jobs.lease-reaper-interval:5s}") + public void recoverExpired() { + for (var job : attachments.findExpiredJobs(50)) { + try { + transaction.executeWithoutResult(status -> { + if (job.attempt() >= maxAttempts) { + attachments.failExpired(job.jobId(), job.attachmentId(), + job.pipelineVersion(), "retry_exhausted"); + } else { + int nextAttempt = job.attempt() + 1; + attachments.requeueExpired( + job.jobId(), job.attachmentId(), job.pipelineVersion(), + nextAttempt, "processing_timeout", + clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))); + } + }); + } catch (RuntimeException error) { + log.warn("Expired attachment lease recovery postponed for job {}", job.jobId()); + return; + } + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java b/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java new file mode 100644 index 0000000..306375e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/RetryPolicy.java @@ -0,0 +1,39 @@ +package com.example.mtusi_messenger.queue; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.concurrent.ThreadLocalRandom; + +@Component +public class RetryPolicy { + private final long baseDelayMillis; + private final long maxDelayMillis; + + public RetryPolicy( + @Value("${messenger.jobs.retry-base-delay:1s}") Duration baseDelay, + @Value("${messenger.jobs.retry-max-delay:30s}") Duration maxDelay) { + if (baseDelay.isNegative() || baseDelay.isZero() || maxDelay.compareTo(baseDelay) < 0) { + throw new IllegalArgumentException("Invalid retry delay configuration"); + } + this.baseDelayMillis = baseDelay.toMillis(); + this.maxDelayMillis = maxDelay.toMillis(); + } + + public Duration delayForAttempt(int nextAttempt) { + int exponent = Math.max(0, Math.min(30, nextAttempt - 2)); + long exponential; + try { + exponential = Math.multiplyExact(baseDelayMillis, 1L << exponent); + } catch (ArithmeticException overflow) { + exponential = maxDelayMillis; + } + long ceiling = Math.min(maxDelayMillis, exponential); + long floor = Math.max(1, ceiling / 2); + long jittered = floor == ceiling + ? ceiling + : ThreadLocalRandom.current().nextLong(floor, ceiling + 1); + return Duration.ofMillis(jittered); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java b/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java new file mode 100644 index 0000000..7b9793b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/queue/UnpublishedJobRepublisher.java @@ -0,0 +1,47 @@ +package com.example.mtusi_messenger.queue; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.observability.DurableTraceContext; +import io.opentelemetry.context.Scope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class UnpublishedJobRepublisher { + private static final Logger log = LoggerFactory.getLogger(UnpublishedJobRepublisher.class); + private final AttachmentRepository attachments; + private final AttachmentJobPublisher publisher; + private final DurableTraceContext tracing; + + public UnpublishedJobRepublisher(AttachmentRepository attachments, + AttachmentJobPublisher publisher, + DurableTraceContext tracing) { + this.attachments = attachments; + this.publisher = publisher; + this.tracing = tracing; + } + + @Scheduled(fixedDelayString = "${messenger.jobs.republish-interval:5s}") + public void publishPending() { + for (var job : attachments.findUnpublishedJobs(50)) { + try { + var traceContext = attachments.findTraceContext(job.jobId()); + if (traceContext.isPresent()) { + try (Scope ignored = tracing.restore(traceContext.get())) { + publisher.publish(job); + } + } else { + publisher.publish(job); + } + attachments.markPublished(job.jobId()); + } catch (RuntimeException error) { + log.warn("Attachment job publication postponed for job {}", job.jobId()); + return; + } + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java new file mode 100644 index 0000000..c2595f3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java @@ -0,0 +1,45 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.Set; +import java.time.Duration; +import java.time.Instant; + +public interface AttachmentRepository { + Attachment saveQueued(Attachment attachment, AttachmentJob job); + Optional findById(UUID id); + List findAllByIdsInChat(UUID chatId, List ids); + Set findAllObjectKeys(); + Set findActiveStagingKeys(); + void markPublished(UUID jobId); + boolean markPublishFaultInjected(UUID jobId); + List findUnpublishedJobs(int limit); + Optional findTraceContext(UUID jobId); + int currentAttempt(UUID jobId); + boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, + int attempt, UUID leaseToken, Duration leaseDuration); + boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken); + boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken); + boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, UUID leaseToken, + int nextAttempt, String errorCode, Instant nextAttemptAt); + CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion, + List variants, boolean injectCrashAfterCommit); + boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, String errorCode); + List findExpiredJobs(int limit); + boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, int nextAttempt, + String errorCode, Instant nextAttemptAt); + boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, String errorCode); + + record CompletionResult(boolean completed, boolean crashAfterCommit) { + } + + record TraceContext(String traceparent, String tracestate) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java b/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java new file mode 100644 index 0000000..66085e4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Chat; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface ChatRepository { + Chat save(Chat chat); + Optional findById(UUID id); + boolean isMember(UUID chatId, UUID userId); + List findAllForUser(UUID userId); + List findPageForUser(UUID userId, UUID afterId, java.time.Instant afterCreatedAt, int limitPlusOne); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java b/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java new file mode 100644 index 0000000..23ea481 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Message; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface MessageRepository { + Message save(Message message); + void lockIdempotencyScope(UUID userId, String method, String path, String key); + void deleteExpiredIdempotency(UUID userId, String method, String path, String key, Instant expiresBefore); + Optional findIdempotent( + UUID userId, String method, String path, String key); + void saveIdempotency(UUID userId, String method, String path, String key, + byte[] requestHash, UUID messageId, Instant createdAt); + List findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java new file mode 100644 index 0000000..d97f66d --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java @@ -0,0 +1,402 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; +import com.example.mtusi_messenger.observability.DurableTraceContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +@Repository +public class PostgresAttachmentRepository implements AttachmentRepository { + private static final String COLUMNS = """ + id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, staging_key, status, error_code, created_at, updated_at + """; + private final JdbcTemplate jdbc; + private final DurableTraceContext tracing; + + public PostgresAttachmentRepository(JdbcTemplate jdbc, DurableTraceContext tracing) { + this.jdbc = jdbc; + this.tracing = tracing; + } + + @Override + public Attachment saveQueued(Attachment attachment, AttachmentJob job) { + jdbc.update(""" + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, staging_key, pipeline_version, + status, error_code, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?) + """, attachment.id(), attachment.chatId(), attachment.ownerId(), + attachment.filename(), attachment.contentType(), attachment.sizeBytes(), + attachment.sha256(), attachment.objectKey(), attachment.stagingKey(), + job.pipelineVersion(), + Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt())); + TraceContext traceContext = tracing.capture(); + jdbc.update(""" + INSERT INTO attachment_jobs + (id, attachment_id, pipeline_version, attempt, status, published, + trace_parent, trace_state, created_at, updated_at) + VALUES (?, ?, ?, ?, 'queued', false, ?, ?, ?, ?) + """, job.jobId(), job.attachmentId(), job.pipelineVersion(), job.attempt(), + traceContext.traceparent(), traceContext.tracestate(), + Timestamp.from(job.createdAt()), Timestamp.from(job.createdAt())); + return attachment; + } + + @Override + public Optional findById(UUID id) { + return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE id = ?", + (rs, rowNum) -> map(rs), id).stream().findFirst(); + } + + @Override + public List findAllByIdsInChat(UUID chatId, List ids) { + if (ids.isEmpty()) { + return List.of(); + } + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + Object[] args = new Object[ids.size() + 1]; + args[0] = chatId; + for (int i = 0; i < ids.size(); i++) { + args[i + 1] = ids.get(i); + } + return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE chat_id = ? AND id IN (" + + placeholders + ") AND status <> 'failed'", + (rs, rowNum) -> map(rs), args); + } + + @Override + public Set findAllObjectKeys() { + Set keys = new HashSet<>(jdbc.query( + "SELECT object_key FROM attachments WHERE status = 'ready'", + (rs, rowNum) -> rs.getString("object_key"))); + keys.addAll(jdbc.query("SELECT object_key FROM attachment_variants", + (rs, rowNum) -> rs.getString("object_key"))); + return keys; + } + + @Override + public Set findActiveStagingKeys() { + return new HashSet<>(jdbc.query(""" + SELECT staging_key + FROM attachments + WHERE status IN ('queued', 'processing') AND staging_key IS NOT NULL + """, (rs, rowNum) -> rs.getString("staging_key"))); + } + + @Override + public void markPublished(UUID jobId) { + jdbc.update(""" + UPDATE attachment_jobs + SET published = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'queued' + """, jobId); + } + + @Override + public boolean markPublishFaultInjected(UUID jobId) { + return jdbc.update(""" + UPDATE attachment_jobs + SET publish_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND publish_fault_injected = false AND status = 'queued' + """, jobId) == 1; + } + + @Override + public List findUnpublishedJobs(int limit) { + return jdbc.query(""" + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE status = 'queued' AND published = false + AND next_attempt_at <= clock_timestamp() + ORDER BY next_attempt_at, created_at, id + LIMIT ? + """, (rs, rowNum) -> mapJob(rs), limit); + } + + @Override + public Optional findTraceContext(UUID jobId) { + return jdbc.query(""" + SELECT trace_parent, trace_state + FROM attachment_jobs + WHERE id = ? AND trace_parent IS NOT NULL + """, (rs, rowNum) -> new TraceContext( + rs.getString("trace_parent"), rs.getString("trace_state")), jobId) + .stream().findFirst(); + } + + @Override + public int currentAttempt(UUID jobId) { + Integer attempt = jdbc.query("SELECT attempt FROM attachment_jobs WHERE id = ?", + (rs, rowNum) -> rs.getInt("attempt"), jobId).stream().findFirst().orElse(null); + return attempt == null ? 1 : attempt; + } + + @Override + public boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, int attempt, + UUID leaseToken, Duration leaseDuration) { + int claimed = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'processing', attempt = GREATEST(attempt, ?), + lease_token = ?, + lease_until = clock_timestamp() + (? * interval '1 millisecond'), + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? + AND pipeline_version = ? + AND (status = 'queued' + OR (status = 'processing' AND lease_until < clock_timestamp())) + AND EXISTS ( + SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ? + AND a.status IN ('queued', 'processing')) + """, attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId, + pipelineVersion, attachmentId, pipelineVersion); + if (claimed == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? + AND status IN ('queued', 'processing') + """, attachmentId, pipelineVersion); + return true; + } + + @Override + public boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken) { + return jdbc.update(""" + UPDATE attachment_jobs + SET before_effect_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'processing' AND lease_token = ? + AND before_effect_fault_injected = false + """, jobId, leaseToken) == 1; + } + + @Override + public boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken) { + return jdbc.update(""" + UPDATE attachment_jobs + SET after_effect_fault_injected = true, updated_at = clock_timestamp() + WHERE id = ? AND status = 'processing' AND lease_token = ? + AND after_effect_fault_injected = false + """, jobId, leaseToken) == 1; + } + + @Override + public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, + int nextAttempt, String errorCode, Instant nextAttemptAt) { + TraceContext traceContext = tracing.capture(); + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'queued', attempt = ?, published = false, + last_error_code = ?, next_attempt_at = ?, + trace_parent = ?, trace_state = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND pipeline_version = ? AND lease_token = ? + """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), + traceContext.traceparent(), traceContext.tracestate(), + jobId, attachmentId, pipelineVersion, leaseToken); + if (updated == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? + AND status IN ('queued', 'processing') + """, attachmentId, pipelineVersion); + return true; + } + + @Override + public CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion, + List variants, boolean injectCrashAfterCommit) { + boolean triggerCrash = injectCrashAfterCommit && Boolean.TRUE.equals(jdbc.query(""" + SELECT NOT result_fault_injected + FROM attachment_jobs + WHERE id = ? AND attachment_id = ? AND status = 'processing' AND lease_token = ? + FOR UPDATE + """, (rs, rowNum) -> rs.getBoolean(1), jobId, attachmentId, leaseToken) + .stream().findFirst().orElse(false)); + int completed = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'ready', published = true, last_error_code = NULL, + lease_token = NULL, lease_until = NULL, + result_fault_injected = result_fault_injected OR ?, + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND pipeline_version = ? AND lease_token = ? + AND EXISTS ( + SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ? + AND a.status = 'processing') + """, triggerCrash, jobId, attachmentId, pipelineVersion, leaseToken, + attachmentId, pipelineVersion); + if (completed == 0) { + return new CompletionResult(false, false); + } + for (AttachmentVariant variant : variants) { + jdbc.update(""" + INSERT INTO attachment_variants + (attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp()) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, attachmentId, pipelineVersion, variant.name(), variant.contentType(), + variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(), + variant.objectKey()); + } + jdbc.update(""" + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? AND status = 'processing' + """, attachmentId, pipelineVersion); + return new CompletionResult(true, triggerCrash); + } + + @Override + public boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion, + UUID leaseToken, String errorCode) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'failed', published = true, last_error_code = ?, + lease_token = NULL, lease_until = NULL, + updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND pipeline_version = ? AND lease_token = ? + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, errorCode, jobId, attachmentId, pipelineVersion, leaseToken, + attachmentId, pipelineVersion); + if (updated == 0) { + return false; + } + jdbc.update(""" + UPDATE attachments + SET status = 'failed', error_code = ?, updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? AND status <> 'ready' + """, errorCode, attachmentId, pipelineVersion); + return true; + } + + @Override + public List findExpiredJobs(int limit) { + return jdbc.query(""" + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE status = 'processing' AND lease_until < clock_timestamp() + ORDER BY lease_until, id + LIMIT ? + """, (rs, rowNum) -> mapJob(rs), limit); + } + + @Override + public boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, + int nextAttempt, + String errorCode, Instant nextAttemptAt) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'queued', attempt = ?, published = false, + last_error_code = ?, next_attempt_at = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND pipeline_version = ? AND lease_until < clock_timestamp() + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId, + pipelineVersion, attachmentId, pipelineVersion); + if (updated == 1) { + jdbc.update(""" + UPDATE attachments SET status = 'queued', error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? AND status = 'processing' + """, attachmentId, pipelineVersion); + } + return updated == 1; + } + + @Override + public boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, + String errorCode) { + int updated = jdbc.update(""" + UPDATE attachment_jobs + SET status = 'failed', published = true, last_error_code = ?, + lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() + WHERE id = ? AND attachment_id = ? AND status = 'processing' + AND pipeline_version = ? AND lease_until < clock_timestamp() + AND EXISTS (SELECT 1 FROM attachments a + WHERE a.id = ? AND a.pipeline_version = ?) + """, errorCode, jobId, attachmentId, pipelineVersion, + attachmentId, pipelineVersion); + if (updated == 1) { + jdbc.update(""" + UPDATE attachments SET status = 'failed', error_code = ?, + updated_at = clock_timestamp() + WHERE id = ? AND pipeline_version = ? AND status <> 'ready' + """, errorCode, attachmentId, pipelineVersion); + } + return updated == 1; + } + + private Attachment map(ResultSet rs) throws SQLException { + UUID id = rs.getObject("id", UUID.class); + return new Attachment( + id, + rs.getObject("chat_id", UUID.class), + rs.getObject("owner_id", UUID.class), + rs.getString("filename"), + rs.getString("content_type"), + rs.getLong("size_bytes"), + rs.getString("sha256"), + rs.getString("object_key"), + rs.getString("staging_key"), + rs.getString("status"), + findVariants(id), + rs.getString("error_code"), + rs.getTimestamp("created_at").toInstant(), + rs.getTimestamp("updated_at").toInstant()); + } + + private List findVariants(UUID attachmentId) { + return jdbc.query(""" + SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = ? + AND pipeline_version = ( + SELECT pipeline_version FROM attachments WHERE id = ?) + ORDER BY name + """, (rs, rowNum) -> new AttachmentVariant( + rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"), + rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"), + rs.getString("object_key")), attachmentId, attachmentId); + } + + private static AttachmentJob mapJob(ResultSet rs) throws SQLException { + return new AttachmentJob( + AttachmentJob.SCHEMA_VERSION, + rs.getObject("id", UUID.class), + AttachmentJob.TYPE, + rs.getObject("attachment_id", UUID.class), + rs.getInt("pipeline_version"), + rs.getInt("attempt"), + rs.getTimestamp("created_at").toInstant()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java new file mode 100644 index 0000000..17d825d --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java @@ -0,0 +1,106 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Chat; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresChatRepository implements ChatRepository { + private final JdbcTemplate jdbc; + + public PostgresChatRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @Transactional(timeout = 5) + public Chat save(Chat chat) { + jdbc.update(""" + INSERT INTO chats (id, title, created_at) + VALUES (?, ?, ?) + """, chat.id(), chat.title(), Timestamp.from(chat.createdAt())); + + for (UUID memberId : chat.memberIds()) { + jdbc.update(""" + INSERT INTO chat_members (chat_id, user_id) + VALUES (?, ?) + """, chat.id(), memberId); + } + return chat; + } + + @Override + public Optional findById(UUID id) { + return jdbc.query(""" + SELECT id, title, created_at + FROM chats + WHERE id = ? + """, (rs, rowNum) -> new ChatRow( + rs.getObject("id", UUID.class), + rs.getString("title"), + rs.getTimestamp("created_at").toInstant()), id) + .stream().findFirst() + .map(this::toChat); + } + + @Override + public boolean isMember(UUID chatId, UUID userId) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = ? AND user_id = ? + ) + """, Boolean.class, chatId, userId)); + } + + @Override + public List findAllForUser(UUID userId) { + return findPageForUser(userId, null, null, Integer.MAX_VALUE); + } + + @Override + public List findPageForUser(UUID userId, UUID afterId, Instant afterCreatedAt, int limitPlusOne) { + StringBuilder sql = new StringBuilder(""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = ? + """); + List parameters = new ArrayList<>(); + parameters.add(userId); + + if (afterCreatedAt != null) { + sql.append(" AND (c.created_at, c.id) > (?, ?) "); + parameters.add(Timestamp.from(afterCreatedAt)); + parameters.add(afterId); + } + sql.append(" ORDER BY c.created_at, c.id LIMIT ? "); + parameters.add(limitPlusOne); + + return jdbc.query(sql.toString(), (rs, rowNum) -> new ChatRow( + rs.getObject("id", UUID.class), + rs.getString("title"), + rs.getTimestamp("created_at").toInstant()), parameters.toArray()) + .stream().map(this::toChat).toList(); + } + + private Chat toChat(ChatRow row) { + List members = jdbc.queryForList(""" + SELECT user_id + FROM chat_members + WHERE chat_id = ? + ORDER BY joined_at, user_id + """, UUID.class, row.id()); + return new Chat(row.id(), row.title(), members, row.createdAt()); + } + + private record ChatRow(UUID id, String title, Instant createdAt) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java new file mode 100644 index 0000000..f9c6748 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java @@ -0,0 +1,172 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.model.IdempotentMessage; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +@Repository +public class PostgresMessageRepository implements MessageRepository { + private final JdbcTemplate jdbc; + + public PostgresMessageRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Message save(Message message) { + jdbc.update(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (?, ?, ?, ?, ?) + """, message.id(), message.chatId(), message.senderId(), message.text(), + Timestamp.from(message.createdAt())); + if (!message.attachmentIds().isEmpty()) { + jdbc.batchUpdate(""" + INSERT INTO message_attachments + (message_id, chat_id, attachment_id, position) + VALUES (?, ?, ?, ?) + """, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement statement, int index) throws SQLException { + statement.setObject(1, message.id()); + statement.setObject(2, message.chatId()); + statement.setObject(3, message.attachmentIds().get(index)); + statement.setInt(4, index); + } + + @Override + public int getBatchSize() { + return message.attachmentIds().size(); + } + }); + } + return message; + } + + @Override + public void lockIdempotencyScope(UUID userId, String method, String path, String key) { + String scope = userId + "\n" + method + "\n" + path + "\n" + key; + jdbc.query( + "SELECT pg_advisory_xact_lock(hashtextextended(?, 0))", + rs -> { + rs.next(); + return null; + }, + scope); + } + + @Override + public void deleteExpiredIdempotency( + UUID userId, + String method, + String path, + String key, + Instant expiresBefore + ) { + jdbc.update(""" + DELETE FROM message_idempotency + WHERE user_id = ? + AND method = ? + AND request_path = ? + AND idempotency_key = ? + AND created_at < ? + """, userId, method, path, key, Timestamp.from(expiresBefore)); + } + + @Override + public Optional findIdempotent( + UUID userId, + String method, + String path, + String key + ) { + Optional result = jdbc.query(""" + SELECT i.request_hash, + m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_idempotency i + JOIN messages m ON m.id = i.message_id + WHERE i.user_id = ? + AND i.method = ? + AND i.request_path = ? + AND i.idempotency_key = ? + """, (rs, rowNum) -> new IdempotentMessage( + rs.getBytes("request_hash"), + new Message( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("sender_id", UUID.class), + rs.getString("text"), + rs.getTimestamp("created_at").toInstant(), + List.of())), + userId, method, path, key).stream().findFirst(); + return result.map(value -> new IdempotentMessage( + value.requestHash(), withAttachments(value.message()))); + } + + @Override + public void saveIdempotency( + UUID userId, + String method, + String path, + String key, + byte[] requestHash, + UUID messageId, + Instant createdAt + ) { + jdbc.update(""" + INSERT INTO message_idempotency + (user_id, method, request_path, idempotency_key, + request_hash, message_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, userId, method, path, key, requestHash, messageId, + Timestamp.from(createdAt)); + } + + @Override + public List findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne) { + StringBuilder sql = new StringBuilder(""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = ? + """); + List parameters = new ArrayList<>(); + parameters.add(chatId); + + if (afterCreatedAt != null) { + sql.append(" AND (created_at, id) > (?, ?) "); + parameters.add(Timestamp.from(afterCreatedAt)); + parameters.add(afterId); + } + sql.append(" ORDER BY created_at, id LIMIT ? "); + parameters.add(limitPlusOne); + + return jdbc.query(sql.toString(), (rs, rowNum) -> new Message( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("sender_id", UUID.class), + rs.getString("text"), + rs.getTimestamp("created_at").toInstant(), + List.of()), parameters.toArray()).stream().map(this::withAttachments).toList(); + } + + private Message withAttachments(Message message) { + List attachmentIds = jdbc.query(""" + SELECT attachment_id + FROM message_attachments + WHERE message_id = ? + ORDER BY position + """, (rs, rowNum) -> rs.getObject("attachment_id", UUID.class), message.id()); + return new Message(message.id(), message.chatId(), message.senderId(), message.text(), + message.createdAt(), attachmentIds); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java new file mode 100644 index 0000000..d0b210b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java @@ -0,0 +1,51 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Session; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresSessionRepository implements SessionRepository { + private final JdbcTemplate jdbc; + + public PostgresSessionRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Session save(Session session) { + jdbc.update(""" + INSERT INTO sessions (id, user_id, token_hash, created_at, expires_at) + VALUES (?, ?, ?, ?, ?) + """, session.id(), session.userId(), session.tokenHash(), + Timestamp.from(session.createdAt()), Timestamp.from(session.expiresAt())); + return session; + } + + @Override + public Optional findActiveUserId(byte[] tokenHash) { + return jdbc.query(""" + SELECT user_id + FROM sessions + WHERE token_hash = ? + AND revoked_at IS NULL + AND expires_at > CURRENT_TIMESTAMP + """, (rs, rowNum) -> rs.getObject("user_id", UUID.class), tokenHash) + .stream().findFirst(); + } + + @Override + public boolean revoke(byte[] tokenHash) { + return jdbc.update(""" + UPDATE sessions + SET revoked_at = CURRENT_TIMESTAMP + WHERE token_hash = ? + AND revoked_at IS NULL + AND expires_at > CURRENT_TIMESTAMP + """, tokenHash) == 1; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java new file mode 100644 index 0000000..e096b48 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java @@ -0,0 +1,72 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.User; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresUserRepository implements UserRepository { + private final JdbcTemplate jdbc; + + public PostgresUserRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public User save(User user, String passwordHash) { + jdbc.update(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (?, ?, ?, ?, ?) + """, user.id(), user.username(), user.displayName(), passwordHash, + Timestamp.from(user.createdAt())); + return user; + } + + @Override + public Optional findById(UUID id) { + return jdbc.query(""" + SELECT id, username, display_name, created_at + FROM users + WHERE id = ? + """, (rs, rowNum) -> new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), id) + .stream().findFirst(); + } + + @Override + public Optional findByUsername(String username) { + return jdbc.query(""" + SELECT id, username, display_name, created_at + FROM users + WHERE username = ? + """, (rs, rowNum) -> new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), username) + .stream().findFirst(); + } + + @Override + public Optional findCredentialsByUsername(String username) { + return jdbc.query(""" + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = ? + """, (rs, rowNum) -> new UserCredentials( + new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), + rs.getString("password_hash")), username) + .stream().findFirst(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java b/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java new file mode 100644 index 0000000..801c9cf --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java @@ -0,0 +1,12 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Session; + +import java.util.Optional; +import java.util.UUID; + +public interface SessionRepository { + Session save(Session session); + Optional findActiveUserId(byte[] tokenHash); + boolean revoke(byte[] tokenHash); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java b/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java new file mode 100644 index 0000000..e486196 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.User; +import java.util.Optional; +import java.util.UUID; + +public interface UserRepository { + User save(User user, String passwordHash); + Optional findById(UUID id); + Optional findByUsername(String username); + Optional findCredentialsByUsername(String username); + + record UserCredentials(User user, String passwordHash) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java b/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java new file mode 100644 index 0000000..213d8a3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java @@ -0,0 +1,130 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.service.AuthenticationService; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Enumeration; +import java.util.List; + +public class ApiAuthenticationFilter extends OncePerRequestFilter { + private static final String AUTHORIZATION = "Authorization"; + private static final String SESSION_PATH = "/api/v1/auth/sessions"; + + private final AuthenticationManager authenticationManager; + private final AuthenticationService sessions; + private final RestAuthenticationEntryPoint basicEntryPoint; + private final RestAuthenticationEntryPoint bearerEntryPoint; + + public ApiAuthenticationFilter( + AuthenticationManager authenticationManager, + AuthenticationService sessions, + RestAuthenticationEntryPoint basicEntryPoint, + RestAuthenticationEntryPoint bearerEntryPoint + ) { + this.authenticationManager = authenticationManager; + this.sessions = sessions; + this.basicEntryPoint = basicEntryPoint; + this.bearerEntryPoint = bearerEntryPoint; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + boolean basicLogin = HttpMethod.POST.matches(request.getMethod()) + && SESSION_PATH.equals(request.getRequestURI()); + if (isPublicRequest(request) && !basicLogin) { + filterChain.doFilter(request, response); + return; + } + + try { + Authentication authentication = basicLogin + ? authenticateBasic(request) + : authenticateBearer(request); + var context = SecurityContextHolder.createEmptyContext(); + context.setAuthentication(authentication); + SecurityContextHolder.setContext(context); + filterChain.doFilter(request, response); + } catch (AuthenticationException e) { + SecurityContextHolder.clearContext(); + (basicLogin ? basicEntryPoint : bearerEntryPoint) + .commence(request, response, e); + } finally { + SecurityContextHolder.clearContext(); + } + } + + private Authentication authenticateBasic(HttpServletRequest request) { + String header = singleAuthorizationHeader(request); + if (!header.startsWith("Basic ") || header.length() == 6) { + throw new BadCredentialsException("Invalid Basic credentials"); + } + try { + byte[] decoded = Base64.getDecoder().decode(header.substring(6)); + String credentials = new String(decoded, StandardCharsets.UTF_8); + int separator = credentials.indexOf(':'); + if (separator <= 0) { + throw new BadCredentialsException("Invalid Basic credentials"); + } + return authenticationManager.authenticate( + UsernamePasswordAuthenticationToken.unauthenticated( + credentials.substring(0, separator), + credentials.substring(separator + 1))); + } catch (IllegalArgumentException e) { + throw new BadCredentialsException("Invalid Basic credentials", e); + } + } + + private Authentication authenticateBearer(HttpServletRequest request) { + String header = singleAuthorizationHeader(request); + if (!header.startsWith("Bearer ") || header.length() == 7) { + throw new BadCredentialsException("Invalid Bearer token"); + } + String token = header.substring(7); + if (!token.matches("[A-Za-z0-9_-]{43}")) { + throw new BadCredentialsException("Invalid Bearer token"); + } + SessionPrincipal principal = sessions.authenticateBearer(token); + return UsernamePasswordAuthenticationToken.authenticated(principal, null, List.of()); + } + + private String singleAuthorizationHeader(HttpServletRequest request) { + Enumeration values = request.getHeaders(AUTHORIZATION); + if (values == null || !values.hasMoreElements()) { + throw new BadCredentialsException("Authentication required"); + } + String value = values.nextElement(); + if (values.hasMoreElements()) { + throw new BadCredentialsException("Multiple Authorization headers"); + } + return value; + } + + private boolean isPublicRequest(HttpServletRequest request) { + String path = request.getRequestURI(); + return path.startsWith("/health/") + || path.equals("/swagger-ui.html") + || path.startsWith("/swagger-ui/") + || path.equals("/v3/api-docs") + || path.startsWith("/v3/api-docs/") + || (HttpMethod.POST.matches(request.getMethod()) + && "/api/v1/users".equals(path)); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java b/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java new file mode 100644 index 0000000..0a0c04d --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java @@ -0,0 +1,63 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.repository.UserRepository; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Component; +import org.springframework.dao.DataAccessException; + +import java.util.List; + +@Component +public class DatabaseAuthenticationProvider implements AuthenticationProvider { + private final UserRepository users; + private final PasswordEncoder passwordEncoder; + private final String dummyPasswordHash; + + public DatabaseAuthenticationProvider(UserRepository users, PasswordEncoder passwordEncoder) { + this.users = users; + this.passwordEncoder = passwordEncoder; + this.dummyPasswordHash = passwordEncoder.encode("dummy-password-used-only-for-timing"); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + String username = authentication.getName(); + String password = String.valueOf(authentication.getCredentials()); + final java.util.Optional stored; + try { + stored = users.findCredentialsByUsername(username); + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + String passwordHash = stored.map(UserRepository.UserCredentials::passwordHash) + .filter(hash -> hash.startsWith("$argon2")) + .orElse(dummyPasswordHash); + + boolean matches; + try { + matches = passwordEncoder.matches(password, passwordHash); + } catch (IllegalArgumentException e) { + matches = false; + } + + if (stored.isEmpty() || !matches) { + throw new BadCredentialsException("Invalid credentials"); + } + + return UsernamePasswordAuthenticationToken.authenticated( + stored.get().user(), + null, + List.of()); + } + + @Override + public boolean supports(Class authentication) { + return UsernamePasswordAuthenticationToken.class.isAssignableFrom(authentication); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java b/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java new file mode 100644 index 0000000..4442f35 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java @@ -0,0 +1,46 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.web.RequestIdFilter; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.web.AuthenticationEntryPoint; + +import java.io.IOException; + +public class RestAuthenticationEntryPoint implements AuthenticationEntryPoint { + private final ObjectMapper mapper; + private final String challenge; + + public RestAuthenticationEntryPoint(ObjectMapper mapper, String challenge) { + this.mapper = mapper; + this.challenge = challenge; + } + + @Override + public void commence( + HttpServletRequest request, + HttpServletResponse response, + AuthenticationException exception + ) throws IOException { + String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE); + boolean unavailable = exception instanceof AuthenticationServiceException; + response.setStatus(unavailable + ? HttpServletResponse.SC_SERVICE_UNAVAILABLE + : HttpServletResponse.SC_UNAUTHORIZED); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + if (!unavailable) { + response.setHeader("WWW-Authenticate", challenge); + } + mapper.writeValue(response.getOutputStream(), new ErrorResponse( + new ErrorResponse.ErrorBody( + unavailable ? "unavailable" : "unauthorized", + unavailable ? "Authentication service temporarily unavailable" : "Authentication required", + requestId))); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java b/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java new file mode 100644 index 0000000..03c3951 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.model.User; + +public record SessionPrincipal(User user, byte[] tokenHash) { + public SessionPrincipal { + tokenHash = tokenHash.clone(); + } + + @Override + public byte[] tokenHash() { + return tokenHash.clone(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java new file mode 100644 index 0000000..415abb0 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java @@ -0,0 +1,106 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class AttachmentReconciler { + private static final Logger log = LoggerFactory.getLogger(AttachmentReconciler.class); + private static final String PREFIX = "attachments/"; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final Duration gracePeriod; + private final boolean dryRun; + private final Clock clock; + + @Autowired + public AttachmentReconciler( + AttachmentRepository attachments, + ObjectStorage storage, + @Value("${messenger.attachments.cleanup-grace-seconds:3600}") long graceSeconds, + @Value("${messenger.attachments.cleanup-dry-run:false}") boolean dryRun + ) { + this(attachments, storage, Duration.ofSeconds(graceSeconds), dryRun, Clock.systemUTC()); + } + + AttachmentReconciler(AttachmentRepository attachments, + ObjectStorage storage, + Duration gracePeriod, + boolean dryRun, + Clock clock) { + if (gracePeriod.isNegative() || gracePeriod.isZero()) { + throw new IllegalArgumentException("Attachment cleanup grace period must be positive"); + } + this.attachments = attachments; + this.storage = storage; + this.gracePeriod = gracePeriod; + this.dryRun = dryRun; + this.clock = clock; + } + + @Scheduled( + fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}", + initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}") + public void scheduledReconcile() { + try { + Report report = reconcile(); + if (report.orphanObjects() > 0 || report.missingObjects() > 0) { + log.warn("Attachment reconciliation: {} orphan, {} deleted, {} missing, dryRun={}", + report.orphanObjects(), report.deletedObjects(), report.missingObjects(), dryRun); + } + } catch (RuntimeException error) { + log.warn("Attachment reconciliation postponed: {}", error.getMessage()); + } + } + + public Report reconcile() { + Set databaseKeys = attachments.findAllObjectKeys(); + var storedObjects = storage.list(PREFIX); + Set storageKeys = new HashSet<>(); + Instant deleteBefore = clock.instant().minus(gracePeriod); + int orphans = 0; + int deleted = 0; + + for (ObjectStorage.StoredObject object : storedObjects) { + storageKeys.add(object.objectKey()); + if (!databaseKeys.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) { + orphans++; + if (!dryRun) { + storage.delete(object.objectKey()); + deleted++; + } + } + } + + int missing = 0; + for (String databaseKey : databaseKeys) { + if (!storageKeys.contains(databaseKey)) { + missing++; + log.error("Attachment metadata references missing object {}", databaseKey); + } + } + return new Report(databaseKeys.size(), storedObjects.size(), orphans, deleted, missing, dryRun); + } + + public record Report(int metadataObjects, + int storageObjects, + int orphanObjects, + int deletedObjects, + int missingObjects, + boolean dryRun) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java new file mode 100644 index 0000000..cff2617 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java @@ -0,0 +1,295 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; +import com.example.mtusi_messenger.queue.AttachmentJobPublisher; +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import com.example.mtusi_messenger.storage.ObjectStorageException; +import com.github.f4b6a3.uuid.UuidCreator; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HexFormat; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +@Service +public class AttachmentService { + private static final List ALLOWED_TYPES = List.of("image/png", "image/jpeg"); + private final AttachmentRepository attachments; + private final ChatService chats; + private final ObjectStorage storage; + private final AttachmentJobPublisher publisher; + private final TransactionTemplate transaction; + private final long maxUploadBytes; + private final long maxImagePixels; + private final Semaphore uploadPermits; + private final long uploadPermitWaitMillis; + private final boolean injectCrashAfterOutboxCommit; + private final int pipelineVersion; + + public AttachmentService( + AttachmentRepository attachments, + ChatService chats, + ObjectStorage storage, + AttachmentJobPublisher publisher, + TransactionTemplate transaction, + @Value("${messenger.attachments.max-upload-bytes:10485760}") long maxUploadBytes, + @Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels, + @Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads, + @Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis, + @Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit, + @Value("${messenger.jobs.pipeline-version:2}") int pipelineVersion + ) { + this.attachments = attachments; + this.chats = chats; + this.storage = storage; + this.publisher = publisher; + this.transaction = transaction; + this.maxUploadBytes = maxUploadBytes; + this.maxImagePixels = maxImagePixels; + if (maxConcurrentUploads < 1 || uploadPermitWaitMillis < 1) { + throw new IllegalArgumentException("Upload concurrency limits must be positive"); + } + this.uploadPermits = new Semaphore(maxConcurrentUploads, true); + this.uploadPermitWaitMillis = uploadPermitWaitMillis; + this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit; + if (pipelineVersion < AttachmentJob.MIN_PIPELINE_VERSION + || pipelineVersion > AttachmentJob.CURRENT_PIPELINE_VERSION) { + throw new IllegalArgumentException("Unsupported attachment pipeline version"); + } + this.pipelineVersion = pipelineVersion; + } + + public Attachment upload(UUID chatId, + UUID ownerId, + String filename, + String declaredContentType, + long declaredLength, + InputStream input) { + chats.getAccessible(chatId, ownerId); + String safeFilename = validateFilename(filename); + String contentType = normalizeContentType(declaredContentType); + if (declaredLength > maxUploadBytes) { + throw ApiExceptions.payloadTooLarge("Attachment exceeds 10 MiB"); + } + + try { + try { + if (!uploadPermits.tryAcquire(uploadPermitWaitMillis, TimeUnit.MILLISECONDS)) { + throw ApiExceptions.tooManyRequests("Upload capacity is temporarily exhausted"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw ApiExceptions.unavailable("Upload was interrupted"); + } + try { + return storeValidated(chatId, ownerId, safeFilename, contentType, input); + } finally { + uploadPermits.release(); + } + } catch (com.example.mtusi_messenger.exception.ApiException error) { + throw error; + } + } + + private Attachment storeValidated(UUID chatId, + UUID ownerId, + String safeFilename, + String contentType, + InputStream input) { + byte[] bytes = readBounded(input); + if (bytes.length == 0) { + throw ApiExceptions.unsupportedMediaType("Image is empty"); + } + validateImage(bytes, contentType); + + UUID id = UuidCreator.getTimeOrderedEpochPlus1(); + String objectKey = "attachments/" + id + "/original"; + String stagingKey = "staging/" + id + "/original"; + Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); + AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, pipelineVersion, now); + Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType, + bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(), + null, now, now); + + storage.put(stagingKey, bytes, contentType); + try { + Attachment saved = transaction.execute(status -> attachments.saveQueued(attachment, job)); + if (injectCrashAfterOutboxCommit && attachments.markPublishFaultInjected(job.jobId())) { + // Opt-in fault injection for the exact DB COMMIT -> broker publish window. + // The persisted outbox row remains unpublished and is recovered after restart. + Runtime.getRuntime().halt(87); + } + try { + publisher.publish(job); + attachments.markPublished(job.jobId()); + return saved; + } catch (RuntimeException publishFailure) { + // The database job and staging object are intentionally retained. + // The scheduled republisher closes the DB-commit -> broker window. + throw ApiExceptions.unavailable("Attachment was staged; queue publication is pending"); + } + } catch (RuntimeException databaseFailure) { + if (databaseFailure instanceof com.example.mtusi_messenger.exception.ApiException) { + throw databaseFailure; + } + try { + storage.delete(stagingKey); + } catch (RuntimeException cleanupFailure) { + databaseFailure.addSuppressed(cleanupFailure); + } + throw databaseFailure; + } + } + + public Attachment getAccessible(UUID attachmentId, UUID userId) { + Attachment attachment = attachments.findById(attachmentId) + .orElseThrow(() -> ApiExceptions.notFound("Attachment not found")); + chats.getAccessible(attachment.chatId(), userId); + return attachment; + } + + public Download download(UUID attachmentId, UUID userId) { + return download(attachmentId, userId, "original"); + } + + public Download download(UUID attachmentId, UUID userId, String variantName) { + Attachment attachment = getAccessible(attachmentId, userId); + if (!"ready".equals(attachment.status())) { + throw ApiExceptions.conflict("Attachment content is not ready"); + } + if ("original".equals(variantName)) { + return new Download(attachment, attachment.contentType(), attachment.filename(), + storage.get(attachment.objectKey())); + } + if (!List.of("thumbnail", "crop", "optimized").contains(variantName)) { + throw ApiExceptions.invalid("Unsupported attachment variant"); + } + AttachmentVariant variant = attachment.variants().stream() + .filter(candidate -> candidate.name().equals(variantName)) + .findFirst() + .orElseThrow(() -> ApiExceptions.notFound("Attachment variant not found")); + String extension = "image/webp".equals(variant.contentType()) ? ".webp" : ".jpg"; + return new Download(attachment, variant.contentType(), variantName + extension, + storage.get(variant.objectKey())); + } + + public void validateForMessage(UUID chatId, List attachmentIds) { + if (attachmentIds.isEmpty()) { + return; + } + if (attachments.findAllByIdsInChat(chatId, attachmentIds).size() != attachmentIds.size()) { + throw ApiExceptions.notFound("Attachment not found"); + } + } + + private byte[] readBounded(InputStream input) { + try { + ByteArrayOutputStream output = new ByteArrayOutputStream( + (int) Math.min(maxUploadBytes, 1024 * 1024)); + byte[] chunk = new byte[8192]; + long total = 0; + int read; + while ((read = input.read(chunk)) != -1) { + total += read; + if (total > maxUploadBytes) { + throw ApiExceptions.payloadTooLarge("Attachment exceeds 10 MiB"); + } + output.write(chunk, 0, read); + } + return output.toByteArray(); + } catch (IOException error) { + throw ApiExceptions.invalid("Cannot read request body"); + } + } + + private void validateImage(byte[] bytes, String declaredContentType) { + ImageIO.setUseCache(false); + try (var imageInput = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) { + if (imageInput == null) { + throw ApiExceptions.unsupportedMediaType("Invalid image"); + } + Iterator readers = ImageIO.getImageReaders(imageInput); + if (!readers.hasNext()) { + throw ApiExceptions.unsupportedMediaType("Invalid PNG/JPEG image"); + } + ImageReader reader = readers.next(); + try { + reader.setInput(imageInput, true, true); + String actualType = switch (reader.getFormatName().toLowerCase(Locale.ROOT)) { + case "png" -> "image/png"; + case "jpeg", "jpg" -> "image/jpeg"; + default -> throw ApiExceptions.unsupportedMediaType("Only PNG and JPEG are supported"); + }; + if (!actualType.equals(declaredContentType)) { + throw ApiExceptions.unsupportedMediaType("Content-Type does not match image bytes"); + } + int width = reader.getWidth(0); + int height = reader.getHeight(0); + if (width <= 0 || height <= 0) { + throw ApiExceptions.unsupportedMediaType("Invalid image dimensions"); + } + if ((long) width * height > maxImagePixels) { + throw ApiExceptions.payloadTooLarge("Image exceeds 40 million pixels"); + } + if (reader.read(0) == null) { + throw ApiExceptions.unsupportedMediaType("Image cannot be decoded"); + } + } finally { + reader.dispose(); + } + } catch (com.example.mtusi_messenger.exception.ApiException error) { + throw error; + } catch (IOException | RuntimeException error) { + throw ApiExceptions.unsupportedMediaType("Corrupted PNG/JPEG image"); + } + } + + private static String validateFilename(String filename) { + if (filename == null || filename.isBlank() || filename.length() > 255 + || filename.equals(".") || filename.equals("..") + || filename.indexOf('/') >= 0 || filename.indexOf('\\') >= 0 + || filename.codePoints().anyMatch(Character::isISOControl)) { + throw ApiExceptions.invalid("Invalid filename"); + } + return filename; + } + + private static String normalizeContentType(String contentType) { + String normalized = contentType == null ? "" : contentType.split(";", 2)[0].trim().toLowerCase(Locale.ROOT); + if (!ALLOWED_TYPES.contains(normalized)) { + throw ApiExceptions.unsupportedMediaType("Only image/png and image/jpeg are supported"); + } + return normalized; + } + + private static String sha256(byte[] bytes) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + public record Download(Attachment attachment, String contentType, String filename, byte[] bytes) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java b/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java new file mode 100644 index 0000000..42abab9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java @@ -0,0 +1,104 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.cache.UserCache; +import com.example.mtusi_messenger.model.Session; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.repository.SessionRepository; +import com.example.mtusi_messenger.repository.UserRepository; +import com.example.mtusi_messenger.security.SessionPrincipal; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataAccessException; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Base64; +import java.util.UUID; + +@Service +public class AuthenticationService { + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + private final UserRepository users; + private final SessionRepository sessions; + private final UserCache userCache; + private final long ttlSeconds; + + public AuthenticationService( + UserRepository users, + SessionRepository sessions, + UserCache userCache, + @Value("${messenger.session.ttl-seconds:3600}") long ttlSeconds + ) { + if (ttlSeconds <= 0) { + throw new IllegalArgumentException("Session TTL must be positive"); + } + this.users = users; + this.sessions = sessions; + this.userCache = userCache; + this.ttlSeconds = ttlSeconds; + } + + public IssuedSession issueSession(User user) { + String token = newToken(); + Instant createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS); + Instant expiresAt = createdAt.plusSeconds(ttlSeconds); + sessions.save(new Session( + UUID.randomUUID(), + user.id(), + hashToken(token), + createdAt, + expiresAt)); + return new IssuedSession(token, expiresAt, user); + } + + public SessionPrincipal authenticateBearer(String token) { + byte[] tokenHash = hashToken(token); + try { + UUID userId = sessions.findActiveUserId(tokenHash) + .orElseThrow(() -> new BadCredentialsException("Invalid or expired session")); + User user = userCache.find(userId).orElseGet(() -> { + User loaded = users.findById(userId) + .orElseThrow(() -> new BadCredentialsException("Invalid or expired session")); + userCache.put(loaded); + return loaded; + }); + return new SessionPrincipal(user, tokenHash); + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + } + + public void revoke(SessionPrincipal principal) { + try { + if (!sessions.revoke(principal.tokenHash())) { + throw new BadCredentialsException("Invalid or expired session"); + } + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + } + + private String newToken() { + byte[] bytes = new byte[32]; + SECURE_RANDOM.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + private byte[] hashToken(String token) { + try { + return MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.US_ASCII)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + public record IssuedSession(String token, Instant expiresAt, User user) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/service/ChatService.java b/src/main/java/com/example/mtusi_messenger/service/ChatService.java new file mode 100644 index 0000000..38eeff2 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/ChatService.java @@ -0,0 +1,38 @@ +package com.example.mtusi_messenger.service; + +import org.springframework.stereotype.Service; +import com.example.mtusi_messenger.dto.CreateChatRequest; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.Chat; +import com.example.mtusi_messenger.repository.ChatRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +@Service +public class ChatService { + private final ChatRepository chats; private final UserService users; private final CursorService cursors; + public ChatService(ChatRepository chats, UserService users, CursorService cursors) { this.chats=chats; this.users=users; this.cursors=cursors; } + public Chat create(CreateChatRequest r, UUID caller) { + Set ids = new LinkedHashSet<>(r.member_ids()); + if (ids.size() != r.member_ids().size() || !ids.contains(caller)) throw ApiExceptions.invalid("Invalid member_ids"); + for (UUID id : ids) if (id == null || !users.exists(id)) throw ApiExceptions.invalid("Invalid member_ids"); + return chats.save(new Chat(UUID.randomUUID(), r.title(), List.copyOf(ids), + Instant.now().truncatedTo(ChronoUnit.MICROS))); + } + public Chat getAccessible(UUID chatId, UUID userId) { + users.get(userId); + if (!chats.isMember(chatId, userId)) throw ApiExceptions.notFound("Chat not found"); + return chats.findById(chatId).orElseThrow(() -> ApiExceptions.notFound("Chat not found")); + } + public List page(UUID userId, int limit, String cursor) { + CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "chats", userId); + return chats.findPageForUser(userId, c == null ? null : c.id(), c == null ? null : c.createdAt(), limit + 1); + } + public String nextCursor(UUID userId, + Chat last, + boolean hasNext + ) { + return hasNext ? cursors.encode("chats", userId, last.createdAt(), last.id()) : null; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/CursorService.java b/src/main/java/com/example/mtusi_messenger/service/CursorService.java new file mode 100644 index 0000000..87cd7ff --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/CursorService.java @@ -0,0 +1,40 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Service; +import com.example.mtusi_messenger.exception.ApiExceptions; +import java.time.Instant; +import java.util.Base64; +import java.util.UUID; + +@Service +public class CursorService { + private final ObjectMapper mapper; + public CursorService(ObjectMapper mapper) { this.mapper = mapper; } + + public String encode(String resource, UUID resourceId, Instant createdAt, UUID id) { + try { + byte[] json = mapper.writeValueAsBytes(new CursorPayload(1, resource, resourceId, createdAt.toString(), id)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(json); + } catch (Exception e) { throw new IllegalStateException(e); } + } + public Cursor decode(String raw, String resource, UUID resourceId) { + if (raw == null || raw.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + try { + CursorPayload p = mapper.readValue(Base64.getUrlDecoder().decode(raw), CursorPayload.class); + if (p.v != 1 + || !resource.equals(p.resource) + || !Objects.equals(resourceId, p.resourceId) + || p.createdAt() == null + || p.createdAt().isBlank() + || p.id() == null + ) throw ApiExceptions.invalid("Invalid cursor"); + return new Cursor(Instant.parse(p.createdAt), p.id); + } catch (ApiException e) { throw e; } + catch (Exception e) { throw ApiExceptions.invalid("Invalid cursor"); } + } + public record Cursor(Instant createdAt, UUID id) {} + private record CursorPayload(int v, String resource, UUID resourceId, String createdAt, UUID id) {} + private static final class Objects { static boolean equals(Object a, Object b) { return java.util.Objects.equals(a,b); } } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/MessageService.java b/src/main/java/com/example/mtusi_messenger/service/MessageService.java new file mode 100644 index 0000000..c164553 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/MessageService.java @@ -0,0 +1,102 @@ +package com.example.mtusi_messenger.service; + +import com.github.f4b6a3.uuid.UuidCreator; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.IdempotentMessage; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import com.example.mtusi_messenger.dto.CreateMessageRequest; +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.repository.MessageRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@Service +public class MessageService { + private static final String METHOD = "POST"; + private final MessageRepository messages; + private final ChatService chats; + private final CursorService cursors; + private final AttachmentService attachments; + private final Duration idempotencyTtl; + + public MessageService( + MessageRepository messages, + ChatService chats, + CursorService cursors, + AttachmentService attachments, + @Value("${messenger.idempotency.ttl-seconds:86400}") long idempotencyTtlSeconds + ) { + if (idempotencyTtlSeconds < 86400) { + throw new IllegalArgumentException("Idempotency TTL must be at least 24 hours"); + } + this.messages = messages; + this.chats = chats; + this.cursors = cursors; + this.attachments = attachments; + this.idempotencyTtl = Duration.ofSeconds(idempotencyTtlSeconds); + } + + @Transactional + public Message create(UUID chatId, UUID senderId, CreateMessageRequest r, String idempotencyKey) { + chats.getAccessible(chatId, senderId); + attachments.validateForMessage(chatId, r.attachment_ids()); + String path = "/api/v1/chats/" + chatId + "/messages"; + byte[] requestHash = hash(r.text(), r.attachment_ids()); + Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); + + // The transaction-scoped PostgreSQL advisory lock serializes the same + // idempotency scope across every API replica. A hash collision only + // causes harmless extra serialization; the full scope is still checked. + messages.lockIdempotencyScope(senderId, METHOD, path, idempotencyKey); + messages.deleteExpiredIdempotency( + senderId, METHOD, path, idempotencyKey, now.minus(idempotencyTtl)); + + var existing = messages.findIdempotent(senderId, METHOD, path, idempotencyKey); + if (existing.isPresent()) { + IdempotentMessage value = existing.get(); + if (!Arrays.equals(value.requestHash(), requestHash)) { + throw ApiExceptions.conflict("Idempotency-Key was already used with another request body"); + } + return value.message(); + } + + Message message = messages.save(new Message( + UuidCreator.getTimeOrderedEpochPlus1(), chatId, senderId, r.text(), now, + r.attachment_ids())); + messages.saveIdempotency( + senderId, METHOD, path, idempotencyKey, requestHash, message.id(), now); + return message; + } + + private static byte[] hash(String text, List attachmentIds) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] textBytes = text.getBytes(StandardCharsets.UTF_8); + digest.update(java.nio.ByteBuffer.allocate(Integer.BYTES).putInt(textBytes.length).array()); + digest.update(textBytes); + for (UUID id : attachmentIds) { + digest.update(java.nio.ByteBuffer.allocate(16) + .putLong(id.getMostSignificantBits()) + .putLong(id.getLeastSignificantBits()).array()); + } + return digest.digest(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + public List page(UUID chatId, UUID userId, int limit, String cursor) { + chats.getAccessible(chatId, userId); + CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "messages", chatId); + return messages.findPage(chatId, c == null ? null : c.createdAt(), c == null ? null : c.id(), limit + 1); + } + public String nextCursor(UUID chatId, Message last, boolean hasNext) { return hasNext ? cursors.encode("messages", chatId, last.createdAt(), last.id()) : null; } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java new file mode 100644 index 0000000..3b14c4b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java @@ -0,0 +1,85 @@ +package com.example.mtusi_messenger.service; + +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.AvailabilityChangeEvent; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import java.util.concurrent.atomic.AtomicBoolean; + +@Service +public class ReadinessService { + private final JdbcTemplate jdbc; + private final StringRedisTemplate redis; + private final ApplicationAvailability availability; + private final ApplicationEventPublisher events; + private final AtomicBoolean dependencyUnavailable = new AtomicBoolean(false); + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); + + public ReadinessService( + JdbcTemplate jdbc, + StringRedisTemplate redis, + ApplicationAvailability availability, + ApplicationEventPublisher events + ) { + this.jdbc = jdbc; + this.redis = redis; + this.availability = availability; + this.events = events; + } + + public boolean probeDependencies() { + try { + // Redis is an optional cache. Ping only warms the lazy connection; + // its failure must not make the authoritative API unavailable. + redis.execute((RedisCallback) connection -> connection.ping()); + } catch (DataAccessException ignored) { + // PostgreSQL remains the source of truth. + } + + try { + // The schema-level probe proves that both the shared idempotency + // schema (lab3) and the attachment schema (lab4/lab5) completed + // before this replica is allowed to serve business traffic. + jdbc.queryForObject(""" + SELECT (SELECT count(*) FROM message_idempotency) + + (SELECT count(*) FROM attachments) + """, Long.class); + markDependenciesAvailable(); + return true; + } catch (DataAccessException e) { + markDependenciesUnavailable(); + return false; + } + } + + public boolean acceptsBusinessTraffic() { + return !shuttingDown.get() + && !dependencyUnavailable.get() + && availability.getReadinessState() == ReadinessState.ACCEPTING_TRAFFIC; + } + + @EventListener(ContextClosedEvent.class) + void onContextClosed() { + shuttingDown.set(true); + } + + private void markDependenciesUnavailable() { + dependencyUnavailable.set(true); + AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC); + } + + private void markDependenciesAvailable() { + boolean recovered = dependencyUnavailable.getAndSet(false); + if (recovered && !shuttingDown.get()) { + AvailabilityChangeEvent.publish(events, this, ReadinessState.ACCEPTING_TRAFFIC); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java b/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java new file mode 100644 index 0000000..f347f89 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/StagingCleaner.java @@ -0,0 +1,65 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Clock; +import java.time.Duration; +import java.util.Set; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true) +public class StagingCleaner { + private static final Logger log = LoggerFactory.getLogger(StagingCleaner.class); + private static final String PREFIX = "staging/"; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final Duration retention; + private final Clock clock; + + @Autowired + public StagingCleaner( + AttachmentRepository attachments, + ObjectStorage storage, + @Value("${messenger.attachments.staging-retention-seconds:86400}") long retentionSeconds) { + this(attachments, storage, Duration.ofSeconds(retentionSeconds), Clock.systemUTC()); + } + + StagingCleaner( + AttachmentRepository attachments, + ObjectStorage storage, + Duration retention, + Clock clock) { + if (retention.isZero() || retention.isNegative()) { + throw new IllegalArgumentException("Staging retention must be positive"); + } + this.attachments = attachments; + this.storage = storage; + this.retention = retention; + this.clock = clock; + } + + @Scheduled( + fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}", + initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}") + public void cleanup() { + try { + Set active = attachments.findActiveStagingKeys(); + var deleteBefore = clock.instant().minus(retention); + for (ObjectStorage.StoredObject object : storage.list(PREFIX)) { + if (!active.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) { + storage.delete(object.objectKey()); + } + } + } catch (RuntimeException error) { + log.warn("Staging cleanup postponed: {}", error.getMessage()); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/UserService.java b/src/main/java/com/example/mtusi_messenger/service/UserService.java new file mode 100644 index 0000000..c8bed8b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/UserService.java @@ -0,0 +1,34 @@ +package com.example.mtusi_messenger.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.security.crypto.password.PasswordEncoder; +import com.example.mtusi_messenger.dto.CreateUserRequest; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.repository.UserRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class UserService { + private final UserRepository users; + private final PasswordEncoder passwordEncoder; + + public User create(CreateUserRequest r) { + String username = r.username(); String display = r.display_name(); + if (users.findByUsername(username).isPresent()) throw ApiExceptions.conflict("Username is already taken"); + String passwordHash = passwordEncoder.encode(r.password()); + try { + return users.save(new User(UUID.randomUUID(), username, display, + Instant.now().truncatedTo(ChronoUnit.MICROS)), passwordHash); + } catch (DuplicateKeyException e) { + throw ApiExceptions.conflict("Username is already taken"); + } + } + public User get(UUID id) { return users.findById(id).orElseThrow(() -> ApiExceptions.unauthorized("Unknown user identity")); } + public boolean exists(UUID id) { return users.findById(id).isPresent(); } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java b/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java new file mode 100644 index 0000000..ea292fe --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java @@ -0,0 +1,27 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiExceptions; + +import java.util.UUID; +import java.util.regex.Pattern; + +public final class UuidValidator { + + private static final Pattern UUID_PATTERN = Pattern.compile( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + ); + + private UuidValidator() {} + + public static UUID parse(String value) { + if (value == null || !UUID_PATTERN.matcher(value).matches()) { + throw ApiExceptions.invalid("Invalid UUID"); + } + + try { + return UUID.fromString(value); + } catch (IllegalArgumentException e) { + throw ApiExceptions.invalid("Invalid UUID"); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java b/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java new file mode 100644 index 0000000..b50f4b4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java @@ -0,0 +1,92 @@ +package com.example.mtusi_messenger.storage; + +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.ListObjectsArgs; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import okhttp3.OkHttpClient; + +import java.io.ByteArrayInputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Component +public class MinioObjectStorage implements ObjectStorage { + private final MinioClient client; + private final String bucket; + + public MinioObjectStorage( + @Value("${messenger.s3.endpoint}") String endpoint, + @Value("${messenger.s3.access-key}") String accessKey, + @Value("${messenger.s3.secret-key}") String secretKey, + @Value("${messenger.s3.bucket}") String bucket, + @Value("${messenger.s3.region:us-east-1}") String region + ) { + OkHttpClient httpClient = new OkHttpClient.Builder() + .connectTimeout(Duration.ofSeconds(2)) + .readTimeout(Duration.ofSeconds(10)) + .writeTimeout(Duration.ofSeconds(10)) + .callTimeout(Duration.ofSeconds(15)) + .build(); + this.client = MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(region) + .httpClient(httpClient) + .build(); + this.bucket = bucket; + } + + @Override + public void put(String objectKey, byte[] bytes, String contentType) { + try (var stream = new ByteArrayInputStream(bytes)) { + client.putObject(PutObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .contentType(contentType) + .stream(stream, bytes.length, -1) + .build()); + } catch (Exception error) { + throw new ObjectStorageException("Cannot write attachment object", error); + } + } + + @Override + public byte[] get(String objectKey) { + try (var stream = client.getObject(GetObjectArgs.builder() + .bucket(bucket).object(objectKey).build())) { + return stream.readAllBytes(); + } catch (Exception error) { + throw new ObjectStorageException("Cannot read attachment object", error); + } + } + + @Override + public void delete(String objectKey) { + try { + client.removeObject(RemoveObjectArgs.builder() + .bucket(bucket).object(objectKey).build()); + } catch (Exception error) { + throw new ObjectStorageException("Cannot remove attachment object", error); + } + } + + @Override + public List list(String prefix) { + try { + List objects = new ArrayList<>(); + for (var result : client.listObjects(ListObjectsArgs.builder() + .bucket(bucket).prefix(prefix).recursive(true).build())) { + var item = result.get(); + objects.add(new StoredObject(item.objectName(), item.lastModified().toInstant(), item.size())); + } + return List.copyOf(objects); + } catch (Exception error) { + throw new ObjectStorageException("Cannot list attachment objects", error); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java new file mode 100644 index 0000000..bb521dd --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.storage; + +import java.time.Instant; +import java.util.List; + +public interface ObjectStorage { + void put(String objectKey, byte[] bytes, String contentType); + byte[] get(String objectKey); + void delete(String objectKey); + List list(String prefix); + + record StoredObject(String objectKey, Instant lastModified, long sizeBytes) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/storage/ObjectStorageException.java b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorageException.java new file mode 100644 index 0000000..6173a46 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorageException.java @@ -0,0 +1,7 @@ +package com.example.mtusi_messenger.storage; + +public class ObjectStorageException extends RuntimeException { + public ObjectStorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java new file mode 100644 index 0000000..d9209c3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java @@ -0,0 +1,56 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.dao.DataAccessException; +import org.springframework.web.HttpMediaTypeNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.exception.ApiException; +import com.example.mtusi_messenger.storage.ObjectStorageException; + +@RestControllerAdvice +public class GlobalExceptionHandler { + @ExceptionHandler(ApiException.class) + ResponseEntity api(ApiException e, HttpServletRequest req) { + var body = body(e.code(), e.getMessage(), req); + var builder = ResponseEntity.status(e.status()); + if (e.status() == HttpStatus.UNAUTHORIZED && e.authenticate() != null) { + builder.header("WWW-Authenticate", e.authenticate()); + } + if (e.status() == HttpStatus.TOO_MANY_REQUESTS) { + builder.header("Retry-After", "1"); + } + return builder.body(body); + } + @ExceptionHandler({MethodArgumentNotValidException.class, MethodArgumentTypeMismatchException.class, HttpMessageNotReadableException.class}) + ResponseEntity bad(Exception e, HttpServletRequest req) { return ResponseEntity.badRequest().body(body("invalid_request", "Invalid request", req)); } + @ExceptionHandler(HttpMediaTypeNotSupportedException.class) + ResponseEntity unsupported(HttpServletRequest req) { return ResponseEntity.status(415).body(body("unsupported_media_type", "Unsupported media type", req)); } + @ExceptionHandler(DataAccessException.class) + ResponseEntity unavailable(DataAccessException e, HttpServletRequest req) { + return ResponseEntity.status(503).body(body("unavailable", "Storage temporarily unavailable", req)); + } + @ExceptionHandler(ObjectStorageException.class) + ResponseEntity storageUnavailable(ObjectStorageException e, HttpServletRequest req) { + return ResponseEntity.status(503).body(body("unavailable", "Object storage temporarily unavailable", req)); + } + @ExceptionHandler(Exception.class) + ResponseEntity internal(Exception e, HttpServletRequest req) { return ResponseEntity.status(500).body(body("internal_error", "Internal server error", req)); } + private ErrorResponse body( + String code, + String message, + HttpServletRequest req + ) { + String id = (String) req.getAttribute(RequestIdFilter.ATTRIBUTE); + + return new ErrorResponse( + new ErrorResponse.ErrorBody(code, message, id) + ); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java b/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java new file mode 100644 index 0000000..9c56a39 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java @@ -0,0 +1,47 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.UUID; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 1) +public class InstanceIdFilter extends OncePerRequestFilter { + public static final String HEADER = "X-Instance-Id"; + private final String instanceId; + + public InstanceIdFilter() { + String configured = System.getenv("INSTANCE_ID"); + this.instanceId = configured == null || configured.isBlank() + ? localHostOrRandom() + : configured; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + response.setHeader(HEADER, instanceId); + filterChain.doFilter(request, response); + } + + private static String localHostOrRandom() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return "api-" + UUID.randomUUID(); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java b/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java new file mode 100644 index 0000000..a4213fc --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.*; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = UnicodeBlankValidator.class) +@Target({ FIELD, PARAMETER, RECORD_COMPONENT, ANNOTATION_TYPE }) +@Retention(RUNTIME) +public @interface NotUnicodeBlank { + String message() default "must not be blank"; + Class[] groups() default {}; + Class[] payload() default {}; +} diff --git a/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java b/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java new file mode 100644 index 0000000..c69594e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java @@ -0,0 +1,56 @@ +package com.example.mtusi_messenger.web; + +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.service.ReadinessService; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; + +@Component +public class ReadinessGuardFilter extends OncePerRequestFilter { + private final ReadinessService readiness; + private final ObjectMapper mapper; + + public ReadinessGuardFilter(ReadinessService readiness, ObjectMapper mapper) { + this.readiness = readiness; + this.mapper = mapper; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + String path = request.getRequestURI(); + return path.startsWith("/health/") + || path.equals("/swagger-ui.html") + || path.startsWith("/swagger-ui/") + || path.startsWith("/v3/api-docs/") + || path.equals("/v3/api-docs"); + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + if (readiness.acceptsBusinessTraffic()) { + filterChain.doFilter(request, response); + return; + } + + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE); + mapper.writeValue(response.getOutputStream(), new ErrorResponse( + new ErrorResponse.ErrorBody( + "unavailable", + "Application is not ready to serve business traffic", + requestId))); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java b/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java new file mode 100644 index 0000000..eef205a --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java @@ -0,0 +1,56 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.web.servlet.HandlerMapping; +import org.springframework.web.filter.OncePerRequestFilter; +import java.io.IOException; +import java.util.UUID; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestIdFilter extends OncePerRequestFilter { + + private static final Logger log = LoggerFactory.getLogger(RequestIdFilter.class); + + public static final String HEADER = "X-Request-Id"; + public static final String ATTRIBUTE = "request_id"; + + @Override + protected void doFilterInternal( + HttpServletRequest req, + HttpServletResponse res, + FilterChain chain + ) throws ServletException, IOException { + + String id = UUID.randomUUID().toString(); + + req.setAttribute(ATTRIBUTE, id); + res.setHeader(HEADER, id); + + try { + MDC.put(ATTRIBUTE, id); + chain.doFilter(req, res); + } finally { + Object routeAttribute = req.getAttribute(HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE); + String route = routeAttribute == null ? null : routeAttribute.toString(); + if (route != null && !route.startsWith("/health/")) { + log.atInfo() + .addKeyValue("event", "http_request_completed") + .addKeyValue("http.request.method", req.getMethod()) + .addKeyValue("http.route", route) + .addKeyValue("http.response.status_code", res.getStatus()) + .log("HTTP request completed"); + } + MDC.remove(ATTRIBUTE); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java new file mode 100644 index 0000000..5989361 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java @@ -0,0 +1,23 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UnicodeBlankValidator + implements ConstraintValidator { + + @Override + public boolean isValid( + String value, + ConstraintValidatorContext context) { + + if (value == null || value.isEmpty()) { + return false; + } + + return value.codePoints().anyMatch(codePoint -> + !Character.isWhitespace(codePoint) + && !Character.isSpaceChar(codePoint) + ); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java new file mode 100644 index 0000000..dde7ca9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java @@ -0,0 +1,35 @@ +package com.example.mtusi_messenger.web; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = UnicodeSizeValidator.class) +@Target({ + FIELD, + METHOD, + PARAMETER, + ANNOTATION_TYPE +}) +@Retention(RUNTIME) +public @interface UnicodeSize { + + String message() default "size must be between {min} and {max} Unicode code points"; + + int min() default 0; + + int max() default Integer.MAX_VALUE; + + Class[] groups() default {}; + + Class[] payload() default {}; +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java new file mode 100644 index 0000000..f978d54 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java @@ -0,0 +1,32 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UnicodeSizeValidator + implements ConstraintValidator { + + private int min; + private int max; + + @Override + public void initialize(UnicodeSize annotation) { + this.min = annotation.min(); + this.max = annotation.max(); + } + + @Override + public boolean isValid( + String value, + ConstraintValidatorContext context + ) { + if (value == null) { + return true; + } + + int codePointCount = value.codePointCount(0, value.length()); + + return codePointCount >= min + && codePointCount <= max; + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java new file mode 100644 index 0000000..e3d42d8 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/worker/AttachmentWorker.java @@ -0,0 +1,315 @@ +package com.example.mtusi_messenger.worker; + +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.AttachmentJob; +import com.example.mtusi_messenger.model.AttachmentVariant; +import com.example.mtusi_messenger.observability.JobTelemetry; +import com.example.mtusi_messenger.queue.RetryPolicy; +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import com.example.mtusi_messenger.storage.ObjectStorageException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.rabbitmq.client.Channel; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Component +@ConditionalOnProperty(name = "messenger.role", havingValue = "worker") +public class AttachmentWorker { + private static final Logger log = LoggerFactory.getLogger(AttachmentWorker.class); + private final ObjectMapper mapper; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final ImageVariantProcessor images; + private final RetryPolicy retryPolicy; + private final TransactionTemplate transaction; + private final JobTelemetry telemetry; + private final int maxAttempts; + private final Duration jobTimeout; + private final Duration leaseDuration; + private final boolean injectCrashBeforeSideEffect; + private final boolean injectCrashAfterSideEffect; + private final boolean injectCrashAfterResultCommit; + private final ExecutorService processingPool; + private final Clock clock = Clock.systemUTC(); + + public AttachmentWorker( + ObjectMapper mapper, + AttachmentRepository attachments, + ObjectStorage storage, + ImageVariantProcessor images, + RetryPolicy retryPolicy, + TransactionTemplate transaction, + JobTelemetry telemetry, + @Value("${messenger.jobs.max-attempts:5}") int maxAttempts, + @Value("${messenger.jobs.timeout:60s}") Duration jobTimeout, + @Value("${messenger.jobs.lease-duration:90s}") Duration leaseDuration, + @Value("${messenger.jobs.fault-crash-before-side-effect:false}") boolean injectCrashBeforeSideEffect, + @Value("${messenger.jobs.fault-crash-after-side-effect:false}") boolean injectCrashAfterSideEffect, + @Value("${messenger.jobs.fault-crash-after-result-commit:false}") boolean injectCrashAfterResultCommit, + @Value("${spring.rabbitmq.listener.simple.concurrency:2}") int concurrency) { + if (maxAttempts < 1 || jobTimeout.isZero() || jobTimeout.isNegative() + || leaseDuration.compareTo(jobTimeout) <= 0 || concurrency < 1) { + throw new IllegalArgumentException("Invalid attachment worker limits"); + } + this.mapper = mapper; + this.attachments = attachments; + this.storage = storage; + this.images = images; + this.retryPolicy = retryPolicy; + this.transaction = transaction; + this.telemetry = telemetry; + this.maxAttempts = maxAttempts; + this.jobTimeout = jobTimeout; + this.leaseDuration = leaseDuration; + this.injectCrashBeforeSideEffect = injectCrashBeforeSideEffect; + this.injectCrashAfterSideEffect = injectCrashAfterSideEffect; + this.injectCrashAfterResultCommit = injectCrashAfterResultCommit; + this.processingPool = Executors.newFixedThreadPool(concurrency, + Thread.ofPlatform().name("attachment-image-", 0).factory()); + } + + @RabbitListener(queues = "${messenger.jobs.queue}") + public void consume(Message message, Channel channel) throws IOException { + long deliveryTag = message.getMessageProperties().getDeliveryTag(); + AttachmentJob job; + try { + job = mapper.readValue(message.getBody(), AttachmentJob.class); + } catch (RuntimeException | IOException invalidMessage) { + telemetry.malformedMessage(); + log.atError() + .addKeyValue("event", "attachment_job_rejected") + .addKeyValue("outcome", "invalid_message") + .log("Rejecting malformed attachment job message"); + channel.basicReject(deliveryTag, false); + return; + } + + Attachment attachment = attachments.findById(job.attachmentId()).orElse(null); + if (attachment == null || "ready".equals(attachment.status()) || "failed".equals(attachment.status())) { + channel.basicAck(deliveryTag, false); + return; + } + + int attempt = Math.max(job.attempt(), attachments.currentAttempt(job.jobId())); + UUID leaseToken = UUID.randomUUID(); + boolean claimed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.claim(job.jobId(), job.attachmentId(), job.pipelineVersion(), + attempt, leaseToken, leaseDuration))); + if (!claimed) { + // A live worker owns the lease, or a terminal state won the race. + // If the owner died, the DB lease reaper will create a new outbox delivery. + channel.basicAck(deliveryTag, false); + return; + } + + long startedNanos = telemetry.started(); + String outcome = "error"; + Future future = processingPool.submit(() -> process(job, leaseToken)); + try { + ProcessOutcome processOutcome = future.get(jobTimeout.toMillis(), TimeUnit.MILLISECONDS); + if (processOutcome.crashAfterCommit()) { + log.atError().addKeyValue("event", "fault_after_result_commit") + .log("Fault injection: halting worker after durable result and before consumer ack"); + Runtime.getRuntime().halt(86); + } + channel.basicAck(deliveryTag, false); + outcome = "success"; + log.atInfo() + .addKeyValue("event", "attachment_job_completed") + .addKeyValue("job.type", AttachmentJob.TYPE) + .addKeyValue("outcome", outcome) + .log("Attachment job completed"); + } catch (TimeoutException timeout) { + future.cancel(true); + retryOrFail(job, leaseToken, attempt, "processing_timeout", deliveryTag, channel); + outcome = attempt >= maxAttempts ? "failed" : "retry"; + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + future.cancel(true); + channel.basicNack(deliveryTag, false, true); + outcome = "interrupted"; + } catch (ExecutionException execution) { + Throwable cause = execution.getCause(); + if (cause instanceof ImageVariantProcessor.ProcessingException imageFailure + && imageFailure.permanent()) { + failPermanent(job, leaseToken, imageFailure.errorCode(), deliveryTag, channel); + outcome = "failed"; + } else { + String code = cause instanceof ImageVariantProcessor.ProcessingException imageFailure + ? imageFailure.errorCode() : "processing_failed"; + retryOrFail(job, leaseToken, attempt, code, deliveryTag, channel); + outcome = attempt >= maxAttempts ? "failed" : "retry"; + } + } finally { + telemetry.finished(startedNanos, job.createdAt(), outcome); + } + } + + private ProcessOutcome process(AttachmentJob job, UUID leaseToken) { + if (injectCrashBeforeSideEffect + && attachments.markBeforeEffectFaultInjected(job.jobId(), leaseToken)) { + log.atError().addKeyValue("event", "fault_before_side_effect") + .log("Fault injection: halting worker before object-storage side effect"); + Runtime.getRuntime().halt(84); + } + Attachment attachment = attachments.findById(job.attachmentId()) + .orElseThrow(() -> new ImageVariantProcessor.ProcessingException( + "attachment_missing", true, "Attachment metadata is missing", null)); + byte[] original = loadOriginal(attachment); + if (original.length != attachment.sizeBytes() || !sha256(original).equals(attachment.sha256())) { + throw new ImageVariantProcessor.ProcessingException( + "invalid_image", true, "Staging checksum mismatch", null); + } + + storage.put(attachment.objectKey(), original, attachment.contentType()); + List generated = + images.createVariants(original, job.pipelineVersion()); + List variants = new ArrayList<>(generated.size()); + for (ImageVariantProcessor.GeneratedVariant item : generated) { + String objectKey = "attachments/" + attachment.id() + "/" + + item.name() + "-v" + job.pipelineVersion() + "." + item.extension(); + storage.put(objectKey, item.bytes(), item.contentType()); + variants.add(new AttachmentVariant( + item.name(), item.contentType(), item.bytes().length, + item.width(), item.height(), sha256(item.bytes()), objectKey)); + } + + if (injectCrashAfterSideEffect + && attachments.markAfterEffectFaultInjected(job.jobId(), leaseToken)) { + log.atError().addKeyValue("event", "fault_after_side_effect") + .log("Fault injection: halting worker after object-storage side effect before result commit"); + Runtime.getRuntime().halt(85); + } + + AttachmentRepository.CompletionResult completion = transaction.execute(status -> + attachments.markReady(job.jobId(), job.attachmentId(), leaseToken, + job.pipelineVersion(), variants, injectCrashAfterResultCommit)); + if (completion == null || !completion.completed()) { + return new ProcessOutcome(false); + } + try { + storage.delete(attachment.stagingKey()); + } catch (RuntimeException cleanupFailure) { + log.atWarn().addKeyValue("event", "staging_cleanup_postponed") + .log("Staging cleanup postponed after attachment processing"); + } + return new ProcessOutcome(completion.crashAfterCommit()); + } + + private byte[] loadOriginal(Attachment attachment) { + try { + return storage.get(attachment.stagingKey()); + } catch (ObjectStorageException missingStaging) { + try { + // A pipeline promotion reuses the immutable final original after + // the previous version safely removed its staging object. + return storage.get(attachment.objectKey()); + } catch (ObjectStorageException missingOriginal) { + missingOriginal.addSuppressed(missingStaging); + throw missingOriginal; + } + } + } + + private void failPermanent( + AttachmentJob job, UUID leaseToken, String errorCode, + long deliveryTag, Channel channel) throws IOException { + boolean failed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markFailed(job.jobId(), job.attachmentId(), job.pipelineVersion(), + leaseToken, errorCode))); + if (failed) { + channel.basicReject(deliveryTag, false); + } else { + channel.basicAck(deliveryTag, false); + } + } + + private void retryOrFail( + AttachmentJob job, UUID leaseToken, int attempt, String errorCode, + long deliveryTag, Channel channel) throws IOException { + try { + if (attempt >= maxAttempts) { + boolean failed = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markFailed(job.jobId(), job.attachmentId(), + job.pipelineVersion(), leaseToken, "retry_exhausted"))); + if (failed) { + log.atError() + .addKeyValue("event", "attachment_job_retry_exhausted") + .addKeyValue("job.type", AttachmentJob.TYPE) + .addKeyValue("outcome", "failed") + .addKeyValue("attempt", attempt) + .addKeyValue("error.code", errorCode) + .log("Attachment job exhausted retry attempts"); + channel.basicReject(deliveryTag, false); + } else { + channel.basicAck(deliveryTag, false); + } + return; + } + + int nextAttempt = attempt + 1; + boolean queued = Boolean.TRUE.equals(transaction.execute(status -> + attachments.markQueuedForRetry( + job.jobId(), job.attachmentId(), job.pipelineVersion(), + leaseToken, nextAttempt, errorCode, + clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt))))); + // The DB outbox is now the durable retry carrier; the current broker + // delivery can be acknowledged without a DB/broker dual write. + channel.basicAck(deliveryTag, false); + if (queued) { + log.atWarn() + .addKeyValue("event", "attachment_job_retry_scheduled") + .addKeyValue("job.type", AttachmentJob.TYPE) + .addKeyValue("outcome", "retry") + .addKeyValue("attempt", nextAttempt) + .addKeyValue("error.code", errorCode) + .log("Attachment job scheduled for retry"); + } else { + log.atInfo().addKeyValue("event", "attachment_job_retry_skipped") + .log("Retry skipped because job lost its lease or became terminal"); + } + } catch (RuntimeException databaseFailure) { + channel.basicNack(deliveryTag, false, true); + } + } + + @PreDestroy + void shutdownPool() { + processingPool.shutdownNow(); + } + + private static String sha256(byte[] bytes) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + private record ProcessOutcome(boolean crashAfterCommit) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java new file mode 100644 index 0000000..f7e9faf --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/worker/ImageVariantProcessor.java @@ -0,0 +1,270 @@ +package com.example.mtusi_messenger.worker; + +import net.coobird.thumbnailator.Thumbnails; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.imageio.IIOImage; +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.ImageWriteParam; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +@Component +public class ImageVariantProcessor { + private static final long MEBIBYTE = 1024L * 1024L; + private static final long OUTPUT_ALLOWANCE_BYTES = 64L * MEBIBYTE; + private final long maxPixels; + private final long memoryBudgetBytes; + private final Semaphore memoryPermits; + private final Duration memoryWait; + + public ImageVariantProcessor( + @Value("${messenger.jobs.worker-max-pixels:40000000}") long maxPixels, + @Value("${messenger.jobs.worker-memory-budget-bytes:536870912}") long memoryBudgetBytes, + @Value("${messenger.jobs.memory-wait:2s}") Duration memoryWait) { + if (maxPixels < 1 || memoryBudgetBytes < MEBIBYTE || memoryWait.isNegative()) { + throw new IllegalArgumentException("Invalid worker image resource limits"); + } + this.maxPixels = maxPixels; + this.memoryBudgetBytes = memoryBudgetBytes; + this.memoryPermits = new Semaphore((int) Math.min(Integer.MAX_VALUE, + Math.max(1, memoryBudgetBytes / MEBIBYTE)), true); + this.memoryWait = memoryWait; + } + + public List createVariants(byte[] original, int pipelineVersion) { + PipelineProfile profile = PipelineProfile.forVersion(pipelineVersion); + ImageInfo info = inspect(original); + long estimatedBytes; + try { + estimatedBytes = Math.addExact(Math.multiplyExact(info.pixels(), 12L), + OUTPUT_ALLOWANCE_BYTES); + } catch (ArithmeticException overflow) { + throw permanent("image_too_large", "Image memory estimate overflow", overflow); + } + if (estimatedBytes > memoryBudgetBytes) { + throw permanent("image_too_large", "Image exceeds worker memory budget", null); + } + int permits = (int) Math.max(1, (estimatedBytes + MEBIBYTE - 1) / MEBIBYTE); + boolean acquired; + try { + acquired = memoryPermits.tryAcquire(permits, memoryWait.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw temporary("worker_interrupted", "Interrupted while waiting for memory budget", interrupted); + } + if (!acquired) { + throw temporary("worker_capacity", "Worker memory budget is currently exhausted", null); + } + + try { + return transform(original, profile); + } finally { + memoryPermits.release(permits); + } + } + + private ImageInfo inspect(byte[] bytes) { + ImageIO.setUseCache(false); + try (var input = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) { + if (input == null) { + throw permanent("invalid_image", "Image input is invalid", null); + } + Iterator readers = ImageIO.getImageReaders(input); + if (!readers.hasNext()) { + throw permanent("invalid_image", "Unsupported image encoding", null); + } + ImageReader reader = readers.next(); + try { + reader.setInput(input, true, true); + int width = reader.getWidth(0); + int height = reader.getHeight(0); + long pixels = Math.multiplyExact((long) width, height); + if (width < 1 || height < 1 || pixels > maxPixels) { + throw permanent("image_too_large", "Image exceeds worker pixel limit", null); + } + return new ImageInfo(width, height, pixels); + } finally { + reader.dispose(); + } + } catch (ProcessingException error) { + throw error; + } catch (IOException | RuntimeException error) { + throw permanent("invalid_image", "Image header cannot be read", error); + } + } + + private List transform(byte[] original, PipelineProfile profile) { + try { + BufferedImage oriented = Thumbnails.of(new ByteArrayInputStream(original)) + .useExifOrientation(true) + .scale(1.0) + .asBufferedImage(); + if (oriented == null || oriented.getWidth() < 1 || oriented.getHeight() < 1) { + throw permanent("invalid_image", "Image cannot be decoded", null); + } + try { + List variants = new ArrayList<>(3); + BufferedImage thumbnail = fit(oriented, 256, BufferedImage.TYPE_INT_RGB, true); + variants.add(encode("thumbnail", "image/jpeg", "jpg", thumbnail, "jpeg", + profile.thumbnailQuality())); + thumbnail.flush(); + + BufferedImage crop = cropSquare(oriented, 128); + variants.add(encode("crop", "image/jpeg", "jpg", crop, "jpeg", + profile.cropQuality())); + crop.flush(); + + int webpType = oriented.getColorModel().hasAlpha() + ? BufferedImage.TYPE_INT_ARGB : BufferedImage.TYPE_INT_RGB; + BufferedImage optimized = fit(oriented, 1280, webpType, !oriented.getColorModel().hasAlpha()); + variants.add(encode("optimized", "image/webp", "webp", optimized, "webp", + profile.webpQuality())); + optimized.flush(); + return List.copyOf(variants); + } finally { + oriented.flush(); + } + } catch (ProcessingException error) { + throw error; + } catch (IOException | RuntimeException error) { + throw permanent("invalid_image", "Image cannot be transformed", error); + } + } + + private static BufferedImage fit(BufferedImage source, int maxSide, int type, boolean whiteBackground) { + double scale = Math.min(1.0, Math.min( + (double) maxSide / source.getWidth(), + (double) maxSide / source.getHeight())); + int width = Math.max(1, (int) Math.floor(source.getWidth() * scale)); + int height = Math.max(1, (int) Math.floor(source.getHeight() * scale)); + BufferedImage target = new BufferedImage(width, height, type); + Graphics2D graphics = qualityGraphics(target, whiteBackground); + try { + graphics.drawImage(source, 0, 0, width, height, null); + } finally { + graphics.dispose(); + } + return target; + } + + private static BufferedImage cropSquare(BufferedImage source, int side) { + int sourceSide = Math.min(source.getWidth(), source.getHeight()); + int x = (source.getWidth() - sourceSide) / 2; + int y = (source.getHeight() - sourceSide) / 2; + BufferedImage target = new BufferedImage(side, side, BufferedImage.TYPE_INT_RGB); + Graphics2D graphics = qualityGraphics(target, true); + try { + graphics.drawImage(source, 0, 0, side, side, + x, y, x + sourceSide, y + sourceSide, null); + } finally { + graphics.dispose(); + } + return target; + } + + private static Graphics2D qualityGraphics(BufferedImage target, boolean whiteBackground) { + Graphics2D graphics = target.createGraphics(); + if (whiteBackground) { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, target.getWidth(), target.getHeight()); + } + graphics.setRenderingHint(RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BICUBIC); + graphics.setRenderingHint(RenderingHints.KEY_RENDERING, + RenderingHints.VALUE_RENDER_QUALITY); + graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, + RenderingHints.VALUE_ANTIALIAS_ON); + return graphics; + } + + private static GeneratedVariant encode( + String name, String contentType, String extension, + BufferedImage image, String format, float quality) throws IOException { + Iterator writers = ImageIO.getImageWritersByFormatName(format); + if (!writers.hasNext()) { + throw new IOException(format + " writer is unavailable"); + } + var writer = writers.next(); + try (var output = new ByteArrayOutputStream(); + var imageOutput = ImageIO.createImageOutputStream(output)) { + writer.setOutput(imageOutput); + ImageWriteParam parameters = writer.getDefaultWriteParam(); + if (parameters.canWriteCompressed()) { + parameters.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); + String[] compressionTypes = parameters.getCompressionTypes(); + if (compressionTypes != null && compressionTypes.length > 0) { + parameters.setCompressionType(compressionTypes[0]); + } + parameters.setCompressionQuality(quality); + } + // Source metadata is intentionally omitted for every generated variant. + writer.write(null, new IIOImage(image, null, null), parameters); + imageOutput.flush(); + return new GeneratedVariant(name, contentType, extension, output.toByteArray(), + image.getWidth(), image.getHeight()); + } finally { + writer.dispose(); + } + } + + private static ProcessingException permanent(String code, String message, Throwable cause) { + return new ProcessingException(code, true, message, cause); + } + + private static ProcessingException temporary(String code, String message, Throwable cause) { + return new ProcessingException(code, false, message, cause); + } + + private record ImageInfo(int width, int height, long pixels) { + } + + private record PipelineProfile( + int version, float thumbnailQuality, float cropQuality, float webpQuality) { + private static PipelineProfile forVersion(int version) { + return switch (version) { + case 1 -> new PipelineProfile(1, 0.85f, 0.88f, 0.82f); + case 2 -> new PipelineProfile(2, 0.87f, 0.90f, 0.84f); + default -> throw permanent( + "unsupported_pipeline", "Unsupported pipeline version " + version, null); + }; + } + } + + public record GeneratedVariant(String name, String contentType, String extension, + byte[] bytes, int width, int height) { + } + + public static final class ProcessingException extends RuntimeException { + private final String errorCode; + private final boolean permanent; + + public ProcessingException( + String errorCode, boolean permanent, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + this.permanent = permanent; + } + + public String errorCode() { + return errorCode; + } + + public boolean permanent() { + return permanent; + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..22e6d17 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,129 @@ +spring: + application: + name: mtusi-messenger + + datasource: + url: ${DATABASE_URL:jdbc:postgresql://localhost:5432/messenger} + username: ${DATABASE_USER:messenger} + password: ${DATABASE_PASSWORD:messenger} + hikari: + maximum-pool-size: ${DATABASE_POOL_SIZE:10} + minimum-idle: ${DATABASE_MIN_IDLE:2} + connection-timeout: 5000 + validation-timeout: 2000 + idle-timeout: 600000 + max-lifetime: 1800000 + + data: + redis: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + connect-timeout: ${REDIS_CONNECT_TIMEOUT:1s} + timeout: ${REDIS_COMMAND_TIMEOUT:500ms} + lettuce: + pool: + max-active: ${REDIS_POOL_MAX_ACTIVE:8} + max-idle: ${REDIS_POOL_MAX_IDLE:8} + min-idle: ${REDIS_POOL_MIN_IDLE:0} + max-wait: ${REDIS_POOL_MAX_WAIT:500ms} + + rabbitmq: + host: ${RABBITMQ_HOST:localhost} + port: ${RABBITMQ_PORT:5672} + username: ${RABBITMQ_USER:messenger} + password: ${RABBITMQ_PASSWORD:messenger} + virtual-host: ${RABBITMQ_VHOST:/} + connection-timeout: ${RABBITMQ_CONNECTION_TIMEOUT:5s} + publisher-confirm-type: correlated + publisher-returns: true + listener: + simple: + acknowledge-mode: manual + default-requeue-rejected: false + prefetch: ${WORKER_PREFETCH:1} + concurrency: ${WORKER_CONCURRENCY:2} + max-concurrency: ${WORKER_CONCURRENCY:2} + + jdbc: + template: + query-timeout: ${DATABASE_QUERY_TIMEOUT_SECONDS:5} + + transaction: + default-timeout: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:5} + + flyway: + # API replicas never race one another for migrations. Compose runs the + # same image once as the dedicated migrate service with this overridden. + enabled: ${FLYWAY_ENABLED:false} + locations: classpath:db/migration + + jackson: + deserialization: + fail-on-unknown-properties: true + + lifecycle: + timeout-per-shutdown-phase: 10s + +server: + address: ${APP_HOST:0.0.0.0} + port: ${APP_PORT:8080} + shutdown: graceful + +# Spring Boot writes one JSON object per line. The OTel agent injects trace_id +# and span_id into MDC and exports the same records through OTLP. Request and +# message bodies, credentials and signed URLs are never configured for capture. +logging: + structured: + format: + console: logstash + +messenger: + role: ${MESSENGER_ROLE:api} + storage: postgres + session: + ttl-seconds: ${SESSION_TTL_SECONDS:3600} + cache: + user-ttl-seconds: ${USER_CACHE_TTL_SECONDS:300} + idempotency: + ttl-seconds: ${IDEMPOTENCY_TTL_SECONDS:86400} + s3: + endpoint: ${S3_ENDPOINT:http://localhost:9000} + region: ${S3_REGION:us-east-1} + bucket: ${S3_BUCKET:messenger-attachments} + access-key: ${S3_ACCESS_KEY_ID:messenger-api} + secret-key: ${S3_SECRET_ACCESS_KEY:messenger-api-local-only} + attachments: + max-upload-bytes: ${MAX_UPLOAD_BYTES:10485760} + max-image-pixels: ${MAX_IMAGE_PIXELS:40000000} + max-concurrent-uploads: ${MAX_CONCURRENT_UPLOADS:3} + upload-permit-wait-millis: ${UPLOAD_PERMIT_WAIT_MILLIS:10000} + staging-retention-seconds: ${STAGING_RETENTION_SECONDS:86400} + cleanup-grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} + cleanup-dry-run: ${ATTACHMENT_CLEANUP_DRY_RUN:false} + reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000} + reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000} + jobs: + pipeline-version: ${ATTACHMENT_PIPELINE_VERSION:2} + exchange: ${ATTACHMENT_JOB_EXCHANGE:attachments.v1} + queue: ${ATTACHMENT_JOB_QUEUE:attachments.process.level4.v1} + routing-key: ${ATTACHMENT_JOB_ROUTING_KEY:attachment.process.v1} + dead-letter-exchange: ${ATTACHMENT_DLQ_EXCHANGE:attachments.dlx.v1} + dead-letter-queue: ${ATTACHMENT_DLQ_QUEUE:attachments.failed.v1} + dead-letter-routing-key: ${ATTACHMENT_DLQ_ROUTING_KEY:attachment.failed.v1} + max-attempts: ${JOB_MAX_ATTEMPTS:5} + retry-base-delay: ${JOB_RETRY_BASE_DELAY:1s} + retry-max-delay: ${JOB_RETRY_MAX_DELAY:30s} + timeout: ${JOB_TIMEOUT:60s} + lease-duration: ${JOB_LEASE_DURATION:90s} + lease-reaper-interval: ${JOB_LEASE_REAPER_INTERVAL:5s} + worker-max-pixels: ${WORKER_MAX_IMAGE_PIXELS:40000000} + worker-memory-budget-bytes: ${WORKER_MEMORY_BUDGET_BYTES:536870912} + memory-wait: ${WORKER_MEMORY_WAIT:2s} + publish-confirm-timeout: ${JOB_PUBLISH_CONFIRM_TIMEOUT:5s} + republish-interval: ${JOB_REPUBLISH_INTERVAL:5s} + fault-crash-after-outbox-commit: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:false} + fault-crash-before-side-effect: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:false} + fault-crash-after-side-effect: ${FAULT_CRASH_AFTER_SIDE_EFFECT:false} + fault-crash-after-result-commit: ${FAULT_CRASH_AFTER_RESULT_COMMIT:false} + observability: + queue-poll-interval: ${OBSERVABILITY_QUEUE_POLL_INTERVAL:5s} diff --git a/src/main/resources/db/migration/V1__create_messenger_schema.sql b/src/main/resources/db/migration/V1__create_messenger_schema.sql new file mode 100644 index 0000000..06885a1 --- /dev/null +++ b/src/main/resources/db/migration/V1__create_messenger_schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT users_username_uk UNIQUE (username), + CONSTRAINT users_username_format_chk + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length_chk + CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT chats_title_length_chk + CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL, + user_id uuid NOT NULL, + joined_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_fk + FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE CASCADE, + CONSTRAINT chat_members_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE RESTRICT +); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT messages_text_length_chk + CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_membership_fk + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members(chat_id, user_id) + ON DELETE RESTRICT +); + +CREATE INDEX chat_members_user_chat_idx + ON chat_members (user_id, chat_id); + +CREATE INDEX chats_created_id_idx + ON chats (created_at, id); + +CREATE INDEX messages_chat_created_id_idx + ON messages (chat_id, created_at, id); + +CREATE INDEX messages_sender_created_idx + ON messages (sender_id, created_at); diff --git a/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql b/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql new file mode 100644 index 0000000..d5d7427 --- /dev/null +++ b/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql @@ -0,0 +1,28 @@ +ALTER TABLE users + ADD COLUMN password_hash text; + +ALTER TABLE users + ALTER COLUMN password_hash SET NOT NULL; + +CREATE TABLE sessions ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL, + token_hash bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + + CONSTRAINT sessions_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + CONSTRAINT sessions_token_hash_uk UNIQUE (token_hash), + CONSTRAINT sessions_token_hash_length_chk + CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_chk + CHECK (expires_at > created_at), + CONSTRAINT sessions_revocation_chk + CHECK (revoked_at IS NULL OR revoked_at >= created_at) +); + +CREATE INDEX sessions_active_expiry_idx + ON sessions (expires_at) + WHERE revoked_at IS NULL; diff --git a/src/main/resources/db/migration/V3__add_message_idempotency.sql b/src/main/resources/db/migration/V3__add_message_idempotency.sql new file mode 100644 index 0000000..dfa127c --- /dev/null +++ b/src/main/resources/db/migration/V3__add_message_idempotency.sql @@ -0,0 +1,25 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL, + method varchar(8) NOT NULL, + request_path text NOT NULL, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT message_idempotency_pk + PRIMARY KEY (user_id, method, request_path, idempotency_key), + CONSTRAINT message_idempotency_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk + FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_method_chk + CHECK (method = 'POST'), + CONSTRAINT message_idempotency_key_format_chk + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_hash_length_chk + CHECK (octet_length(request_hash) = 32) +); + +CREATE INDEX message_idempotency_created_idx + ON message_idempotency (created_at); diff --git a/src/main/resources/db/migration/V4__add_attachments.sql b/src/main/resources/db/migration/V4__add_attachments.sql new file mode 100644 index 0000000..ca018eb --- /dev/null +++ b/src/main/resources/db/migration/V4__add_attachments.sql @@ -0,0 +1,53 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_length_chk; +ALTER TABLE messages ADD CONSTRAINT messages_text_length_chk + CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_uk UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL, + size_bytes bigint NOT NULL, + sha256 char(64) NOT NULL, + object_key text NOT NULL, + status varchar(16) NOT NULL DEFAULT 'ready', + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + + CONSTRAINT attachments_id_chat_uk UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_uk UNIQUE (object_key), + CONSTRAINT attachments_owner_membership_fk + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id) ON DELETE RESTRICT, + CONSTRAINT attachments_content_type_chk + CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_chk + CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_chk + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status_chk CHECK (status = 'ready'), + CONSTRAINT attachments_filename_chk + CHECK (char_length(filename) BETWEEN 1 AND 255) +); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL, + + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_uk UNIQUE (message_id, position), + CONSTRAINT message_attachments_message_fk + FOREIGN KEY (message_id, chat_id) + REFERENCES messages(id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk + FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments(id, chat_id) ON DELETE RESTRICT, + CONSTRAINT message_attachments_position_chk CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql b/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql new file mode 100644 index 0000000..46525a6 --- /dev/null +++ b/src/main/resources/db/migration/V5__add_attachment_processing_jobs.sql @@ -0,0 +1,57 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk; +ALTER TABLE attachments + ADD COLUMN staging_key text, + ADD COLUMN error_code varchar(64), + ADD CONSTRAINT attachments_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD CONSTRAINT attachments_error_state_chk + CHECK ((status = 'failed' AND error_code IS NOT NULL) + OR (status <> 'failed' AND error_code IS NULL)); + +CREATE UNIQUE INDEX attachments_staging_key_uk + ON attachments (staging_key) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name varchar(32) NOT NULL, + content_type varchar(32) NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 char(64) NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL, + + CONSTRAINT attachment_variants_pk + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_uk UNIQUE (object_key), + CONSTRAINT attachment_variants_name_chk + CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type_chk + CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + CONSTRAINT attachment_variants_size_chk CHECK (size_bytes > 0), + CONSTRAINT attachment_variants_dimensions_chk CHECK (width > 0 AND height > 0), + CONSTRAINT attachment_variants_sha256_chk CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 1, + status varchar(16) NOT NULL DEFAULT 'queued', + published boolean NOT NULL DEFAULT false, + last_error_code varchar(64), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + + CONSTRAINT attachment_jobs_pipeline_chk CHECK (pipeline_version > 0), + CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt > 0), + CONSTRAINT attachment_jobs_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed')) +); + +CREATE INDEX attachment_jobs_publish_idx + ON attachment_jobs (published, status, created_at) + WHERE published = false AND status = 'queued'; diff --git a/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql b/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql new file mode 100644 index 0000000..1a8539c --- /dev/null +++ b/src/main/resources/db/migration/V6__add_job_leases_replay_and_fault_injection.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachment_jobs + ADD COLUMN next_attempt_at timestamptz, + ADD COLUMN lease_token uuid, + ADD COLUMN lease_until timestamptz, + ADD COLUMN publish_fault_injected boolean NOT NULL DEFAULT false, + ADD COLUMN result_fault_injected boolean NOT NULL DEFAULT false; + +UPDATE attachment_jobs SET next_attempt_at = created_at WHERE next_attempt_at IS NULL; +UPDATE attachment_jobs +SET status = 'queued', published = false, last_error_code = 'lease_migration_recovery' +WHERE status = 'processing'; +UPDATE attachments SET status = 'queued', error_code = NULL WHERE status = 'processing'; +ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET NOT NULL; +ALTER TABLE attachment_jobs ALTER COLUMN next_attempt_at SET DEFAULT clock_timestamp(); + +ALTER TABLE attachment_jobs + ADD CONSTRAINT attachment_jobs_lease_chk CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_until IS NOT NULL) + OR (status <> 'processing' AND lease_token IS NULL AND lease_until IS NULL) + ); + +DROP INDEX attachment_jobs_publish_idx; +CREATE INDEX attachment_jobs_publish_idx + ON attachment_jobs (next_attempt_at, created_at, id) + WHERE published = false AND status = 'queued'; +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until) + WHERE status = 'processing'; + +CREATE TABLE attachment_job_replays ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT, + requested_by varchar(128) NOT NULL, + previous_attempt integer NOT NULL, + requested_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_job_replays_actor_chk + CHECK (char_length(requested_by) BETWEEN 1 AND 128) +); + +CREATE INDEX attachment_job_replays_job_idx + ON attachment_job_replays (job_id, requested_at); diff --git a/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql b/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql new file mode 100644 index 0000000..dd50c74 --- /dev/null +++ b/src/main/resources/db/migration/V7__versioned_pipeline_and_fault_points.sql @@ -0,0 +1,33 @@ +ALTER TABLE attachments + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD CONSTRAINT attachments_pipeline_version_chk CHECK (pipeline_version > 0); + +ALTER TABLE attachment_jobs + DROP CONSTRAINT attachment_jobs_attachment_id_key, + DROP CONSTRAINT attachment_jobs_status_chk, + ADD COLUMN before_effect_fault_injected boolean NOT NULL DEFAULT false, + ADD COLUMN after_effect_fault_injected boolean NOT NULL DEFAULT false, + ADD CONSTRAINT attachment_jobs_status_chk + CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')); + +CREATE UNIQUE INDEX attachment_jobs_attachment_pipeline_uk + ON attachment_jobs (attachment_id, pipeline_version); + +CREATE TABLE attachment_pipeline_promotions ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + from_version integer NOT NULL, + to_version integer NOT NULL, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE RESTRICT, + requested_by varchar(128) NOT NULL, + requested_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_pipeline_promotions_versions_chk + CHECK (from_version > 0 AND to_version > from_version), + CONSTRAINT attachment_pipeline_promotions_actor_chk + CHECK (char_length(requested_by) BETWEEN 1 AND 128), + CONSTRAINT attachment_pipeline_promotions_target_uk + UNIQUE (attachment_id, to_version) +); + +CREATE INDEX attachment_pipeline_promotions_attachment_idx + ON attachment_pipeline_promotions (attachment_id, requested_at); diff --git a/src/main/resources/db/migration/V8__persist_job_trace_context.sql b/src/main/resources/db/migration/V8__persist_job_trace_context.sql new file mode 100644 index 0000000..1e7c6f8 --- /dev/null +++ b/src/main/resources/db/migration/V8__persist_job_trace_context.sql @@ -0,0 +1,12 @@ +ALTER TABLE attachment_jobs + ADD COLUMN trace_parent varchar(55), + ADD COLUMN trace_state text, + ADD CONSTRAINT attachment_jobs_trace_parent_chk CHECK ( + trace_parent IS NULL + OR trace_parent ~ '^[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$' + ); + +COMMENT ON COLUMN attachment_jobs.trace_parent IS + 'W3C traceparent used only to continue a durable retry trace'; +COMMENT ON COLUMN attachment_jobs.trace_state IS + 'W3C tracestate used only to continue a durable retry trace'; diff --git a/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java b/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java new file mode 100644 index 0000000..6a71efe --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java @@ -0,0 +1,368 @@ +package com.example.mtusi_messenger; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest +@AutoConfigureMockMvc +class MtusiMessengerApplicationTests { + + private static final String USERS_URL = "/api/v1/users"; + private static final String CHATS_URL = "/api/v1/chats"; + private static final String SESSIONS_URL = "/api/v1/auth/sessions"; + private static final String TEST_PASSWORD = "correct horse battery staple"; + + @Autowired + private MockMvc mockMvc; + + @Autowired + private ObjectMapper objectMapper; + + private User alice; + private User bob; + private User eve; + + @BeforeEach + void setUp() throws Exception { + String suffix = UUID.randomUUID() + .toString() + .replace("-", "") + .substring(0, 12); + + alice = registerUser("alice_" + suffix, "Alice"); + bob = registerUser("bob_" + suffix, "Bob"); + eve = registerUser("eve_" + suffix, "Eve"); + } + + @Test + void shouldReturn400WhenJsonIsMalformed() throws Exception { + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content("{ \"username\": ")) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn400WhenPasswordContainsOnlyWhitespace() throws Exception { + String requestBody = objectMapper.writeValueAsString( + Map.of( + "username", "blank_password_" + UUID.randomUUID().toString().substring(0, 8), + "display_name", "Blank password probe", + "password", " " + ) + ); + + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldNotAllowNonMemberToReadChatMessages() throws Exception { + String chatId = createChat(alice, bob); + + mockMvc.perform(get(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(eve)) + .param("limit", "10")) + .andExpect(status().isNotFound()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("not_found")); + } + + @Test + void shouldNotAllowNonMemberToSendMessage() throws Exception { + String chatId = createChat(alice, bob); + + String requestBody = """ + { + "text": "Hello from Eve" + } + """; + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(eve)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isNotFound()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("not_found")); + } + + @Test + void shouldReturn400WhenMessageTextIsEmpty() throws Exception { + String chatId = createChat(alice, bob); + + String requestBody = """ + { + "text": "" + } + """; + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn400WhenMessageTextIsMissing() throws Exception { + String chatId = createChat(alice, bob); + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn409WhenUsernameAlreadyExists() throws Exception { + String requestBody = """ + { + "username": "%s", + "display_name": "Another Alice", + "password": "correct horse battery staple" + } + """.formatted(alice.username()); + + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isConflict()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("conflict")); + } + + @Test + void shouldReturnAllMessagesAcrossMultiplePagesWithoutDuplicates() + throws Exception { + + String chatId = createChat(alice, bob); + + int messageCount = 5; + int pageSize = 2; + int expectedPageCount = 3; + + List expectedTexts = new ArrayList<>(); + + for (int i = 1; i <= messageCount; i++) { + String text = "Message " + i; + sendMessage(chatId, alice, text); + expectedTexts.add(text); + } + + List actualIds = new ArrayList<>(); + List actualTexts = new ArrayList<>(); + + String cursor = null; + int pageCount = 0; + + while (true) { + var request = get(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .param("limit", String.valueOf(pageSize)); + + if (cursor != null) { + request.param("cursor", cursor); + } + + MvcResult result = mockMvc.perform(request) + .andExpect(status().isOk()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.items").isArray()) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + JsonNode items = response.get("items"); + + assertNotNull(items); + assertTrue(items.size() <= pageSize); + + for (JsonNode item : items) { + actualIds.add(item.get("id").asText()); + actualTexts.add(item.get("text").asText()); + } + + pageCount++; + + JsonNode nextCursor = response.get("next_cursor"); + + if (nextCursor.isNull()) { + break; + } + + cursor = nextCursor.asText(); + + assertFalse(cursor.isBlank(), + "next_cursor must not be blank when more pages exist"); + + assertTrue(pageCount <= messageCount, + "Pagination did not terminate"); + } + + assertEquals(expectedPageCount, pageCount); + + assertEquals(messageCount, actualIds.size()); + assertEquals(messageCount, actualTexts.size()); + + assertEquals(actualIds.size(), new HashSet<>(actualIds).size()); + + assertEquals(expectedTexts, actualTexts); + } + + @Test + void shouldIgnoreXUserIdForProtectedEndpoints() throws Exception { + mockMvc.perform(get(USERS_URL + "/me") + .header("X-User-Id", alice.id())) + .andExpect(status().isUnauthorized()) + .andExpect(header().string("WWW-Authenticate", "Bearer realm=\"messenger\"")); + } + + @Test + void shouldRevokeCurrentSession() throws Exception { + mockMvc.perform(delete(SESSIONS_URL + "/current") + .header("Authorization", bearer(alice))) + .andExpect(status().isNoContent()); + + mockMvc.perform(get(USERS_URL + "/me") + .header("Authorization", bearer(alice))) + .andExpect(status().isUnauthorized()) + .andExpect(header().string("WWW-Authenticate", "Bearer realm=\"messenger\"")); + } + + @Test + void shouldRejectWrongPasswordWithBasicChallenge() throws Exception { + String basic = Base64.getEncoder().encodeToString( + (alice.username() + ":definitely-wrong-password") + .getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + mockMvc.perform(post(SESSIONS_URL) + .header("Authorization", "Basic " + basic)) + .andExpect(status().isUnauthorized()) + .andExpect(header().string( + "WWW-Authenticate", + "Basic realm=\"messenger\", charset=\"UTF-8\"")); + } + + private User registerUser(String username, String displayName) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + Map.of( + "username", username, + "display_name", displayName, + "password", TEST_PASSWORD + ) + ); + + MvcResult result = mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + String basic = Base64.getEncoder().encodeToString( + (username + ":" + TEST_PASSWORD).getBytes(java.nio.charset.StandardCharsets.UTF_8)); + MvcResult loginResult = mockMvc.perform(post(SESSIONS_URL) + .header("Authorization", "Basic " + basic)) + .andExpect(status().isCreated()) + .andReturn(); + JsonNode session = objectMapper.readTree(loginResult.getResponse().getContentAsString()); + + return new User( + response.get("id").asText(), + response.get("username").asText(), + session.get("token").asText() + ); + } + + private String createChat(User owner, User member) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + java.util.Map.of( + "title", "Test chat " + UUID.randomUUID(), + "member_ids", List.of(owner.id(), member.id()) + ) + ); + + MvcResult result = mockMvc.perform(post(CHATS_URL) + .header("Authorization", bearer(owner)) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + return response.get("id").asText(); + } + + private void sendMessage(String chatId, User sender, String text) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + java.util.Map.of("text", text) + ); + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(sender)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")); + } + + private static String bearer(User user) { + return "Bearer " + user.token(); + } + + private record User(String id, String username, String token) { + } +} diff --git a/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java b/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java new file mode 100644 index 0000000..fdc0534 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java @@ -0,0 +1,31 @@ +package com.example.mtusi_messenger.cache; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class UserCacheTest { + + @Test + void shouldEvictJsonNullAndReturnCacheMiss() { + UUID userId = UUID.randomUUID(); + String key = "messenger:user:" + userId; + StringRedisTemplate redis = mock(StringRedisTemplate.class); + @SuppressWarnings("unchecked") + ValueOperations values = mock(ValueOperations.class); + when(redis.opsForValue()).thenReturn(values); + when(values.get(key)).thenReturn("null"); + UserCache cache = new UserCache(redis, new ObjectMapper(), 300); + + assertTrue(cache.find(userId).isEmpty()); + verify(redis).delete(key); + } +} diff --git a/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java b/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java new file mode 100644 index 0000000..5ca590d --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/queue/RetryPolicyTest.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.queue; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RetryPolicyTest { + @Test + void appliesExponentialBackoffWithBoundedJitter() { + RetryPolicy policy = new RetryPolicy(Duration.ofSeconds(1), Duration.ofSeconds(30)); + + for (int attempt = 2; attempt <= 8; attempt++) { + long ceiling = Math.min(30_000L, 1_000L << (attempt - 2)); + long delay = policy.delayForAttempt(attempt).toMillis(); + assertTrue(delay >= Math.max(1, ceiling / 2)); + assertTrue(delay <= ceiling); + } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java new file mode 100644 index 0000000..49573d2 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java @@ -0,0 +1,75 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class AttachmentReconcilerTest { + private static final Instant NOW = Instant.parse("2026-11-11T12:00:00Z"); + + @Test + void deletesOnlyOrphansOlderThanGracePeriodAndReportsMissingObjects() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findAllObjectKeys()).thenReturn(Set.of( + "attachments/present/original", "attachments/missing/original")); + FakeStorage storage = new FakeStorage(List.of( + object("attachments/present/original", NOW.minusSeconds(7200)), + object("attachments/old-orphan/original", NOW.minusSeconds(7200)), + object("attachments/fresh-orphan/original", NOW.minusSeconds(30)))); + AttachmentReconciler reconciler = new AttachmentReconciler(repository, storage, + Duration.ofHours(1), false, Clock.fixed(NOW, ZoneOffset.UTC)); + + AttachmentReconciler.Report report = reconciler.reconcile(); + + assertEquals(List.of("attachments/old-orphan/original"), storage.deleted); + assertEquals(1, report.orphanObjects()); + assertEquals(1, report.deletedObjects()); + assertEquals(1, report.missingObjects()); + } + + @Test + void dryRunNeverDeletesOldOrphan() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findAllObjectKeys()).thenReturn(Set.of()); + FakeStorage storage = new FakeStorage(List.of( + object("attachments/old-orphan/original", NOW.minusSeconds(7200)))); + AttachmentReconciler reconciler = new AttachmentReconciler(repository, storage, + Duration.ofHours(1), true, Clock.fixed(NOW, ZoneOffset.UTC)); + + AttachmentReconciler.Report report = reconciler.reconcile(); + + assertEquals(List.of(), storage.deleted); + assertEquals(1, report.orphanObjects()); + assertEquals(0, report.deletedObjects()); + } + + private static ObjectStorage.StoredObject object(String key, Instant modified) { + return new ObjectStorage.StoredObject(key, modified, 100); + } + + private static final class FakeStorage implements ObjectStorage { + private final List objects; + private final List deleted = new ArrayList<>(); + + private FakeStorage(List objects) { + this.objects = objects; + } + + @Override public void put(String objectKey, byte[] bytes, String contentType) { } + @Override public byte[] get(String objectKey) { return new byte[0]; } + @Override public void delete(String objectKey) { deleted.add(objectKey); } + @Override public List list(String prefix) { return objects; } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java new file mode 100644 index 0000000..5df23c6 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java @@ -0,0 +1,201 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiException; +import com.example.mtusi_messenger.model.Attachment; +import com.example.mtusi_messenger.model.Chat; +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.queue.AttachmentJobPublisher; +import com.example.mtusi_messenger.storage.ObjectStorage; +import com.example.mtusi_messenger.storage.ObjectStorageException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.ByteArrayInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class AttachmentServiceTest { + private final UUID chatId = UUID.randomUUID(); + private final UUID ownerId = UUID.randomUUID(); + private AttachmentRepository repository; + private ChatService chats; + private InMemoryStorage storage; + private AttachmentJobPublisher publisher; + private AttachmentService service; + private byte[] png; + + @BeforeEach + void setUp() throws Exception { + repository = mock(AttachmentRepository.class); + chats = mock(ChatService.class); + storage = new InMemoryStorage(); + publisher = mock(AttachmentJobPublisher.class); + TransactionTemplate transaction = mock(TransactionTemplate.class); + when(transaction.execute(any())).thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + TransactionCallback callback = invocation.getArgument(0); + return callback.doInTransaction(mock(TransactionStatus.class)); + }); + when(chats.getAccessible(chatId, ownerId)).thenReturn( + new Chat(chatId, "test", List.of(ownerId), Instant.now())); + when(repository.saveQueued(any(), any())).thenAnswer(invocation -> invocation.getArgument(0)); + service = new AttachmentService(repository, chats, storage, publisher, transaction, + 10_485_760, 40_000_000, 3, 10_000, false, 2); + png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); + } + + @Test + void stagesValidPngAndPublishesDurableJob() { + Attachment attachment = service.upload(chatId, ownerId, "photo.png", "image/png", + png.length, new ByteArrayInputStream(png)); + + assertEquals("photo.png", attachment.filename()); + assertEquals("image/png", attachment.contentType()); + assertEquals(png.length, attachment.sizeBytes()); + assertTrue(attachment.objectKey().matches("attachments/[0-9a-f-]+/original")); + assertTrue(attachment.stagingKey().matches("staging/[0-9a-f-]+/original")); + assertEquals("queued", attachment.status()); + assertFalse(attachment.objectKey().contains("photo.png")); + assertArrayEquals(png, storage.objects.get(attachment.stagingKey())); + assertFalse(storage.objects.containsKey(attachment.objectKey())); + verify(repository).saveQueued(eq(attachment), argThat(job -> job.pipelineVersion() == 2)); + verify(publisher).publish(any()); + } + + @Test + void rejectsMimeSpoofingBeforeS3Write() { + ApiException error = assertThrows(ApiException.class, () -> service.upload( + chatId, ownerId, "photo.jpg", "image/jpeg", png.length, + new ByteArrayInputStream(png))); + + assertEquals(HttpStatus.UNSUPPORTED_MEDIA_TYPE, error.status()); + assertTrue(storage.objects.isEmpty()); + } + + @Test + void rejectsOversizedBodyBeforeReadingIt() { + ApiException error = assertThrows(ApiException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", 10_485_761, + new ByteArrayInputStream(png))); + + assertEquals(HttpStatus.PAYLOAD_TOO_LARGE, error.status()); + assertTrue(storage.objects.isEmpty()); + } + + @Test + void rejectsAttachmentFromAnotherChat() { + UUID attachmentId = UUID.randomUUID(); + when(repository.findAllByIdsInChat(chatId, List.of(attachmentId))).thenReturn(List.of()); + + ApiException error = assertThrows(ApiException.class, + () -> service.validateForMessage(chatId, List.of(attachmentId))); + + assertEquals(HttpStatus.NOT_FOUND, error.status()); + } + + @Test + void surfacesS3OutageWithoutWritingMetadata() { + storage.failure = true; + + assertThrows(ObjectStorageException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", png.length, + new ByteArrayInputStream(png))); + verify(repository, never()).saveQueued(any(), any()); + } + + @Test + void interruptedRequestDoesNotCreateS3Object() { + java.io.InputStream source = new java.io.InputStream() { + private int position; + + @Override + public int read() throws IOException { + if (position >= 16_384) { + throw new IOException("client disconnected"); + } + return png[position++]; + } + }; + + ApiException error = assertThrows(ApiException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", -1, source)); + + assertEquals(HttpStatus.BAD_REQUEST, error.status()); + assertTrue(storage.objects.isEmpty()); + verify(repository, never()).saveQueued(any(), any()); + } + + @Test + void databaseFailureCompensatesAlreadyWrittenS3Object() { + when(repository.saveQueued(any(), any())).thenThrow(new IllegalStateException("database commit failed")); + + assertThrows(IllegalStateException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", png.length, + new ByteArrayInputStream(png))); + + assertTrue(storage.objects.isEmpty(), "compensating delete must remove the orphan"); + } + + @Test + void downloadsOnlyAfterChatAuthorization() { + Attachment attachment = new Attachment(UUID.randomUUID(), chatId, ownerId, "photo.png", + "image/png", png.length, "a".repeat(64), "attachments/key/original", + Instant.now(), Instant.now()); + when(repository.findById(attachment.id())).thenReturn(Optional.of(attachment)); + storage.objects.put(attachment.objectKey(), png); + + AttachmentService.Download download = service.download(attachment.id(), ownerId); + + verify(chats).getAccessible(chatId, ownerId); + assertArrayEquals(png, download.bytes()); + } + + private static final class InMemoryStorage implements ObjectStorage { + private final Map objects = new HashMap<>(); + private boolean failure; + + @Override + public void put(String objectKey, byte[] bytes, String contentType) { + if (failure) throw new ObjectStorageException("S3 unavailable", new IOExceptionForTest()); + objects.put(objectKey, bytes.clone()); + } + + @Override + public byte[] get(String objectKey) { + if (failure) throw new ObjectStorageException("S3 unavailable", new IOExceptionForTest()); + return objects.get(objectKey).clone(); + } + + @Override + public void delete(String objectKey) { + objects.remove(objectKey); + } + + @Override + public List list(String prefix) { + return objects.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(prefix)) + .map(entry -> new StoredObject(entry.getKey(), Instant.now(), entry.getValue().length)) + .toList(); + } + } + + private static final class IOExceptionForTest extends RuntimeException { + } + +} diff --git a/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java b/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java new file mode 100644 index 0000000..03c90ef --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/service/StagingCleanerTest.java @@ -0,0 +1,55 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StagingCleanerTest { + private static final Instant NOW = Instant.parse("2026-11-18T12:00:00Z"); + + @Test + void keepsActiveAndFreshStagingButDeletesExpiredUnreferencedObject() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findActiveStagingKeys()).thenReturn(Set.of("staging/active/original")); + FakeStorage storage = new FakeStorage(List.of( + object("staging/active/original", NOW.minusSeconds(172800)), + object("staging/failed/original", NOW.minusSeconds(172800)), + object("staging/fresh/original", NOW.minusSeconds(60)))); + StagingCleaner cleaner = new StagingCleaner(repository, storage, Duration.ofDays(1), + Clock.fixed(NOW, ZoneOffset.UTC)); + + cleaner.cleanup(); + + assertEquals(List.of("staging/failed/original"), storage.deleted); + } + + private static ObjectStorage.StoredObject object(String key, Instant modified) { + return new ObjectStorage.StoredObject(key, modified, 100); + } + + private static final class FakeStorage implements ObjectStorage { + private final List objects; + private final List deleted = new ArrayList<>(); + + private FakeStorage(List objects) { + this.objects = objects; + } + + @Override public void put(String objectKey, byte[] bytes, String contentType) { } + @Override public byte[] get(String objectKey) { return new byte[0]; } + @Override public void delete(String objectKey) { deleted.add(objectKey); } + @Override public List list(String prefix) { return objects; } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java b/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java new file mode 100644 index 0000000..a8374d1 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java @@ -0,0 +1,62 @@ +package com.example.mtusi_messenger.web; + +import com.example.mtusi_messenger.service.ReadinessService; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ReadinessGuardFilterTest { + private final ReadinessService readiness = mock(ReadinessService.class); + private final ObjectMapper mapper = new ObjectMapper(); + private final ReadinessGuardFilter filter = new ReadinessGuardFilter(readiness, mapper); + + @Test + void refusingReplicaRejectsBusinessTrafficBeforeItReachesTheChain() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(false); + var request = new MockHttpServletRequest("GET", "/api/v1/users/me"); + request.setAttribute(RequestIdFilter.ATTRIBUTE, "request-1"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertEquals(503, response.getStatus()); + assertNull(chain.getRequest()); + JsonNode body = mapper.readTree(response.getContentAsByteArray()); + assertEquals("unavailable", body.at("/error/code").asText()); + assertEquals("request-1", body.at("/error/request_id").asText()); + } + + @Test + void readyReplicaPassesBusinessTrafficToTheChain() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(true); + var request = new MockHttpServletRequest("GET", "/api/v1/users/me"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertSame(request, chain.getRequest()); + } + + @Test + void healthProbeIsNeverBlockedByTheBusinessGuard() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(false); + var request = new MockHttpServletRequest("GET", "/health/ready"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertSame(request, chain.getRequest()); + } +} diff --git a/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java new file mode 100644 index 0000000..9601467 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/worker/ImageVariantProcessorTest.java @@ -0,0 +1,94 @@ +package com.example.mtusi_messenger.worker; + +import org.junit.jupiter.api.Test; + +import javax.imageio.ImageIO; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ImageVariantProcessorTest { + private final byte[] original; + + ImageVariantProcessorTest() throws Exception { + original = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); + } + + @Test + void createsThumbnailCropAndOptimizedWebp() throws Exception { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1)); + + Map variants = processor + .createVariants(original, 2).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + + assertEquals(3, variants.size()); + assertImage(variants.get("thumbnail"), "image/jpeg", 256, 160); + assertImage(variants.get("crop"), "image/jpeg", 128, 128); + assertImage(variants.get("optimized"), "image/webp", 320, 200); + byte[] webp = variants.get("optimized").bytes(); + assertEquals("RIFF", new String(webp, 0, 4, StandardCharsets.US_ASCII)); + assertEquals("WEBP", new String(webp, 8, 4, StandardCharsets.US_ASCII)); + } + + @Test + void rejectsImageWhoseEstimateExceedsWorkerMemoryBudget() { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 1L * 1024 * 1024, Duration.ofSeconds(1)); + + var error = assertThrows(ImageVariantProcessor.ProcessingException.class, + () -> processor.createVariants(original, 2)); + + assertTrue(error.permanent()); + assertEquals("image_too_large", error.errorCode()); + } + + @Test + void versionsProduceSeparateDeterministicProfiles() { + ImageVariantProcessor processor = new ImageVariantProcessor( + 40_000_000, 512L * 1024 * 1024, Duration.ofSeconds(1)); + + Map version1 = processor + .createVariants(original, 1).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + Map version2 = processor + .createVariants(original, 2).stream() + .collect(Collectors.toMap(ImageVariantProcessor.GeneratedVariant::name, + Function.identity())); + + assertFalse(Arrays.equals(version1.get("optimized").bytes(), + version2.get("optimized").bytes())); + assertTrue(Arrays.equals(version2.get("optimized").bytes(), + processor.createVariants(original, 2).stream() + .filter(item -> item.name().equals("optimized")) + .findFirst().orElseThrow().bytes())); + } + + private static void assertImage( + ImageVariantProcessor.GeneratedVariant variant, + String contentType, int width, int height) throws Exception { + assertNotNull(variant); + assertEquals(contentType, variant.contentType()); + assertEquals(width, variant.width()); + assertEquals(height, variant.height()); + var decoded = ImageIO.read(new ByteArrayInputStream(variant.bytes())); + assertNotNull(decoded); + assertEquals(width, decoded.getWidth()); + assertEquals(height, decoded.getHeight()); + } +} diff --git a/student_tests/README.md b/student_tests/README.md new file mode 100644 index 0000000..0bd7b29 --- /dev/null +++ b/student_tests/README.md @@ -0,0 +1,29 @@ +# Student-owned tests + +This directory contains additional experiments written for this implementation. +The course-provided black-box suite remains in `tests/`; Spring tests remain in +`src/test/java`. + +- `idempotency_concurrent.py` — one key through multiple API replicas. +- `readiness_gate.py` — both replicas become unhealthy after the required + schema table disappears; readiness and business traffic must return 503, + then recover after the table is restored. +- `failover_each.py` — sequential process failover of both Compose replicas. +- `rolling_update.py` — sequential replacement under write load. +- `node_failover.py` — hard loss of one worker in the disposable three-node + Swarm lab, task relocation and confirmed-data verification. +- `lab4_negative.py` — oversized/corrupt/MIME-spoofed uploads, unsafe filename, + unknown attachment, foreign user and cross-chat binding; every user error is 4xx. +- `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and + aggregate API container memory sampling through `docker stats` (lab5 `202` + upload contract). +- `lab5_worker_recovery.py` — durable worker stop/restart, all three variants and + idempotent duplicate delivery. +- `lab5_fault_injection.py` — crashes after outbox commit and after result commit + but before consumer acknowledgement. +- `lab5_dlq_replay_lease.py` — permanent failure DLQ routing, guarded audited + replay, and expired processing-lease recovery. +- `lab5_pipeline_idempotency.py` — versioned pipeline v1/v2, audited promotion, + `superseded` fencing, duplicate/reordered/stale deliveries, and crashes before + and after the S3 side effect. The script prints invariant checks and a final + `PASS` line; it uses `.env.example` for the disposable test stand by default. diff --git a/student_tests/concurrent_upload_memory.py b/student_tests/concurrent_upload_memory.py new file mode 100644 index 0000000..ad17b23 --- /dev/null +++ b/student_tests/concurrent_upload_memory.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run 10+ simultaneous valid PNG uploads and sample aggregate API container RAM.""" +import argparse +import binascii +import concurrent.futures +import json +import random +import re +import struct +import subprocess +import sys +import threading +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run # noqa: E402 + + +def png_chunk(kind, payload): + return (struct.pack('>I', len(payload)) + kind + payload + + struct.pack('>I', binascii.crc32(kind + payload) & 0xffffffff)) + + +def make_png(width=1800, height=1200): + rng = random.Random(20261111) + rows = bytearray() + for _ in range(height): + rows.append(0) + rows.extend(rng.randbytes(width * 3)) + return (b'\x89PNG\r\n\x1a\n' + + png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + png_chunk(b'IDAT', zlib.compress(bytes(rows), 1)) + + png_chunk(b'IEND', b'')) + + +def memory_bytes(value): + match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)', value.strip()) + if not match: + raise ValueError('unrecognized Docker memory value: ' + value) + number, unit = match.groups() + factors = {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000**2, + 'MiB': 1024**2, 'GB': 1000**3, 'GiB': 1024**3} + return int(float(number) * factors[unit]) + + +def api_containers(env_file): + result = subprocess.run( + ['docker', 'compose', '--env-file', env_file, 'ps', '-q', 'api'], + cwd=ROOT, check=True, capture_output=True, text=True) + ids = [line for line in result.stdout.splitlines() if line] + check(len(ids) >= 2, 'memory experiment requires at least two API containers') + return ids + + +def sample_total_memory(container_ids): + result = subprocess.run( + ['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids], + check=True, capture_output=True, text=True) + usages = [] + for line in result.stdout.splitlines(): + if line.strip(): + usages.append(memory_bytes(line.split('/', 1)[0].strip())) + check(len(usages) == len(container_ids), 'docker stats did not return every API container') + return sum(usages) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-env-file', default='.env') + parser.add_argument('--concurrency', type=int, default=12) + parser.add_argument('--output', default='evidence/upload-memory.json') + args = parser.parse_args() + check(args.concurrency >= 10, 'experiment requires concurrency >= 10') + + bootstrap = Client(args.base_url, timeout=30) + alice, bob, _, chat = bootstrap.bootstrap() + payload = make_png() + check(len(payload) <= 10_485_760, 'generated PNG exceeds upload limit') + containers = api_containers(args.compose_env_file) + baseline = sample_total_memory(containers) + peak = baseline + samples = [] + stop = threading.Event() + + def monitor(): + nonlocal peak + while not stop.is_set(): + try: + current = sample_total_memory(containers) + samples.append(current) + peak = max(peak, current) + except (subprocess.SubprocessError, Failure, ValueError): + pass + stop.wait(0.1) + + barrier = threading.Barrier(args.concurrency) + + def upload(index): + client = Client(args.base_url, timeout=30) + barrier.wait(timeout=30) + path = (f"/api/v1/chats/{chat['id']}/attachments" + f"?filename=memory-{index}.png") + return client.call('POST', path, 202, actor=alice, raw=payload, + headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id'] + + monitor_thread = threading.Thread(target=monitor, daemon=True) + monitor_thread.start() + started = time.monotonic() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + ids = list(pool.map(upload, range(args.concurrency))) + finally: + stop.set() + monitor_thread.join(timeout=5) + duration = time.monotonic() - started + check(len(ids) == args.concurrency and len(set(ids)) == len(ids), + 'not every concurrent upload completed exactly once') + + result = { + 'concurrency': args.concurrency, + 'successful_uploads': len(ids), + 'payload_bytes_each': len(payload), + 'duration_seconds': round(duration, 3), + 'api_replicas': len(containers), + 'memory_samples': len(samples), + 'baseline_api_memory_bytes': baseline, + 'peak_api_memory_bytes': peak, + 'peak_increase_bytes': max(0, peak - baseline), + 'configured_max_concurrent_uploads_per_replica': 3, + 'configured_max_upload_bytes': 10_485_760, + 'configured_max_image_pixels': 40_000_000, + } + output = ROOT / args.output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print(json.dumps(result, ensure_ascii=False)) + print('PASS: concurrent upload memory experiment') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/failover_each.py b/student_tests/failover_each.py new file mode 100644 index 0000000..996f072 --- /dev/null +++ b/student_tests/failover_each.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Stop every observed API replica in turn and record recovery metrics.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def executable(path, label): + value = Path(path).expanduser().resolve() + check(value.is_file() and os.access(value, os.X_OK), label + ' must be executable') + return value + + +def observe_instances(client, actor, attempts=60): + result = set() + for _ in range(attempts): + result.add(client.call( + 'GET', '/api/v1/users/me', actor=actor, schema='User' + )['headers']['X-Instance-Id']) + return result + + +def message_history(client, actor, chat_id): + path = '/api/v1/chats/' + chat_id + '/messages?limit=100' + result = [] + for _ in range(1000): + page = client.call( + 'GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + if page['next_cursor'] is None: + return result + path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor=' + + quote(page['next_cursor'], safe='')) + raise Failure('history pagination exceeded safety bound') + + +def main(): + def configure(parser): + parser.add_argument('--action-script', required=True) + parser.add_argument('--restore-script', required=True) + parser.add_argument('--window', type=float, default=15) + parser.add_argument('--output', default=None) + args = arguments('Sequential failover of every API replica', configure) + check(5 <= args.window <= 60, 'window must be 5..60 seconds') + action = executable(args.action_script, 'action script') + restore = executable(args.restore_script, 'restore script') + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + targets = sorted(observe_instances(client, alice)) + check(len(targets) >= 2, 'expected at least two API replicas') + targets = targets[:2] + summaries = [] + + for target in targets: + check(target in observe_instances(client, alice), + 'target replica is not observable before stop') + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + process = subprocess.Popen( + [str(action)], env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + started = time.monotonic() + first_recovery = None + reads = writes = failures = 0 + confirmed = [] + key = uuid.uuid4().hex + text = 'failover-each-' + target + '-' + key + + while time.monotonic() - started < args.window: + try: + if (reads + writes) % 2 == 0: + response = client.message(alice, chat, text, key=key) + confirmed.append(response['json']) + writes += 1 + key = uuid.uuid4().hex + text = 'failover-each-' + target + '-' + key + else: + client.call('GET', '/api/v1/users/me', actor=bob, schema='User') + reads += 1 + if process.poll() == 0 and first_recovery is None: + first_recovery = time.monotonic() - started + except Failure: + failures += 1 + time.sleep(0.05) + + check(process.wait(timeout=5) == 0, 'stop-one action failed') + elapsed = time.monotonic() - started + check(first_recovery is not None and first_recovery <= 10, + 'recovery exceeded 10 seconds') + history = message_history(client, bob, chat['id']) + history_ids = {message['id'] for message in history} + check(all(message['id'] in history_ids for message in confirmed), + 'confirmed message disappeared after failover') + + summaries.append({ + 'target_instance_id': target, + 'window_seconds': round(elapsed, 3), + 'successful_reads': reads, + 'successful_writes': writes, + 'failed_attempts': failures, + 'successful_rps': round((reads + writes) / elapsed, 3), + 'first_recovery_seconds': round(first_recovery, 3), + }) + + check(subprocess.run( + [str(restore)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=100).returncode == 0, + 'restore action failed') + check(target in observe_instances(client, alice), + 'stopped replica did not return after restore') + + report = { + 'scenario': 'sequential failover of each API replica', + 'replicas': len(targets), + 'results': summaries, + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/idempotency_concurrent.py b/student_tests/idempotency_concurrent.py new file mode 100644 index 0000000..1ed1f04 --- /dev/null +++ b/student_tests/idempotency_concurrent.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Prove that one idempotency scope is shared across API replicas.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Concurrent cross-replica idempotency check') + bootstrap = Client(args.base_url, args.ca_file, timeout=5) + alice, _, _, chat = bootstrap.bootstrap() + key = uuid.uuid4().hex + text = 'Один результат для всех реплик ' + key + path = '/api/v1/chats/' + chat['id'] + '/messages' + + def send(_): + client = Client(args.base_url, args.ca_file, timeout=10) + return client.call( + 'POST', + path, + 201, + actor=alice, + body={'text': text}, + headers={'Idempotency-Key': key}, + schema='Message') + + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(send, range(32))) + + messages = [response['json'] for response in responses] + instances = { + response['headers']['X-Instance-Id'] for response in responses + } + check(all(message == messages[0] for message in messages), + 'same key returned different messages') + check(len(instances) >= 2, + 'requests did not exercise at least two API replicas') + + page = bootstrap.call( + 'GET', path + '?limit=100', actor=alice, schema='MessagePage')['json'] + matches = [item for item in page['items'] if item['id'] == messages[0]['id']] + check(len(matches) == 1, 'idempotent message is missing or duplicated') + print('PASS: 32 concurrent requests used ' + str(len(instances)) + + ' replicas and created one message') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/lab4_negative.py b/student_tests/lab4_negative.py new file mode 100644 index 0000000..ae28543 --- /dev/null +++ b/student_tests/lab4_negative.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Negative black-box checks for lab4; uses only synthetic users and files.""" +import argparse +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) + +from client import Client, Failure, check, run # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--expect-s3-unavailable', action='store_true') + args = parser.parse_args() + + client = Client(args.base_url, timeout=20) + alice, bob, eve, chat = client.bootstrap() + png = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() + upload_path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png' + + if args.expect_s3_unavailable: + response = client.call('POST', upload_path, 503, actor=alice, raw=png, + headers={'Content-Type': 'image/png'}) + check(response['json']['error']['code'] == 'unavailable', 'S3 outage must return unavailable') + print('PASS: unavailable S3 returns contract JSON 503') + return + + attachment, original = client.attachment(alice, chat) + client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve) + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 404, actor=eve) + + other_chat = client.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]}, + schema='Chat')['json'] + client.call('POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404, + actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, + body={'text': '', 'attachment_ids': [attachment['id']]}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 404, + actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, + body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 400, + actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, + body={'text': '', 'attachment_ids': [attachment['id'], attachment['id']]}) + + client.call('POST', upload_path, 415, actor=alice, raw=png, + headers={'Content-Type': 'image/jpeg'}) + client.call('POST', upload_path, 415, actor=alice, raw=b'\x89PNG\r\n\x1a\ntruncated', + headers={'Content-Type': 'image/png'}) + client.call('POST', upload_path, 413, actor=alice, raw=b'x' * (10_485_760 + 1), + headers={'Content-Type': 'image/png'}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../escape.png', + 400, actor=alice, raw=png, headers={'Content-Type': 'image/png'}) + + downloaded = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', + actor=bob) + check(downloaded['bytes'] == original, 'authorized download changed bytes') + print('PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/lab5_dlq_replay_lease.py b/student_tests/lab5_dlq_replay_lease.py new file mode 100644 index 0000000..b36b969 --- /dev/null +++ b/student_tests/lab5_dlq_replay_lease.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Exercise DLQ routing, guarded replay and expired processing leases.""" + +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, text=True, + capture_output=capture) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def queue_ready(queue): + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", capture=True) + for line in listing.splitlines(): + columns = line.split("\t") + if len(columns) == 2 and columns[0] == queue: + return int(columns[1]) + raise Failure("queue not found: " + queue) + + +def wait_ready(client, actor, attachment, seconds=120): + return client.wait_ready(actor, attachment, wait_seconds=seconds) + + +def replay(job_id): + subprocess.run( + ["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "replay-failed.ps1"), + "-JobId", job_id, "-RequestedBy", "integration-test", + "-EnvFile", ".env.example"], + cwd=ROOT, check=True, text=True) + + +def main(): + client = Client("http://localhost:8080") + alice, bob, _, chat = client.bootstrap() + + # A permanently malformed delivery is rejected and dead-lettered. + before_dlq = queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload={}", + 'properties={"delivery_mode":2,"content_type":"application/json"}') + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) <= before_dlq: + time.sleep(0.5) + check(queue_ready(env_value("ATTACHMENT_DLQ_QUEUE")) == before_dlq + 1, + "malformed permanent failure was not dead-lettered exactly once") + + # Replay is allowed only from failed and creates one audit record. + command("stop", "worker") + failed, _ = client.attachment(alice, chat) + job_id = database_scalar( + "SELECT id FROM attachment_jobs WHERE attachment_id = '" + failed["id"] + "';") + database_scalar( + "UPDATE attachment_jobs SET status='failed', published=true, " + "last_error_code='retry_exhausted', lease_token=NULL, lease_until=NULL " + "WHERE id='" + job_id + "'; " + "UPDATE attachments SET status='failed', error_code='retry_exhausted' " + "WHERE id='" + failed["id"] + "'; SELECT 1;") + replay(job_id) + replay(job_id) + audit_count = database_scalar( + "SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';") + check(audit_count == "1", "replay was not guarded or was audited more than once") + command("start", "worker") + ready = wait_ready(client, bob, failed) + check(ready["status"] == "ready", "replayed failed job did not become ready") + replay(job_id) + check(database_scalar( + "SELECT count(*) FROM attachment_job_replays WHERE job_id='" + job_id + "';") == "1", + "replay changed an already successful job") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + failed["id"] + "';") == "3", + "replay did not preserve exactly one variant set") + + # The API reaper returns an expired processing lease to the durable outbox. + command("stop", "worker") + leased, _ = client.attachment(alice, chat) + lease_job = database_scalar( + "SELECT id FROM attachment_jobs WHERE attachment_id='" + leased["id"] + "';") + database_scalar( + "UPDATE attachment_jobs SET status='processing', attempt=1, published=true, " + "lease_token=gen_random_uuid(), lease_until=clock_timestamp()-interval '1 second' " + "WHERE id='" + lease_job + "'; " + "UPDATE attachments SET status='processing' WHERE id='" + leased["id"] + "'; SELECT 1;") + command("start", "worker") + lease_ready = wait_ready(client, bob, leased) + check(lease_ready["status"] == "ready", "expired processing lease was not recovered") + check(int(database_scalar( + "SELECT attempt FROM attachment_jobs WHERE id='" + lease_job + "';")) >= 2, + "expired lease did not consume a bounded retry attempt") + print("PASS: DLQ, guarded replay and expired lease recovery") + + +if __name__ == "__main__": + run(main) diff --git a/student_tests/lab5_fault_injection.py b/student_tests/lab5_fault_injection.py new file mode 100644 index 0000000..73c0be5 --- /dev/null +++ b/student_tests/lab5_fault_injection.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Prove recovery across DB-commit/publish and result-commit/ack crashes.""" + +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False, environment=None): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, text=True, + capture_output=capture, env=environment) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def recreate(service, fault_name=None): + environment = os.environ.copy() + environment["FAULT_CRASH_AFTER_OUTBOX_COMMIT"] = "false" + environment["FAULT_CRASH_AFTER_RESULT_COMMIT"] = "false" + if fault_name: + environment[fault_name] = "true" + command("up", "-d", "--no-deps", "--force-recreate", service, + environment=environment) + + +def wait_http(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready") + + +def wait_job(attachment_id, expected="ready", seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + status = database_scalar( + "SELECT status FROM attachment_jobs WHERE attachment_id = '" + + attachment_id + "';") + if status == expected: + return + if status == "failed": + raise Failure("fault-injected job reached failed") + time.sleep(1) + raise Failure("fault-injected job did not reach " + expected) + + +def latest_attachment(chat_id, fault_column): + row = database_scalar( + "SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + "WHERE a.chat_id = '" + chat_id + "' AND j." + fault_column + + " = true ORDER BY a.created_at DESC LIMIT 1;") + check(bool(row), "fault marker was not persisted") + return row + + +def main(): + client = Client("http://localhost:8080") + wait_http(client) + alice, bob, _, chat = client.bootstrap() + original = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + # Crash exactly after attachment + outbox COMMIT, before the request thread publishes. + recreate("api", "FAULT_CRASH_AFTER_OUTBOX_COMMIT") + wait_http(client) + try: + client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=outbox.png", + 202, actor=alice, raw=original, headers={"Content-Type": "image/png"}, + schema="Attachment") + raise Failure("outbox fault did not terminate the API request") + except Failure as error: + if "outbox fault did not" in str(error): + raise + + outbox_attachment = latest_attachment(chat["id"], "publish_fault_injected") + wait_job(outbox_attachment) + recreate("api") + wait_http(client) + recovered = client.call( + "GET", "/api/v1/attachments/" + outbox_attachment, + actor=bob, schema="Attachment")["json"] + check(recovered["status"] == "ready", "outbox recovery did not publish the job") + check(len(recovered["variants"]) == 3, "outbox recovery did not finish all variants") + + # Crash after S3 effects + result COMMIT, before RabbitMQ consumer ack. + recreate("worker", "FAULT_CRASH_AFTER_RESULT_COMMIT") + queued = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=ack-window.png", + 202, actor=alice, raw=original, headers={"Content-Type": "image/png"}, + schema="Attachment")["json"] + wait_job(queued["id"]) + ready = client.wait_ready(bob, queued, wait_seconds=120) + check(len(ready["variants"]) == 3, "result crash lost variants") + result_marker = database_scalar( + "SELECT result_fault_injected FROM attachment_jobs WHERE attachment_id = '" + + queued["id"] + "';") + check(result_marker == "t", "result commit fault marker was not persisted") + recreate("worker") + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + queues = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if "attachments.process.level4.v1\t0\t0" in queues: + break + time.sleep(1) + variant_count = database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id = '" + + queued["id"] + "';") + check(variant_count == "3", "redelivery duplicated committed variants") + print("PASS: outbox commit/publish crash and result commit/ack crash recovered") + + +if __name__ == "__main__": + run(main) diff --git a/student_tests/lab5_pipeline_idempotency.py b/student_tests/lab5_pipeline_idempotency.py new file mode 100644 index 0000000..7532524 --- /dev/null +++ b/student_tests/lab5_pipeline_idempotency.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +"""Prove version fencing and idempotent convergence under duplicate/crash faults.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] +FAULTS = ( + "FAULT_CRASH_AFTER_OUTBOX_COMMIT", + "FAULT_CRASH_BEFORE_SIDE_EFFECT", + "FAULT_CRASH_AFTER_SIDE_EFFECT", + "FAULT_CRASH_AFTER_RESULT_COMMIT", +) + + +def command(*args, capture=False, environment=None, check_result=True): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, text=True, capture_output=capture, + env=environment, check=check_result) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def compose_environment(**overrides): + environment = os.environ.copy() + for name in FAULTS: + environment[name] = "false" + environment.update({name: str(value) for name, value in overrides.items()}) + return environment + + +def recreate(service, **overrides): + command( + "up", "-d", "--no-deps", "--force-recreate", service, + environment=compose_environment(**overrides)) + + +def wait_http(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready") + + +def wait_database(sql, expected, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if database_scalar(sql) == expected: + return + time.sleep(0.5) + raise Failure("database condition did not become " + expected + ": " + sql) + + +def job_row(attachment_id, version): + row = database_scalar( + "SELECT id || '|' || attempt || '|' || " + "to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + check(bool(row), "pipeline job is missing") + return row.split("|") + + +def publish(attachment_id, version): + job_id, attempt, created_at = job_row(attachment_id, version) + payload = json.dumps({ + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": version, + "attempt": int(attempt), + "created_at": created_at, + }, separators=(",", ":")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload=" + payload, + 'properties={"delivery_mode":2,"content_type":"application/json"}') + + +def promote(attachment_id, version): + subprocess.run( + ["powershell", "-NoProfile", "-File", str(ROOT / "scripts" / "promote-pipeline.ps1"), + "-AttachmentId", attachment_id, "-PipelineVersion", str(version), + "-RequestedBy", "idempotency-test", "-EnvFile", ".env.example"], + cwd=ROOT, check=True, text=True) + + +def variant_snapshot(attachment_id, version): + return database_scalar( + "SELECT string_agg(name || ':' || sha256 || ':' || object_key, ',' ORDER BY name) " + "FROM attachment_variants WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + + +def job_status(attachment_id, version): + return database_scalar( + "SELECT status || '|' || published || '|' || coalesce(last_error_code, '') " + "FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(version) + ";") + + +def variant_count(attachment_id, version): + return database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + attachment_id + "' AND pipeline_version=" + str(version) + ";") + + +def object_count(attachment_id): + script = ( + 'mc alias set local http://minio:9000 "$MINIO_ROOT_USER" ' + '"$MINIO_ROOT_PASSWORD" >/dev/null && ' + 'mc ls --recursive "local/$S3_BUCKET/attachments/' + attachment_id + + '/" 2>/dev/null | wc -l') + output = command( + "run", "--rm", "--no-deps", "--entrypoint", "/bin/sh", + "minio-init", "-c", script, capture=True) + return int(output.splitlines()[-1]) + + +def wait_marker(attachment_id, column, seconds=60): + wait_database( + "SELECT " + column + "::text FROM attachment_jobs WHERE attachment_id='" + + attachment_id + "' ORDER BY pipeline_version DESC LIMIT 1;", "true", seconds) + + +def wait_queue_empty(seconds=150): + deadline = time.monotonic() + seconds + marker = env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0" + while time.monotonic() < deadline: + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if marker in listing: + return + time.sleep(0.5) + raise Failure("main queue did not drain") + + +def wait_worker_consumer(seconds=150): + deadline = time.monotonic() + seconds + queue = env_value("ATTACHMENT_JOB_QUEUE") + while time.monotonic() < deadline: + listing = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "consumers", capture=True) + for line in listing.splitlines(): + columns = line.split("\t") + if len(columns) == 2 and columns[0] == queue and int(columns[1]) > 0: + return + time.sleep(1) + raise Failure("worker did not register a RabbitMQ consumer") + + +def assert_ready_invariants(attachment_id, expected_version): + row = database_scalar( + "SELECT pipeline_version || '|' || status FROM attachments WHERE id='" + + attachment_id + "';") + check(row == str(expected_version) + "|ready", "active pipeline is not ready") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + attachment_id + + "' AND pipeline_version=" + str(expected_version) + ";") == "3", + "active pipeline does not have exactly three variants") + check(database_scalar( + "SELECT count(*) FROM attachment_jobs WHERE attachment_id='" + attachment_id + + "' AND status IN ('queued','processing');") == "0", + "acknowledged attachment still has non-terminal work") + + +def fault_case(client, alice, bob, chat, flag, marker, expected_objects_before_commit): + recreate( + "worker", **{flag: "true", "JOB_TIMEOUT": "5s", "JOB_LEASE_DURATION": "8s"}) + wait_worker_consumer() + attachment, _ = client.attachment(alice, chat) + wait_marker(attachment["id"], marker, seconds=150) + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + attachment["id"] + "';") == "0", + "result metadata committed before the injected pre-commit crash") + check(object_count(attachment["id"]) == expected_objects_before_commit, + "unexpected object count at injected fault point") + ready = client.wait_ready(bob, attachment, wait_seconds=150) + check(ready["status"] == "ready", "faulted job was lost") + assert_ready_invariants(attachment["id"], 2) + check(int(database_scalar( + "SELECT attempt FROM attachment_jobs WHERE attachment_id='" + + attachment["id"] + "' AND pipeline_version=2;")) >= 2, + "crashed job was not recovered through a later managed attempt") + check(object_count(attachment["id"]) == 4, + "idempotent retry did not converge to one deterministic object set") + recreate("worker") + return attachment["id"] + + +def main(): + client = Client("http://localhost:8080") + recreate("api", ATTACHMENT_PIPELINE_VERSION="1") + wait_http(client) + alice, bob, _, chat = client.bootstrap() + + # Complete v1, promote to v2, then deliver newer work before stale v1 and + # duplicate v2 messages. Arrival order intentionally differs from creation order. + versioned, _ = client.attachment(alice, chat) + client.wait_ready(bob, versioned, wait_seconds=120) + version1 = variant_snapshot(versioned["id"], 1) + check(bool(version1), "v1 result is missing") + command("stop", "worker") + promote(versioned["id"], 2) + publish(versioned["id"], 2) + publish(versioned["id"], 2) + publish(versioned["id"], 1) + command("up", "-d", "--no-deps", "worker") + client.wait_ready(bob, versioned, wait_seconds=120) + wait_queue_empty() + version2 = variant_snapshot(versioned["id"], 2) + check(bool(version2) and version2 != version1, + "v2 did not produce a distinct immutable versioned result") + check(database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id='" + + versioned["id"] + "';") == "6", "versioned rows were lost or duplicated") + stable = version2 + publish(versioned["id"], 1) + publish(versioned["id"], 2) + wait_queue_empty() + check(variant_snapshot(versioned["id"], 2) == stable, + "stale or duplicate delivery overwrote the active v2 result") + assert_ready_invariants(versioned["id"], 2) + + # Promote a still-queued v1 intent to v2. The old job must become + # superseded, and its already-durable message must be a harmless no-op. + command("stop", "worker") + superseded, _ = client.attachment(alice, chat) + check(job_status(superseded["id"], 1).startswith("queued|"), + "new upload did not start as a queued v1 job") + promote(superseded["id"], 2) + check(job_status(superseded["id"], 1) == "superseded|true|superseded_by_v2", + "queued v1 job was not fenced as superseded by v2") + check(variant_count(superseded["id"], 1) == "0", + "superseded job already has v1 result rows") + publish(superseded["id"], 1) + command("up", "-d", "--no-deps", "worker") + client.wait_ready(bob, superseded, wait_seconds=120) + wait_queue_empty() + check(variant_count(superseded["id"], 1) == "0", + "stale superseded v1 delivery created v1 result rows") + check(variant_count(superseded["id"], 2) == "3", + "promoted v2 job did not produce exactly three variants") + assert_ready_invariants(superseded["id"], 2) + + recreate("api", ATTACHMENT_PIPELINE_VERSION="2") + wait_http(client) + before_id = fault_case( + client, alice, bob, chat, "FAULT_CRASH_BEFORE_SIDE_EFFECT", + "before_effect_fault_injected", 0) + after_id = fault_case( + client, alice, bob, chat, "FAULT_CRASH_AFTER_SIDE_EFFECT", + "after_effect_fault_injected", 4) + + # Existing level-4 fault point proves redelivery after result COMMIT but before ack. + recreate("worker", FAULT_CRASH_AFTER_RESULT_COMMIT="true") + wait_worker_consumer() + committed, _ = client.attachment(alice, chat) + wait_marker(committed["id"], "result_fault_injected") + client.wait_ready(bob, committed, wait_seconds=120) + wait_queue_empty() + assert_ready_invariants(committed["id"], 2) + recreate("worker") + + confirmed = database_scalar( + "SELECT count(*) FROM attachments WHERE id IN ('" + versioned["id"] + "','" + + superseded["id"] + "','" + before_id + "','" + after_id + "','" + + committed["id"] + "') AND status='ready';") + check(confirmed == "5", "a confirmed task was lost") + print("PASS: version fencing, superseded, duplicate/reorder and " + "pre/post-side-effect crashes") + + +if __name__ == "__main__": + try: + run(main) + finally: + try: + recreate("api", ATTACHMENT_PIPELINE_VERSION="2") + recreate("worker") + except Exception: + pass diff --git a/student_tests/lab5_worker_recovery.py b/student_tests/lab5_worker_recovery.py new file mode 100644 index 0000000..dadcf1a --- /dev/null +++ b/student_tests/lab5_worker_recovery.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Exercise durable queued upload, component restarts and duplicate delivery. + +Run from the repository root against the local compose.yaml: + python student_tests/lab5_worker_recovery.py +The script preserves volumes and leaves the complete stack running. +""" + +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +COMPOSE = ["docker", "compose", "--env-file", ".env.example"] + + +def command(*args, capture=False): + result = subprocess.run( + [*COMPOSE, *args], cwd=ROOT, check=True, + text=True, capture_output=capture) + return result.stdout.strip() if capture else "" + + +def database_scalar(sql): + return command( + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, capture=True) + + +def env_value(name): + for line in (ROOT / ".env.example").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise Failure("missing " + name + " in .env.example") + + +def wait_http_ready(client, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready after restart") + + +def wait_rabbit(seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + result = subprocess.run( + [*COMPOSE, "exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping"], + cwd=ROOT, text=True, capture_output=True) + if result.returncode == 0: + return + time.sleep(2) + raise Failure("RabbitMQ did not become ready after restart") + + +def main(): + client = Client("http://localhost:8080") + command("stop", "worker") + try: + alice, bob, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload did not remain queued") + client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", + 409, actor=bob) + linked = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(linked["attachment_ids"] == [attachment["id"]], + "queued attachment could not be linked") + + command("restart", "api") + command("restart", "rabbitmq") + wait_rabbit() + wait_http_ready(client) + command("start", "worker") + + ready = client.wait_ready(bob, attachment, wait_seconds=120) + check(ready["status"] == "ready", "attachment did not become ready") + downloaded = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, + "expected thumbnail, crop and optimized variants") + check((variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160), + "thumbnail dimensions differ from contract") + check((variants["crop"]["width"], variants["crop"]["height"]) == (128, 128), + "crop dimensions differ from contract") + crop = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=crop", + actor=bob) + check(crop["headers"].get_content_type() == "image/jpeg", "crop is not JPEG") + check(jpeg_size(crop["bytes"]) == (128, 128), "crop JPEG has wrong dimensions") + optimized = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized", + actor=bob) + check(optimized["headers"].get_content_type() == "image/webp", "optimized is not WebP") + check(optimized["bytes"][:4] == b"RIFF" and optimized["bytes"][8:12] == b"WEBP", + "optimized payload has no WebP signature") + + job_row = database_scalar( + "SELECT id || '|' || pipeline_version || '|' || attempt || '|' || " + "to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM attachment_jobs WHERE attachment_id = '" + attachment["id"] + "';") + job_id, pipeline_version, attempt, created_at = job_row.split("|") + payload = json.dumps({ + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment["id"], + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at, + }, separators=(",", ":")) + command( + "exec", "-T", "rabbitmq", "rabbitmqadmin", + "-u", env_value("RABBITMQ_USER"), + "-p", env_value("RABBITMQ_PASSWORD"), + "publish", "exchange=" + env_value("ATTACHMENT_JOB_EXCHANGE"), + "routing_key=" + env_value("ATTACHMENT_JOB_ROUTING_KEY"), + "payload=" + payload, + 'properties={"delivery_mode":2,"content_type":"application/json"}') + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + queued = command( + "exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", + "name", "messages_ready", "messages_unacknowledged", capture=True) + if env_value("ATTACHMENT_JOB_QUEUE") + "\t0\t0" in queued: + break + time.sleep(0.5) + variant_count = database_scalar( + "SELECT count(*) FROM attachment_variants WHERE attachment_id = '" + + attachment["id"] + "';") + status = database_scalar( + "SELECT status FROM attachments WHERE id = '" + attachment["id"] + "';") + check(variant_count == "3", "duplicate delivery changed the three variant rows") + check(status == "ready", "duplicate delivery changed terminal ready state") + check(hashlib.sha256(downloaded["bytes"]).hexdigest() == attachment["sha256"], + "original checksum changed") + print("PASS: stopped worker, durable restart recovery and duplicate delivery") + finally: + command("start", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student_tests/node_failover.py b/student_tests/node_failover.py new file mode 100644 index 0000000..2cf00a6 --- /dev/null +++ b/student_tests/node_failover.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Hard-stop one Swarm worker and prove API task relocation and data safety.""" +import json +import atexit +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import subprocess +import sys +import time +import uuid +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def command(*parts, timeout=30): + result = subprocess.run( + parts, capture_output=True, text=True, timeout=timeout) + check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3])) + return result.stdout.strip() + + +def api_tasks(manager): + output = command( + 'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api', + '--filter', 'desired-state=running', + '--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}') + result = [] + for line in output.splitlines(): + fields = line.split('\t') + if len(fields) == 3 and fields[2].startswith('Running'): + result.append({'task_id': fields[0], 'node': fields[1]}) + return result + + +def history(client, actor, chat_id): + path = '/api/v1/chats/' + chat_id + '/messages?limit=100' + result = [] + for _ in range(1000): + page = client.call('GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + if page['next_cursor'] is None: + return result + path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor=' + + quote(page['next_cursor'], safe='')) + raise Failure('history pagination exceeded safety bound') + + +def observe(client, actor, attempts=40): + def request(_): + local = Client(client.base, timeout=2) + return local.call( + 'GET', '/api/v1/users/me', actor=actor, schema='User') + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(request, range(attempts))) + return {response['headers']['X-Instance-Id'] for response in responses} + + +def restore_worker(manager, node): + subprocess.run( + ['docker', 'start', node], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30) + deadline = time.monotonic() + 90 + status = '' + while time.monotonic() < deadline: + result = subprocess.run( + ['docker', 'exec', manager, 'docker', 'node', 'ls', + '--filter', 'name=' + node, '--format', '{{.Status}}'], + capture_output=True, text=True, timeout=30) + status = result.stdout.strip() + if result.returncode == 0 and status == 'Ready': + return True + time.sleep(2) + return False + + +def main(): + def configure(parser): + parser.add_argument('--manager-container', default='hla-swarm-manager') + parser.add_argument('--window', type=float, default=30) + parser.add_argument('--output', default=None) + args = arguments('Swarm worker loss and API relocation', configure) + check(15 <= args.window <= 120, 'window must be 15..120 seconds') + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + before_instances = observe(client, alice) + check(len(before_instances) >= 2, 'expected two API instances before node loss') + before_tasks = api_tasks(args.manager_container) + check(len(before_tasks) == 2, 'expected two running API tasks') + candidates = [task for task in before_tasks + if task['node'].startswith('hla-swarm-worker')] + check(bool(candidates), 'no API task is running on a worker node') + victim = candidates[0] + confirmed_before = client.message( + alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json'] + + started = time.monotonic() + command('docker', 'stop', '--time', '1', victim['node'], timeout=15) + restore_at_exit = lambda: restore_worker(args.manager_container, victim['node']) + atexit.register(restore_at_exit) + first_recovery = None + reads = writes = failures = 0 + confirmed_during = [] + key = uuid.uuid4().hex + text = 'during-node-loss-' + key + while time.monotonic() - started < args.window: + try: + if (reads + writes) % 2 == 0: + response = client.message(alice, chat, text, key=key) + confirmed_during.append(response['json']) + writes += 1 + key = uuid.uuid4().hex + text = 'during-node-loss-' + key + else: + client.call('GET', '/api/v1/users/me', actor=bob, schema='User') + reads += 1 + if first_recovery is None: + first_recovery = time.monotonic() - started + except Failure: + failures += 1 + time.sleep(0.05) + + deadline = time.monotonic() + 90 + after_tasks = [] + while time.monotonic() < deadline: + after_tasks = api_tasks(args.manager_container) + if (len(after_tasks) == 2 + and all(task['node'] != victim['node'] for task in after_tasks) + and any(task['task_id'] != victim['task_id'] for task in after_tasks)): + break + time.sleep(2) + check(len(after_tasks) == 2, 'API service did not return to two tasks') + check(all(task['node'] != victim['node'] for task in after_tasks), + 'API task was not relocated away from failed node') + check(first_recovery is not None and first_recovery <= 10, + 'API did not recover within 10 seconds') + + stored = history(client, bob, chat['id']) + stored_ids = {message['id'] for message in stored} + check(confirmed_before['id'] in stored_ids, + 'pre-failure confirmed message was lost') + check(all(message['id'] in stored_ids for message in confirmed_during), + 'message confirmed during node loss was lost') + check(restore_worker(args.manager_container, victim['node']), + 'failed worker did not rejoin the Swarm') + atexit.unregister(restore_at_exit) + after_instances = observe(client, alice, attempts=96) + check(len(after_instances - before_instances) >= 1, + 'replacement API process was not visible through X-Instance-Id') + + elapsed = time.monotonic() - started + report = { + 'scenario': 'hard loss of one Swarm worker', + 'failed_node': victim['node'], + 'old_task_id': victim['task_id'], + 'tasks_before': before_tasks, + 'tasks_after_relocation': after_tasks, + 'instance_ids_before': sorted(before_instances), + 'instance_ids_after': sorted(after_instances), + 'successful_reads': reads, + 'successful_writes': writes, + 'failed_attempts': failures, + 'first_recovery_seconds': round(first_recovery, 3), + 'successful_rps': round((reads + writes) / args.window, 3), + 'experiment_seconds_including_rejoin': round(elapsed, 3), + 'confirmed_data_preserved': True, + 'database_node_failed': False, + 'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not ' + 'protected by this experiment.'), + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/readiness_gate.py b/student_tests/readiness_gate.py new file mode 100644 index 0000000..1a9df4c --- /dev/null +++ b/student_tests/readiness_gate.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Reproduce and guard against serving business traffic from unready APIs.""" +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + +ORIGINAL_TABLE = 'message_idempotency' +TEMPORARY_TABLE = 'message_idempotency_readiness_test' + + +def command(*args): + result = subprocess.run( + args, + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False) + if result.returncode != 0: + output = result.stdout.strip() + raise Failure('command failed: ' + ' '.join(args[:3]) + + (': ' + output if output else '')) + return result.stdout.strip() + + +def rename_table(source, target): + sql = 'ALTER TABLE ' + source + ' RENAME TO ' + target + ';' + command( + 'docker', 'compose', 'exec', '-T', 'postgres', 'sh', '-ec', + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" ' + '-d "$POSTGRES_DB" -c \'' + sql + '\'') + + +def api_container_ids(): + output = command('docker', 'compose', 'ps', '-q', 'api') + return [line.strip() for line in output.splitlines() if line.strip()] + + +def health_status(container_id): + return command( + 'docker', 'inspect', + '--format={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}', + container_id) + + +def wait_for_all_health(expected, timeout): + deadline = time.monotonic() + timeout + last = [] + while time.monotonic() < deadline: + containers = api_container_ids() + last = [health_status(container) for container in containers] + if len(containers) >= 2 and all(value == expected for value in last): + return containers + time.sleep(1) + raise Failure('API health did not become ' + expected + + '; last statuses: ' + repr(last)) + + +def configure(parser): + parser.add_argument( + '--health-timeout', type=float, default=45.0, + help='seconds to wait for both Docker health states') + + +def main(): + args = arguments('Readiness must gate business traffic through the LB', configure) + client = Client(args.base_url, args.ca_file, timeout=8) + client.call('GET', '/health/ready', 200, schema='Health') + actor = client.user('readiness_gate') + + renamed = False + try: + rename_table(ORIGINAL_TABLE, TEMPORARY_TABLE) + renamed = True + containers = wait_for_all_health('unhealthy', args.health_timeout) + + client.call('GET', '/health/ready', 503) + instances = set() + for _ in range(8): + response = client.call( + 'GET', '/api/v1/users/me', 503, actor=actor) + instances.add(response['headers']['X-Instance-Id']) + check(instances, 'no refusing API instance was observed') + finally: + if renamed: + rename_table(TEMPORARY_TABLE, ORIGINAL_TABLE) + + wait_for_all_health('healthy', args.health_timeout) + # Nginx may retain a passive failure for fail_timeout=2s. + time.sleep(3) + client.call('GET', '/health/ready', 200, schema='Health') + client.call('GET', '/api/v1/users/me', 200, actor=actor, schema='User') + print('PASS: ' + str(len(containers)) + + ' unhealthy API containers refused business traffic with 503 ' + + 'and recovered after schema restoration') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/rolling_update.py b/student_tests/rolling_update.py new file mode 100644 index 0000000..20effca --- /dev/null +++ b/student_tests/rolling_update.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Run closed-loop writes while API replicas are replaced one at a time.""" +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument('--action-script', required=True) + parser.add_argument('--duration', type=float, default=180) + parser.add_argument('--concurrency', type=int, default=4) + parser.add_argument('--output', default=None) + args = arguments('Rolling replacement under write load', configure) + check(30 <= args.duration <= 300, 'duration must be 30..300 seconds') + check(1 <= args.concurrency <= 32, 'concurrency must be 1..32') + action = Path(args.action_script).expanduser().resolve() + check(action.is_file() and os.access(action, os.X_OK), + 'action script must be executable') + bootstrap = Client(args.base_url, args.ca_file, timeout=5) + alice, _, _, chat = bootstrap.bootstrap() + barrier = threading.Barrier(args.concurrency + 1) + + def worker(worker_id): + client = Client(args.base_url, args.ca_file, timeout=5) + barrier.wait() + deadline = time.monotonic() + args.duration + latencies = [] + errors = 0 + instances = set() + index = 0 + while time.monotonic() < deadline: + started = time.monotonic() + try: + response = client.message( + alice, chat, 'rolling-%s-%s' % (worker_id, index)) + latencies.append(time.monotonic() - started) + instances.add(response['headers']['X-Instance-Id']) + except Failure: + errors += 1 + index += 1 + return latencies, errors, instances + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [pool.submit(worker, number) + for number in range(args.concurrency)] + barrier.wait() + time.sleep(3) + action_started = time.monotonic() + process = subprocess.run( + [str(action)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=args.duration - 5) + action_seconds = time.monotonic() - action_started + check(process.returncode == 0, 'rolling action failed') + results = [future.result() for future in futures] + + elapsed = time.monotonic() - started + latencies = sorted(value for values, _, _ in results for value in values) + errors = sum(value for _, value, _ in results) + instances = set().union(*(value for _, _, value in results)) + total = len(latencies) + errors + + def percentile(fraction): + if not latencies: + return None + index = max(0, math.ceil(fraction * len(latencies)) - 1) + return round(1000 * latencies[index], 3) + + report = { + 'scenario': 'sequential API replacement under closed-loop writes', + 'requested_duration_seconds': args.duration, + 'elapsed_seconds': round(elapsed, 3), + 'rolling_action_seconds': round(action_seconds, 3), + 'concurrency': args.concurrency, + 'observed_instance_ids': sorted(instances), + 'successful_requests': len(latencies), + 'failed_requests': errors, + 'error_ratio': round(errors / total, 6) if total else None, + 'successful_rps': round(len(latencies) / elapsed, 3), + 'successful_latency_ms': { + 'p50': percentile(.50), + 'p95': percentile(.95), + 'p99': percentile(.99), + }, + 'note': 'Closed-loop result on one host; success latency excludes failures.', + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + check(bool(latencies), 'no successful writes') + check(len(instances) >= 3, + 'rolling replacement did not expose replacement instance IDs') + + +if __name__ == '__main__': + run(main) diff --git a/swarm/nginx.Dockerfile b/swarm/nginx.Dockerfile new file mode 100644 index 0000000..9d74ead --- /dev/null +++ b/swarm/nginx.Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.27.4-bookworm +COPY swarm/nginx.conf /etc/nginx/nginx.conf +HEALTHCHECK --interval=2s --timeout=2s --retries=15 CMD nginx -t diff --git a/swarm/nginx.conf b/swarm/nginx.conf new file mode 100644 index 0000000..74b00a1 --- /dev/null +++ b/swarm/nginx.conf @@ -0,0 +1,86 @@ +worker_processes auto; +worker_shutdown_timeout 10s; + +events { + worker_connections 1024; +} + +http { + client_max_body_size 11m; + client_body_timeout 10s; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log warn; + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 1s; + + upstream api_backend { + zone api_backend 64k; + least_conn; + server tasks.hla-app_api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 32; + } + + server { + listen 80; + server_name _; + proxy_intercept_errors on; + error_page 502 503 504 = @no_ready_api; + + location = /nginx-health { + default_type application/json; + return 200 '{"status":"ok"}'; + } + + location @no_ready_api { + internal; + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id nginx always; + add_header Cache-Control "no-store" always; + return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}'; + } + + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 2; + } + + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + proxy_connect_timeout 500ms; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + proxy_next_upstream off; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/swarm/stack.app.yaml b/swarm/stack.app.yaml new file mode 100644 index 0000000..b5c542d --- /dev/null +++ b/swarm/stack.app.yaml @@ -0,0 +1,88 @@ +version: "3.9" + +services: + api: + image: hla-swarm-registry:5000/hla-lab6-api:swarm + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: jdbc:postgresql://hla-state_postgres:5432/${DATABASE_NAME:-messenger} + DATABASE_USER: ${DATABASE_USER:-messenger} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + DATABASE_POOL_SIZE: 10 + DATABASE_MIN_IDLE: 2 + DATABASE_QUERY_TIMEOUT_SECONDS: 5 + DATABASE_TRANSACTION_TIMEOUT_SECONDS: 5 + REDIS_HOST: hla-state_redis + REDIS_PORT: 6379 + REDIS_CONNECT_TIMEOUT: 1s + REDIS_COMMAND_TIMEOUT: 500ms + SESSION_TTL_SECONDS: 3600 + USER_CACHE_TTL_SECONDS: 300 + IDEMPOTENCY_TTL_SECONDS: 86400 + S3_ENDPOINT: http://hla-state_minio:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required} + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + MAX_CONCURRENT_UPLOADS: 3 + UPLOAD_PERMIT_WAIT_MILLIS: 10000 + ATTACHMENT_CLEANUP_GRACE_SECONDS: 3600 + ATTACHMENT_CLEANUP_DRY_RUN: "false" + ATTACHMENT_RECONCILE_INTERVAL_MILLIS: 600000 + ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: 60000 + FLYWAY_ENABLED: "false" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 45s + networks: [backend] + deploy: + replicas: 2 + endpoint_mode: dnsrr + placement: + max_replicas_per_node: 1 + restart_policy: + condition: on-failure + delay: 2s + update_config: + parallelism: 1 + order: start-first + monitor: 15s + failure_action: rollback + rollback_config: + parallelism: 1 + order: stop-first + resources: + limits: + cpus: "0.75" + memory: 768M + + nginx: + image: hla-swarm-registry:5000/hla-lab6-nginx:swarm + ports: + - target: 80 + published: 8080 + protocol: tcp + mode: ingress + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.lb == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.25" + memory: 128M + +networks: + backend: + external: true + name: hla-state_backend diff --git a/swarm/stack.state.yaml b/swarm/stack.state.yaml new file mode 100644 index 0000000..29f30e7 --- /dev/null +++ b/swarm/stack.state.yaml @@ -0,0 +1,123 @@ +version: "3.9" + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_DB: ${DATABASE_NAME:-messenger} + POSTGRES_USER: ${DATABASE_USER:-messenger} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 30 + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "1.00" + memory: 768M + + redis: + image: redis:7.4-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.25" + memory: 256M + + minio: + image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required} + volumes: + - minio-data:/bitnami/minio/data + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:9000/minio/health/live"] + interval: 2s + timeout: 2s + retries: 30 + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.75" + memory: 512M + + minio-init: + image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required} + configs: + - source: minio_api_policy + target: /policy/minio-api-policy.json + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"; do sleep 2; done + mc mb --ignore-existing "local/$$S3_BUCKET" + mc anonymous set none "local/$$S3_BUCKET" + sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json + mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" || true + mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json || true + mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID" + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + delay: 2s + max_attempts: 20 + +networks: + backend: + driver: overlay + attachable: true + +volumes: + postgres-data: + redis-data: + minio-data: + +configs: + minio_api_policy: + file: ../infra/minio-api-policy.json diff --git a/tests/attachment_load.py b/tests/attachment_load.py new file mode 100644 index 0000000..25612dc --- /dev/null +++ b/tests/attachment_load.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Create queued attachment jobs without printing credentials or message bodies.""" +import json +from pathlib import Path +import time + +from client import Client, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument('--count', type=int, default=10) + parser.add_argument('--interval', type=float, default=0.1) + parser.add_argument('--output', default=None) + + args = arguments('Synthetic attachment load for worker telemetry', configure) + check(1 <= args.count <= 1000, 'count must be 1..1000') + check(0 <= args.interval <= 60, 'interval must be 0..60 seconds') + client = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + started = time.monotonic() + accepted = 0 + errors = 0 + for _ in range(args.count): + try: + client.attachment(alice, chat) + accepted += 1 + except Exception: + errors += 1 + if args.interval: + time.sleep(args.interval) + report = { + 'workload': 'synthetic PNG attachment uploads', + 'requested': args.count, + 'accepted': accepted, + 'errors': errors, + 'elapsed_seconds': round(time.monotonic() - started, 3), + 'contains_credentials': False, + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(text + '\n', encoding='utf-8') + print(text) + check(accepted > 0, 'no attachment jobs were accepted') + + +if __name__ == '__main__': + run(main) diff --git a/tests/client.py b/tests/client.py index f0499a2..38159a0 100644 --- a/tests/client.py +++ b/tests/client.py @@ -14,7 +14,7 @@ import uuid ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) class Failure(AssertionError): pass