forked from mtusi2026/mtusi2026lab6
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe3d849f59 | ||
|
|
872a1acc0c | ||
|
|
935de06050 |
+69
-20
@@ -1,30 +1,79 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Copy to .env for a local synthetic environment. Never commit .env.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
|
||||
DATABASE_NAME=messenger
|
||||
DATABASE_USER=messenger
|
||||
DATABASE_PASSWORD=replace-with-a-long-random-database-password
|
||||
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=replace-with-a-long-random-rabbitmq-password
|
||||
RABBITMQ_VHOST=/
|
||||
ATTACHMENT_JOB_EXCHANGE=attachments.v1
|
||||
ATTACHMENT_JOB_QUEUE=attachments.process.level4.v1
|
||||
ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1
|
||||
ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1
|
||||
ATTACHMENT_DLQ_QUEUE=attachments.failed.v1
|
||||
ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1
|
||||
ATTACHMENT_PIPELINE_VERSION=2
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_RETRY_BASE_DELAY=1s
|
||||
JOB_RETRY_MAX_DELAY=30s
|
||||
JOB_TIMEOUT=60s
|
||||
JOB_LEASE_DURATION=90s
|
||||
JOB_LEASE_REAPER_INTERVAL=5s
|
||||
WORKER_MAX_IMAGE_PIXELS=40000000
|
||||
WORKER_MEMORY_BUDGET_BYTES=536870912
|
||||
WORKER_MEMORY_WAIT=2s
|
||||
|
||||
# Fault injection is opt-in and intended only for an isolated test stand.
|
||||
FAULT_CRASH_AFTER_OUTBOX_COMMIT=false
|
||||
FAULT_CRASH_BEFORE_SIDE_EFFECT=false
|
||||
FAULT_CRASH_AFTER_SIDE_EFFECT=false
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT=false
|
||||
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-api-secret
|
||||
MINIO_ROOT_USER=minio-root
|
||||
MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret
|
||||
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
MAX_CONCURRENT_UPLOADS=3
|
||||
UPLOAD_PERMIT_WAIT_MILLIS=10000
|
||||
|
||||
# Staging objects are deleted only after the final result is committed.
|
||||
# A separate lifecycle/cleanup policy may remove abandoned staging objects
|
||||
# after this retention interval; it must never remove active jobs.
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
ATTACHMENT_CLEANUP_GRACE_SECONDS=3600
|
||||
ATTACHMENT_CLEANUP_DRY_RUN=false
|
||||
ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000
|
||||
ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000
|
||||
|
||||
# Observability (lab6). The worker must set a distinct OTEL_SERVICE_NAME.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
OTEL_TRACES_SAMPLER=parentbased_traceidratio
|
||||
# Balanced lab profile: keep 25% of complete parent-based traces; keep all logs/metrics.
|
||||
OTEL_TRACES_SAMPLER_ARG=0.25
|
||||
OTEL_METRIC_EXPORT_INTERVAL=30000
|
||||
OTEL_BSP_MAX_QUEUE_SIZE=2048
|
||||
OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512
|
||||
OTEL_BSP_EXPORT_TIMEOUT=5000
|
||||
OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=256
|
||||
OBSERVABILITY_QUEUE_POLL_INTERVAL=5s
|
||||
|
||||
GRAFANA_PORT=3000
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
GRAFANA_ADMIN_PASSWORD=replace-with-a-long-random-grafana-password
|
||||
PROMETHEUS_PORT=9090
|
||||
LOKI_PORT=3100
|
||||
TEMPO_PORT=3200
|
||||
|
||||
@@ -1,18 +1,59 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: EgorAnti / БПИ2301
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Целевая оценка (3/4/5): 5
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Трек 5, если заявлен: «Цена наблюдаемости» — сравнение telemetry off, full 100%/10s, economy 5%/60s и balanced 25%/30s при одинаковом профиле нагрузки.
|
||||
|
||||
Что реализовано и что перенесено:
|
||||
|
||||
- из lab5 перенесён и сохранён работающий минимум API/worker, PostgreSQL, Redis, RabbitMQ, MinIO, outbox/retry/DLQ и публичные тесты;
|
||||
- добавлены OpenTelemetry Java agent/API для API и worker, W3C trace context через RabbitMQ и durable retry, структурные OTLP logs, metrics и реальные traces;
|
||||
- в общий Compose добавлены OTel Collector с memory limiter/batch/bounded queues, Prometheus, Loki, Tempo и Grafana с persistent volumes;
|
||||
- provisioned datasources, импортируемый dashboard, p50/p95/p99 из histogram, error ratio, queue depth/age, DB pool, worker saturation, SLI/SLO и alert rules;
|
||||
- воспроизведены отказ worker, отказ PostgreSQL, retry через MinIO и недоступность Loki/Tempo с firing/resolved evidence;
|
||||
- проведён эксперимент стоимости телеметрии, проверены ingest/CPU/RAM/latency/sampling loss и выбран balanced-профиль: 25% parent-based traces, metrics 30s, все logs.
|
||||
|
||||
Команды запуска из чистого clone:
|
||||
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
# Заменить replace-with-* уникальными локальными значениями.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
$env:PYTHONUTF8 = "1"
|
||||
make check PYTHON=python
|
||||
make test BASE_URL=http://localhost:8080 PYTHON=python
|
||||
powershell -File scripts/experiment-observability-cost.ps1 `
|
||||
-BaseUrl http://localhost:8080 -LoadSeconds 45 -Concurrency 4
|
||||
```
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md:
|
||||
|
||||
- `make check PYTHON=python`: `PASS: offline skeleton structure, local refs and Python syntax`;
|
||||
- `make test BASE_URL=http://127.0.0.1:18080 PYTHON=python`: `PASS: lab6 public API scenario`;
|
||||
- `docker compose --env-file .env config --quiet`: успешно;
|
||||
- balanced работает с `OTEL_TRACES_SAMPLER_ARG=0.25`, `OTEL_METRIC_EXPORT_INTERVAL=30000`; итоговый `make test` прошёл;
|
||||
- во всех четырёх cost-режимах 0 load errors и 0 Collector transport-lost spans; normalized Tempo+Loki ingest balanced — 2200.52 B/request против 6918.72 B/request в full;
|
||||
- sampling-probe: 40 контролируемых HTTP 500, 8 sampled и 32 unsampled logs; sampled trace найден в Tempo (200), unsampled отсутствует (404);
|
||||
- полный отчёт: [`REPORT.md`](../REPORT.md); raw evidence: [`evidence/lab6-20261119-203024-observability-cost/`](../evidence/lab6-20261119-203024-observability-cost/).
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
|
||||
- стенд локальный и однорепличный, telemetry retention — 72 часа/3 дня;
|
||||
- cost experiment — один последовательный closed-loop прогон на Docker Desktop; абсолютные latency/CPU требуют повторов и confidence intervals перед production sizing;
|
||||
- head sampling принимает решение до outcome/duration: balanced может не сохранить около 75% единичных ошибок и редких slow traces, однако их logs и metric aggregates сохраняются;
|
||||
- на Windows перед `make check` нужен `PYTHONUTF8=1`, иначе Python с CP1251 не читает UTF-8 OpenAPI JSON.
|
||||
|
||||
Источники и использование ИИ:
|
||||
|
||||
- использованы официальные спецификации/документация OpenTelemetry, W3C Trace Context, Prometheus histograms, Grafana provisioning, Loki и Tempo; ссылки зафиксированы в `contracts/observability.md` и `REPORT.md`;
|
||||
- OpenAI Codex использовался для переноса lab5, реализации конфигурации/инструментации, подготовки воспроизводимых scripts, запуска проверок и анализа evidence; итог подтверждён командами и raw-артефактами выше.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
@@ -15,6 +15,7 @@ data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
.gradle/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
# Stage 1: Build
|
||||
FROM eclipse-temurin:23-jdk AS build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY gradlew .
|
||||
COPY gradle gradle
|
||||
COPY build.gradle settings.gradle ./
|
||||
|
||||
COPY src src
|
||||
|
||||
RUN --mount=type=cache,target=/root/.gradle \
|
||||
chmod +x gradlew && ./gradlew bootJar --no-daemon
|
||||
|
||||
|
||||
# Stage 2: Runtime
|
||||
FROM eclipse-temurin:23-jre
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /app/build/libs/*.jar app.jar
|
||||
|
||||
ARG OTEL_JAVA_AGENT_VERSION=2.33.1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends curl \
|
||||
&& curl --fail --location --silent --show-error \
|
||||
"https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_JAVA_AGENT_VERSION}/opentelemetry-javaagent.jar" \
|
||||
--output /opt/opentelemetry-javaagent.jar \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd --system --create-home messenger
|
||||
|
||||
COPY --chmod=0755 infra/otel/docker-entrypoint.sh /usr/local/bin/messenger-entrypoint
|
||||
|
||||
USER messenger
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
|
||||
CMD curl --fail --silent --show-error http://localhost:8080/health/ready || exit 1
|
||||
|
||||
ENTRYPOINT ["messenger-entrypoint"]
|
||||
@@ -1,37 +1,169 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: EgorAnti / БПИ2301
|
||||
- Целевая оценка: 5 (трек «Цена наблюдаемости»)
|
||||
- Commit/PR: заполнить после публикации
|
||||
- Реализация: OpenTelemetry Java agent/API, OTel Collector, Prometheus, Loki, Tempo, Grafana
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker с Compose v2, Java 23 только для локальной Gradle-сборки и Python 3.9+ для публичных проверок.
|
||||
|
||||
## Архитектура и решения
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
# Заменить все replace-with-* уникальными локальными значениями.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
docker compose ps
|
||||
python tests/smoke.py --base-url http://localhost:8080
|
||||
python tests/load.py --base-url http://localhost:8080 --duration 60 --concurrency 8 `
|
||||
--output evidence/lab6-http-load.json
|
||||
python tests/attachment_load.py --base-url http://localhost:8080 --count 20 `
|
||||
--output evidence/lab6-attachment-load.json
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Grafana: `http://localhost:3000`; Prometheus: `http://localhost:9090`. Учётные данные Grafana берутся только из `.env`. Безопасная остановка сохраняет volumes: `docker compose down`. Намеренное удаление учебных данных: `docker compose down --volumes`.
|
||||
|
||||
## Архитектура и границы
|
||||
|
||||
API и worker запускаются из одного образа, но с разными `MESSENGER_ROLE` и `OTEL_SERVICE_NAME`. В образ встроен закреплённый Java agent 2.33.1. Он создаёт реальные server/client/JDBC/RabbitMQ producer/consumer spans, переносит W3C `traceparent`/`tracestate` в AMQP headers, экспортирует runtime/HTTP/DB metrics и связывает logs с активным span. Для durable retry контекст сохраняется в `attachment_jobs.trace_parent/trace_state` (Flyway V8) и восстанавливается перед повторным publish: retry producer становится потомком предыдущего consumer span. Собственные метрики worker создаются через стабильный OTel API 1.67.0.
|
||||
|
||||
Collector принимает OTLP/HTTP и OTLP/gRPC. Перед экспортом применяются memory limiter, удаление потенциально чувствительных attributes и batch. Metrics доступны Prometheus через Collector exporter, logs отправляются OTLP в Loki, traces — OTLP в Tempo. Grafana provisioned из Git и не является хранилищем.
|
||||
|
||||
| Компонент | Назначение |
|
||||
| --- | --- |
|
||||
| Prometheus | Хранит числовые временные ряды и вычисляет rate, error ratio, histogram percentiles и alert expressions. |
|
||||
| Loki | Хранит структурные API/worker logs и позволяет найти событие по `service_name`, `event` и `trace_id`. |
|
||||
| Tempo | Хранит распределённые traces/spans и показывает причинную цепочку HTTP → broker → worker → DB/S3. |
|
||||
| Grafana | Запрашивает три источника, связывает log ↔ trace и отображает dashboard; сама не заменяет эти хранилища. |
|
||||
|
||||
Collector имеет лимит 384 MiB, batch 512 и exporter queues по 2048 элементов. Retry ограничен 60 секундами. SDK использует bounded span queue 2048, batch 512 и export timeout 5 секунд. Поэтому отказ telemetry backend не блокирует business threads и не вызывает неограниченный рост RAM; после исчерпания буферов допустима измеримая потеря telemetry.
|
||||
|
||||
## Соответствие сигналов
|
||||
|
||||
| Контракт | Реальный сигнал / запрос |
|
||||
| --- | --- |
|
||||
| HTTP counter | `http_server_request_duration_seconds_count`, labels `http_request_method`, `http_route`, `http_response_status_code` |
|
||||
| HTTP duration | `http_server_request_duration_seconds_bucket`; p50/p95/p99 через `histogram_quantile` по сумме buckets реплик |
|
||||
| Jobs processed/failed | `messenger_jobs_processed_total`, labels только `job_type` и `outcome` |
|
||||
| Job duration | `messenger_job_processing_duration_seconds_bucket` |
|
||||
| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket` |
|
||||
| Queue | `messenger_queue_depth`, `messenger_queue_oldest_job_age_seconds`; источник — durable DB queue |
|
||||
| Worker saturation | `messenger_worker_busy / messenger_worker_capacity` |
|
||||
| DB pool/runtime | auto-instrumentation `db_client_connections_*`, `jvm_*`, `process_*` |
|
||||
| Logs | OTLP logs в Loki с `service_name`, `severity`, `event`, `trace_id`, `span_id`, `request_id` |
|
||||
| Traces | Tempo: servlet server, RabbitMQ producer/consumer, JDBC/Redis/S3 client spans |
|
||||
|
||||
`user_id`, `chat_id`, `message_id`, `attachment_id`, request ID, filename и raw URL не используются как metric labels. Authorization/session tokens, bodies, изображения, S3 credentials и signed URLs не захватываются. Collector дополнительно удаляет известные sensitive keys. Технические job/attachment UUID также удалены из новых structured worker logs.
|
||||
|
||||
## Dashboard: запросы и единицы
|
||||
|
||||
| Панель | PromQL | Единица |
|
||||
| --- | --- | --- |
|
||||
| API RPS | `sum by (http_request_method) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m]))` | requests/s |
|
||||
| API 5xx ratio | `(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5..",http_route!~"/health/.*"}[5m])) or vector(0)) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m])), 0.001)` | ratio 0..1, в Grafana показывается как %; без 5xx возвращает 0, а не `No data` |
|
||||
| API p50/p95/p99 | `histogram_quantile(Q, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api",http_route!~"/health/.*"}[5m])))`, где `Q=0.50/0.95/0.99` | seconds |
|
||||
| Queue depth | `max(messenger_queue_depth{service_name="messenger-api"})` | jobs (count) |
|
||||
| Oldest job age | `max(messenger_queue_oldest_job_age_seconds{service_name="messenger-api"})` | seconds |
|
||||
| DB pool | `sum by (service_name, state) (db_client_connections_usage{service_name=~"messenger-(api|worker)"})` и `sum by (service_name) (db_client_connections_pending_requests{service_name=~"messenger-(api|worker)"})` | connections/requests (count) |
|
||||
| Worker saturation | `messenger_worker_busy / clamp_min(messenger_worker_capacity, 1)` с фильтром `service_name="messenger-worker"` | ratio 0..1, в Grafana показывается как % |
|
||||
|
||||
Percentile вычисляются из cumulative histogram buckets: сначала `rate(..._bucket[5m])`, затем `sum by (le)` объединяет buckets всех API-реплик, и только после этого применяется `histogram_quantile`. Готовые p50/p95/p99 отдельных реплик не вычисляются и не усредняются.
|
||||
|
||||
## SLI/SLO и alerts
|
||||
|
||||
API availability SLI: число 5xx business responses / все business responses за 5 минут; health routes исключены, 4xx не считаются отказом сервера. Учебный SLO — не менее 99% успешных business responses. Warning срабатывает при error ratio >5% в течение 2 минут.
|
||||
|
||||
Attachment readiness SLI: p95 времени от durable acceptance до `ready` за 10 минут. Учебная цель — p95 <60 секунд. Alert `AttachmentQueueLagHigh` срабатывает, если возраст старейшей незавершённой job >60 секунд ещё 1 минуту. Есть отдельные alerts на worker saturation и потерю Collector export.
|
||||
|
||||
| Alert | Условие и окно | `for` | Проверка |
|
||||
| --- | --- | --- | --- |
|
||||
| `MessengerApiHighErrorRate` | 5xx / все business requests за 5m >5%; health исключены | 2m | PostgreSQL stop: firing value 0.964; после восстановления ratio 0 и alert resolved |
|
||||
| `AttachmentQueueLagHigh` | oldest durable job age >60s | 1m | worker stop: depth 3, age 138s, firing; после drain depth/age 0 и alert resolved |
|
||||
| `AttachmentWorkerSaturated` | busy / capacity ≥100% | 2m | правило загружено; runbook сохранён |
|
||||
| `TelemetryExportFailures` | скорость failed trace/log exports >0 за 5m | 1m | правило загружено; bounded exporter runbook сохранён |
|
||||
|
||||
## Воспроизводимые инциденты и доказательства
|
||||
|
||||
### 1. Worker остановлен
|
||||
|
||||
```powershell
|
||||
powershell -File scripts/incident-worker-stop.ps1 -DurationSeconds 150 -JobCount 10
|
||||
```
|
||||
|
||||
Скрипт останавливает только worker, создаёт синтетические attachment jobs, ждёт firing queue-lag alert, сохраняет Prometheus values/alerts, запускает worker и сохраняет состояние после drain. Результат помещается в `evidence/lab6-<timestamp>-worker-stop/` и не содержит credentials.
|
||||
|
||||
Проверено повторно 19.11.2026: API принял 3/3 uploads при остановленном worker; `queue_depth=3`, `oldest_job_age=138`, alert `AttachmentQueueLagHigh` был `firing`. После восстановления `queue_depth=0`, `oldest_job_age=0`, active alert отсутствует (`alert_resolved=true`). Raw evidence: `evidence/lab6-20261119-192857-worker-stop/`.
|
||||
|
||||
### 2. PostgreSQL недоступен
|
||||
|
||||
```powershell
|
||||
powershell -File scripts/incident-database-stop.ps1 -DurationSeconds 165
|
||||
```
|
||||
|
||||
Скрипт ограниченно останавливает PostgreSQL, опрашивает readiness и business endpoint, сохраняет status/latency, Prometheus, Loki и Tempo snapshots, затем обязательно запускает БД в `finally` и проверяет восстановление.
|
||||
|
||||
Проверено повторно 19.11.2026: 33 readiness/business probes сформировали контролируемые 503; `MessengerApiHighErrorRate` перешёл в `firing` со значением 0.964. Loki/Tempo сохранили JDBC connection failures и error spans. После запуска PostgreSQL readiness восстановился, пятиминутное error ratio стало 0, active alert исчез (`alert_resolved=true`). Raw evidence: `evidence/lab6-20261119-193359-database-stop/`.
|
||||
|
||||
### 3. Отказ telemetry backend
|
||||
|
||||
```powershell
|
||||
powershell -File scripts/incident-telemetry-backend.ps1 -BaseUrl http://localhost:8080
|
||||
```
|
||||
|
||||
Проверено 19.11.2026 остановкой Loki и Tempo при работающих Collector/Prometheus. `make test` прошёл; за 75 секунд выполнено 3060/3060 успешных POST messages, ошибок 0. По 14 samples максимум RSS Collector составил 173644186 bytes (165.6 MiB) при container limit 512 MiB и memory limiter 384 MiB. Максимальная exporter queue — 66 из capacity 2048. После 60-секундного retry budget Collector учёл 3364 failed spans и 926 failed log records: это измеримая ограниченная потеря, а не блокировка business threads или бесконечный рост памяти. После запуска backend обе `/ready` вернули 200, queue size вернулся к 0. Raw evidence: `evidence/lab6-20261119-194742-telemetry-backend/`.
|
||||
|
||||
Поведение ограничено на двух уровнях: SDK имеет BSP queue 2048/batch 512/export timeout 5s; Collector имеет memory limiter 384 MiB, batch 512, exporter queue 2048 и retry не дольше 60s. Когда backend не возвращается в budget, элементы отбрасываются с ростом `otelcol_exporter_send_failed_*`; сообщения приложения при этом продолжают фиксироваться в PostgreSQL/RabbitMQ.
|
||||
|
||||
### Сквозной retry trace
|
||||
|
||||
```powershell
|
||||
powershell -File scripts/incident-retry-trace.ps1 -BaseUrl http://localhost:8080
|
||||
```
|
||||
|
||||
Скрипт принимает один синтетический upload при остановленном worker, кратко делает MinIO недоступным для первой обработки, автоматически восстанавливает сервисы и проверяет parent-цепочку в Tempo/Loki. Проверено 19.11.2026: trace `a3dc1f8fabc7dc732fc3f502007f7654` содержит 3 producer и 3 consumer spans. Каждый retry consumer имеет retry producer как parent; финальная попытка содержит S3 PUT/DELETE и JDBC result commit. Raw evidence: `evidence/lab6-20261119-191738-retry-trace/`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Команда | Артефакт |
|
||||
| --- | --- | --- |
|
||||
| Compose воспроизводим | `docker compose --env-file .env config --quiet` | вывод команды |
|
||||
| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8080` | вывод команды |
|
||||
| Metrics/logs/traces | HTTP и attachment load; dashboard `messenger-observability` | `evidence/README.md` и timestamped raw evidence |
|
||||
| API → queue → worker trace + retry | `scripts/incident-retry-trace.ps1` | trace `a3dc1f8fabc7dc732fc3f502007f7654`, raw summary в `evidence/lab6-20261119-191738-retry-trace/` |
|
||||
| Worker failure/recovery | `scripts/incident-worker-stop.ps1` | timestamped evidence directory |
|
||||
| DB failure/recovery | `scripts/incident-database-stop.ps1` | timestamped evidence directory |
|
||||
| Importable dashboard | автоматический provisioning | `infra/otel/grafana/dashboards/messenger-overview.json` |
|
||||
| Alert rules/runbooks | Prometheus rule load | `infra/otel/prometheus-alerts.yaml`, `infra/otel/runbooks.md` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Цена наблюдаемости и выбранный профиль
|
||||
|
||||
## Эксперименты и отказы
|
||||
Воспроизводимый сценарий `scripts/experiment-observability-cost.ps1` последовательно пересоздаёт только API и worker, делает одинаковый 10-секундный прогрев и запускает для каждого режима один и тот же closed-loop профиль: 45 секунд, concurrency 4, POST messages, без клиентских retry. JVM в каждом режиме холодная; БД, broker, Collector и хранилища общие. Ingest сравнивается как дельта монотонных счётчиков и дополнительно нормализуется на успешный запрос. CPU — среднее `docker stats` за измеряемую фазу, RAM — peak RSS. Это локальный сравнительный опыт, а не capacity benchmark: closed-loop имеет coordinated omission, а один прогон не позволяет ранжировать близкие sampling-профили по latency.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Проверенный запуск: `evidence/lab6-20261119-203024-observability-cost/`. Во всех режимах ошибок нагрузки было 0, а Collector transport loss был 0 spans.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Режим | Traces / metrics | RPS | p50 / p95 / p99, ms | API CPU avg / RAM peak | Collector CPU avg / RAM peak |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| off | agent/SDK выключен | 40.55 | 97 / 198 / 275 | 66.50% / 363.0 MiB | 0.05% / 144.5 MiB |
|
||||
| full | 100% / 10s | 22.42 | 185 / 315 / 406 | 69.18% / 405.7 MiB | 2.17% / 150.2 MiB |
|
||||
| economy | 5% / 60s | 14.62 | 287 / 406 / 502 | 68.56% / 411.8 MiB | 0.88% / 152.2 MiB |
|
||||
| balanced | 25% / 30s | 16.93 | 210 / 391 / 409 | 67.62% / 406.6 MiB | 0.54% / 148.5 MiB |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Одинаковая concurrency насыщала лимит API, поэтому CPU близок, а цена видна в throughput/latency: относительно off full дал -44.7% RPS и +59.1% p95; balanced — -58.2% RPS и +97.6% p95. Немонотонность full/economy/balanced означает, что основная application-side цена — сам agent, создание metrics/logs/context и экспорт, а не только число сохранённых spans; различия одного локального прогона нельзя приписывать sampler без повторов. Память API с agent выросла примерно на 43–49 MiB, worker — на 81–109 MiB. Sampling прежде всего снижает Collector/backend cost.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| Режим | Accepted spans / logs / metric points | Нормализованное сохранение traces | Tempo+Loki bytes / успешный запрос | Потеря Collector |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| off | 0 / 0 / 0 | 0% | 0 B | 0 spans |
|
||||
| full | 12346 / 1019 / 711 | 100% | 6918.72 B | 0 spans |
|
||||
| economy | 447 / 671 / 115 | 5.52% | 1108.25 B | 0 spans |
|
||||
| balanced | 2200 / 775 / 238 | 23.48% | 2200.52 B | 0 spans |
|
||||
|
||||
Нормализованный balanced ingest trace+log оказался на 68.2% ниже full. Логи не sample-ятся, поэтому создают полезный нижний предел ingest. Metrics — cumulative counters/histograms; интервал 30 секунд уменьшает число export points, но не превращает percentile в среднее и остаётся короче минутного queue-lag alert window. Интервал 60 секунд откладывает свежесть почти на весь alert window и вместе с 5% traces слишком сильно ухудшает диагностику.
|
||||
|
||||
`parentbased_traceidratio` принимает решение в начале root trace, до знания HTTP status и duration. Поэтому при balanced один отдельный 5xx или редкий медленный запрос имеет примерно 75% шанс не попасть в Tempo; при economy — примерно 95%. Если таких событий `k`, вероятность не сохранить ни одного при balanced равна `0.75^k`. Проверочная серия 40 контролируемых 500 responses дала 8 sampled и 32 unsampled logs: sampled trace `00d814f7907510c8066b49c89cefd0ec` найден в Tempo (HTTP 200), unsampled `495125068d65af4c622b34ada5196db5` отсутствовал (HTTP 404). При этом структурный log есть для всех 40, а metrics продолжают считать status и histogram duration.
|
||||
|
||||
Выбран balanced профиль: parent-based 25% traces, metrics export 30s, logs без sampling. Он сохраняет целую API → broker → worker parent-цепочку для выбранного trace и уменьшает storage ingest, не задерживая minute-scale alerts. Tail sampling мог бы гарантированно оставлять error/slow traces, но для много минут живущих async retry traces потребовал бы большого decision window и памяти Collector; короткое окно разрезало бы цепочку. Поэтому здесь риск редких пропусков принят явно, а симптомы остаются в несэмплируемых logs/metrics. Для production следующий шаг — отдельные bounded tail policies либо error-triggered trace retention с проверкой длинных queue traces.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Локальный стенд однорепличный для telemetry storage и не является production HA. Retention — 72 часа/3 дня. Queue age берётся из durable DB job queue, а не peek первого RabbitMQ message; это сохраняет измеримость при остановленном worker и соответствует outbox как источнику истины. Cost experiment использует один последовательный closed-loop прогон на общем Docker Desktop host; абсолютные CPU/latency и немонотонные различия sampling-профилей требуют повторов и confidence intervals перед production sizing. Runtime evidence в `evidence/` получено на локальном Compose 19.11.2026; для защиты можно дополнительно приложить screenshots тех же запросов в Grafana Explore.
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
plugins {
|
||||
id 'java'
|
||||
id 'org.springframework.boot' version '4.1.1'
|
||||
id 'io.spring.dependency-management' version '1.1.7'
|
||||
}
|
||||
|
||||
group = 'com.example'
|
||||
version = '0.0.1-SNAPSHOT'
|
||||
|
||||
java {
|
||||
toolchain {
|
||||
languageVersion = JavaLanguageVersion.of(23)
|
||||
}
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
|
||||
// implementation 'org.springframework.boot:spring-boot-starter-security'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webmvc'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-validation'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-amqp'
|
||||
implementation 'org.springframework.boot:spring-boot-flyway'
|
||||
implementation 'org.flywaydb:flyway-core'
|
||||
implementation 'org.flywaydb:flyway-database-postgresql'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-security'
|
||||
implementation 'org.bouncycastle:bcprov-jdk18on:1.84'
|
||||
implementation 'org.apache.commons:commons-pool2'
|
||||
implementation 'io.minio:minio:8.6.0'
|
||||
implementation 'net.coobird:thumbnailator:0.4.20'
|
||||
implementation 'org.sejda.imageio:webp-imageio:0.1.6'
|
||||
// The runtime SDK is supplied by the pinned OpenTelemetry Java agent.
|
||||
// The API dependency keeps custom business metrics vendor-neutral.
|
||||
implementation 'io.opentelemetry:opentelemetry-api:1.67.0'
|
||||
runtimeOnly 'org.postgresql:postgresql'
|
||||
|
||||
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
|
||||
implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.3'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
// testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test'
|
||||
// testImplementation 'org.springframework.boot:spring-boot-starter-security-test'
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-webmvc-test'
|
||||
testCompileOnly 'org.projectlombok:lombok'
|
||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||
testAnnotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
implementation("com.github.f4b6a3:uuid-creator:6.1.1")
|
||||
|
||||
}
|
||||
|
||||
tasks.named('test') {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
name: hla-lab6
|
||||
|
||||
x-api-environment: &api-environment
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_TRACES_EXPORTER: otlp
|
||||
OTEL_METRICS_EXPORTER: otlp
|
||||
OTEL_LOGS_EXPORTER: otlp
|
||||
OTEL_PROPAGATORS: tracecontext,baggage
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-0.25}
|
||||
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-30000}
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048}
|
||||
OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512}
|
||||
OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-5000}
|
||||
OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT: ${OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT:-256}
|
||||
OTEL_RESOURCE_ATTRIBUTES: service.version=0.0.1,deployment.environment.name=${APP_ENV:-lab}
|
||||
OTEL_INSTRUMENTATION_LOGBACK_APPENDER_EXPERIMENTAL_MDC_ATTRIBUTES_INCLUDED: request_id
|
||||
OTEL_INSTRUMENTATION_COMMON_LOGGING_STRUCTURED_ATTRIBUTES_INCLUDED: event,job.type,outcome,attempt,error.code,http.request.method,http.route,http.response.status_code
|
||||
OTEL_INSTRUMENTATION_HTTP_SERVER_CAPTURE_REQUEST_HEADERS: ""
|
||||
OTEL_INSTRUMENTATION_HTTP_SERVER_CAPTURE_RESPONSE_HEADERS: ""
|
||||
DATABASE_URL: jdbc:postgresql://postgres:5432/${DATABASE_NAME:-messenger}
|
||||
DATABASE_USER: ${DATABASE_USER:-messenger}
|
||||
DATABASE_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10}
|
||||
DATABASE_MIN_IDLE: ${DATABASE_MIN_IDLE:-2}
|
||||
DATABASE_QUERY_TIMEOUT_SECONDS: ${DATABASE_QUERY_TIMEOUT_SECONDS:-5}
|
||||
DATABASE_TRANSACTION_TIMEOUT_SECONDS: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:-5}
|
||||
REDIS_HOST: redis
|
||||
REDIS_PORT: 6379
|
||||
REDIS_CONNECT_TIMEOUT: ${REDIS_CONNECT_TIMEOUT:-1s}
|
||||
REDIS_COMMAND_TIMEOUT: ${REDIS_COMMAND_TIMEOUT:-500ms}
|
||||
REDIS_POOL_MAX_ACTIVE: ${REDIS_POOL_MAX_ACTIVE:-8}
|
||||
REDIS_POOL_MAX_IDLE: ${REDIS_POOL_MAX_IDLE:-8}
|
||||
REDIS_POOL_MIN_IDLE: ${REDIS_POOL_MIN_IDLE:-0}
|
||||
REDIS_POOL_MAX_WAIT: ${REDIS_POOL_MAX_WAIT:-500ms}
|
||||
USER_CACHE_TTL_SECONDS: ${USER_CACHE_TTL_SECONDS:-300}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
|
||||
UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000}
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_PORT: 5672
|
||||
RABBITMQ_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||
RABBITMQ_VHOST: ${RABBITMQ_VHOST:-/}
|
||||
ATTACHMENT_JOB_EXCHANGE: ${ATTACHMENT_JOB_EXCHANGE:-attachments.v1}
|
||||
ATTACHMENT_JOB_QUEUE: ${ATTACHMENT_JOB_QUEUE:-attachments.process.level4.v1}
|
||||
ATTACHMENT_JOB_ROUTING_KEY: ${ATTACHMENT_JOB_ROUTING_KEY:-attachment.process.v1}
|
||||
ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1}
|
||||
ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1}
|
||||
ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1}
|
||||
ATTACHMENT_PIPELINE_VERSION: ${ATTACHMENT_PIPELINE_VERSION:-2}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_RETRY_BASE_DELAY: ${JOB_RETRY_BASE_DELAY:-1s}
|
||||
JOB_RETRY_MAX_DELAY: ${JOB_RETRY_MAX_DELAY:-30s}
|
||||
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
|
||||
JOB_LEASE_DURATION: ${JOB_LEASE_DURATION:-90s}
|
||||
JOB_LEASE_REAPER_INTERVAL: ${JOB_LEASE_REAPER_INTERVAL:-5s}
|
||||
WORKER_MAX_IMAGE_PIXELS: ${WORKER_MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_MEMORY_BUDGET_BYTES: ${WORKER_MEMORY_BUDGET_BYTES:-536870912}
|
||||
WORKER_MEMORY_WAIT: ${WORKER_MEMORY_WAIT:-2s}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600}
|
||||
ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false}
|
||||
ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000}
|
||||
ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:-60000}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${DATABASE_NAME:-messenger}
|
||||
POSTGRES_USER: ${DATABASE_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
mem_limit: 768m
|
||||
networks: [backend]
|
||||
|
||||
redis:
|
||||
image: redis:7.4-bookworm
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
networks: [backend]
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-management-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||
RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/}
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
volumes:
|
||||
- rabbitmq-data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
start_period: 60s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 1024m
|
||||
networks: [backend]
|
||||
|
||||
minio:
|
||||
image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
volumes:
|
||||
- minio-data:/bitnami/minio/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://localhost:9000/minio/health/live"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
cpus: 0.75
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
minio-init:
|
||||
image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
volumes:
|
||||
- ./infra/minio-api-policy.json:/policy/minio-api-policy.json:ro
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json
|
||||
attempt=0
|
||||
until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" \
|
||||
&& mc mb --ignore-existing "local/$$S3_BUCKET" \
|
||||
&& mc anonymous set none "local/$$S3_BUCKET" \
|
||||
&& (mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" \
|
||||
|| mc admin user info local "$$S3_ACCESS_KEY_ID") \
|
||||
&& (mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json \
|
||||
|| mc admin policy info local messenger-attachments-rw) \
|
||||
&& mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID"
|
||||
do
|
||||
attempt=$$((attempt + 1))
|
||||
if [ "$$attempt" -ge 30 ]; then
|
||||
echo "MinIO bootstrap did not complete after 30 attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
private-bucket-check:
|
||||
profiles: ["verification"]
|
||||
image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc
|
||||
depends_on:
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set anonymous http://minio:9000
|
||||
if mc ls "anonymous/$$S3_BUCKET" >/dev/null 2>&1; then
|
||||
echo "FAIL: anonymous bucket listing succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
mc alias set restricted http://minio:9000 "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||
mc ls "restricted/$$S3_BUCKET" >/dev/null
|
||||
if mc admin info restricted >/dev/null 2>&1; then
|
||||
echo "FAIL: API credentials have administrative access" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: bucket is private and API credentials are non-admin"
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
# One-shot Flyway runner. It is invoked explicitly before API replicas.
|
||||
migrate:
|
||||
image: hla-lab6-app:local
|
||||
build:
|
||||
context: .
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "true"
|
||||
MIGRATE_ONLY: "true"
|
||||
MESSENGER_ROLE: migration
|
||||
SPRING_MAIN_WEB_APPLICATION_TYPE: none
|
||||
OTEL_SDK_DISABLED: "true"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
networks: [backend]
|
||||
|
||||
api:
|
||||
image: hla-lab6-app:local
|
||||
build:
|
||||
context: .
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "false"
|
||||
MESSENGER_ROLE: api
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
FAULT_CRASH_AFTER_OUTBOX_COMMIT: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:-false}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 180s
|
||||
init: true
|
||||
stop_grace_period: 12s
|
||||
restart: unless-stopped
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
networks: [backend]
|
||||
|
||||
worker:
|
||||
image: hla-lab6-app:local
|
||||
environment:
|
||||
<<: *api-environment
|
||||
FLYWAY_ENABLED: "false"
|
||||
MESSENGER_ROLE: worker
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
SPRING_MAIN_WEB_APPLICATION_TYPE: none
|
||||
JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70
|
||||
FAULT_CRASH_BEFORE_SIDE_EFFECT: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:-false}
|
||||
FAULT_CRASH_AFTER_SIDE_EFFECT: ${FAULT_CRASH_AFTER_SIDE_EFFECT:-false}
|
||||
FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
disable: true
|
||||
init: true
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
mem_limit: 1024m
|
||||
networks: [backend]
|
||||
|
||||
nginx:
|
||||
image: nginx:1.27.4-bookworm
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:80"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "nginx", "-t"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [backend]
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.165.0
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_started
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.14.0
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=3d
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/prometheus-alerts.yaml:/etc/prometheus/prometheus-alerts.yaml:ro
|
||||
- prometheus-data:/prometheus
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.8
|
||||
command: ["-config.file=/etc/loki/local-config.yaml"]
|
||||
ports:
|
||||
- "127.0.0.1:${LOKI_PORT:-3100}:3100"
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro
|
||||
- loki-data:/loki
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:3.1.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
ports:
|
||||
- "127.0.0.1:${TEMPO_PORT:-3200}:3200"
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo-data:/var/tempo
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:13.3.0
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_UNIFIED_ALERTING_ENABLED: "true"
|
||||
volumes:
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_started
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [backend]
|
||||
|
||||
networks:
|
||||
backend:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
minio-data:
|
||||
rabbitmq-data:
|
||||
prometheus-data:
|
||||
loki-data:
|
||||
tempo-data:
|
||||
grafana-data:
|
||||
+80
-3
@@ -1,5 +1,82 @@
|
||||
# Доказательства
|
||||
# Проверенные наблюдения lab6 — 19.11.2026
|
||||
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
Все данные ниже получены из запущенного Compose-стека, а не из предположений. Нагрузка использовала только синтетические PNG и пользователей; credentials и bodies в evidence не записывались.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
Grafana 13.3.0 отдельно прошла runtime provisioning check: `/api/health` вернул `database=ok`, API перечислил datasources `Loki`, `Prometheus`, `Tempo` и dashboard `Messenger observability`.
|
||||
|
||||
`make test BASE_URL=http://127.0.0.1:18080 PYTHON=python` выполнен с включённым Java agent и работающим Collector; публичный сценарий завершился `PASS`.
|
||||
|
||||
Свежий trace этого запуска `128cf1e355c405bc421ef4e0b5daa0e0` найден одновременно в Tempo и Loki. API log `http_request_completed` содержит route template `/api/v1/chats/{chatId}/attachments`, status 202 и `span_id=cd375b5c0e8ea3ab`; worker log `attachment_job_completed` содержит outcome `success` и `span_id=10bac10a3d80b9a6`. В Tempo producer `attachments.v1 publish` является родителем consumer `attachment.process.v1 process`; в той же trace присутствуют JDBC и S3 PUT/DELETE spans обоих сервисов.
|
||||
|
||||
## Сквозная корреляция
|
||||
|
||||
Smoke прошёл за 8.35 с после прогрева JVM. Для attachment upload Tempo вернул trace `376c8caf07126a6950cf8c826a9d1b6e` с одной цепочкой:
|
||||
|
||||
`messenger-api POST /api/v1/chats/{chatId}/attachments` → RabbitMQ producer → `messenger-worker attachment.process.v1 process` → JDBC/S3 child spans.
|
||||
|
||||
Loki вернул structured event `attachment_job_completed` с тем же `trace_id`, `span_id=a8a1bffac4fb70f1`, `service_name=messenger-worker`, `outcome=success`. Collector internal metrics на момент проверки: 1830 accepted spans и 55 accepted log records. В metric series использовались route templates и low-cardinality labels; UUID, request ID, filename и raw URL отсутствовали в labels.
|
||||
|
||||
## Отказ 1: worker остановлен
|
||||
|
||||
Raw files: `lab6-20261119-131448-worker-stop/`.
|
||||
|
||||
- workload: 5 requested, 5 accepted, 0 errors;
|
||||
- failure: queue depth 5, oldest age 139 s, `AttachmentQueueLagHigh` firing;
|
||||
- recovery: queue depth 0, oldest age 0, successful jobs снова увеличиваются.
|
||||
|
||||
Причина по сохранённым наблюдениям: API продолжал принимать задачи, но consumer отсутствовал; росли только durable queue depth/age. После возврата consumer backlog исчез.
|
||||
|
||||
## Отказ 2: PostgreSQL остановлен
|
||||
|
||||
Raw files: `lab6-20261119-131929-database-stop/`.
|
||||
|
||||
- семь readiness/business probes подряд вернули HTTP 503;
|
||||
- цикл probe занимал 3988.8–4537.5 ms;
|
||||
- Prometheus сохранил ненулевую скорость 5xx;
|
||||
- Loki сохранил `Queue telemetry sample failed: CannotGetJdbcConnectionException` и `HikariPool-1 - Failed to validate connection ... This connection has been closed`, включая коррелируемый trace `7b47023dcfebb5738de24578b483931a`;
|
||||
- Tempo TraceQL `{ span.db.system = "postgresql" && status = error }` вернул PostgreSQL error spans; trace `7b47023dcfebb5738de24578b483931a` длился 5030 ms. Запрос `{ name = "GET /health/ready" && status = error }` вернул 5-секундные error traces;
|
||||
- после запуска PostgreSQL readiness снова прошёл (`recovery.txt`).
|
||||
|
||||
Причина по сохранённым наблюдениям: задержка находилась в PostgreSQL/JDBC connection path, а не в CPU-работе HTTP handler.
|
||||
|
||||
## W3C context через durable retry
|
||||
|
||||
Raw files: `lab6-20261119-191738-retry-trace/`.
|
||||
|
||||
Один синтетический upload был принят при остановленном worker, затем MinIO кратко останавливался. Trace `a3dc1f8fabc7dc732fc3f502007f7654` содержит 3 producer и 3 consumer spans в одной trace:
|
||||
|
||||
`upload server` → `initial publish` → `consumer #1` → `retry publish` → `consumer #2` → `retry publish` → `consumer #3` → `S3 PUT/DELETE + DB commit`.
|
||||
|
||||
Parent IDs в `retry-trace-summary.json` показывают, что каждый consumer является ребёнком соответствующего producer, а каждый retry producer — ребёнком предыдущего consumer. Loki в этой же trace содержит `attachment_job_retry_scheduled` (`attempt=2`, `error.code=processing_failed`) и финальный `attachment_job_completed` (`outcome=success`).
|
||||
|
||||
## Dashboard и PromQL
|
||||
|
||||
Grafana runtime API загрузил dashboard version 3. Все 13 Prometheus targets dashboard были выполнены через `/api/v1/query` со статусом `success`: RPS, error ratio, p50/p95/p99, job outcomes/duration, time-to-ready, queue depth/age, worker saturation и DB pool. В здоровом состоянии error ratio возвращает одну series со значением `0`, а не `No data`.
|
||||
|
||||
Панель API latency использует `histogram_quantile` после `sum by (le)` над `rate(..._bucket[5m])`; percentile реплик не усредняются. Grafana units проверены через API: latency/oldest age — seconds, error ratio/worker saturation — percent, queue depth и DB pool — count. Для совмещённой queue-панели depth использует левую ось, age — правую.
|
||||
|
||||
## Alert lifecycle: firing → resolved
|
||||
|
||||
`AttachmentQueueLagHigh`: raw files `lab6-20261119-192857-worker-stop/`. При остановленном worker API принял 3/3 jobs, depth был 3, oldest age 138s, состояние alert — `firing`. После запуска worker depth/age стали 0, в recovery snapshot active alert отсутствует и `alert_resolved=true`.
|
||||
|
||||
`MessengerApiHighErrorRate`: raw files `lab6-20261119-193359-database-stop/`. При остановленном PostgreSQL выполнено 33 контролируемых readiness/business probe; alert достиг `firing` со значением 0.964. После восстановления БД readiness стал успешным, error ratio вернулся к 0, active alert отсутствует и `alert_resolved=true`.
|
||||
|
||||
Внешняя доставка намеренно не настраивалась: критерий проверяется локальными Prometheus rule states и timestamped JSON snapshots.
|
||||
|
||||
## Отказ telemetry backend
|
||||
|
||||
Raw files: `lab6-20261119-194742-telemetry-backend/`.
|
||||
|
||||
Loki и Tempo были остановлены при работающих Collector/Prometheus. `make test` завершился `PASS`; 75-секундная нагрузка записала 3060/3060 сообщений без ошибок. В 14 последовательных samples Collector достиг максимум 173644186 bytes (165.6 MiB) RSS при лимитах 384 MiB memory limiter / 512 MiB container. Exporter queue достигала 66 из 2048.
|
||||
|
||||
После исчерпания ограниченного retry budget счётчики показали 3364 failed spans и 926 failed log records. Это ожидаемая измеримая потеря telemetry: business traffic не блокировался, RAM и очереди оставались ограниченными. После восстановления Loki/Tempo оба `/ready` вернули 200, exporter queue стала 0. Воспроизводимый сценарий: `scripts/incident-telemetry-backend.ps1`.
|
||||
|
||||
## Цена наблюдаемости
|
||||
|
||||
Raw files: `lab6-20261119-203024-observability-cost/`; сценарий: `scripts/experiment-observability-cost.ps1`.
|
||||
|
||||
При одинаковых duration 45s / concurrency 4 / workload профиль off показал p95 198.133ms и 40.547 RPS; full (100% traces, metrics 10s) — 315.289ms и 22.419 RPS; economy (5%/60s) — 405.639ms и 14.622 RPS; balanced (25%/30s) — 391.490ms и 16.934 RPS. Все четыре режима завершились без ошибок. CPU/RAM взяты из raw `*-resource-samples.json`; итоговая таблица находится в `comparison.csv`/`summary.json`.
|
||||
|
||||
Normalized trace retention составил 5.52% для economy и 23.48% для balanced. Tempo+Loki ingest на успешный request: full 6918.72B, economy 1108.25B, balanced 2200.52B. Во всех режимах дельта Collector failed/enqueue-failed/refused была 0: недостающие spans — результат решения SDK sampler, а не транспортная потеря.
|
||||
|
||||
На balanced выполнено 40 контролируемых 500 requests: logs содержали 8 sampled и 32 unsampled trace IDs. Sampled trace вернул Tempo HTTP 200, unsampled — 404. Это прямое подтверждение, что head sampler принимает решение до outcome и может потерять отдельную ошибку или редкий slow trace; logs/metrics при этом остаются. После эксперимента `make test` прошёл. Выбран профиль 25% traces / metrics 30s / все logs; детали компромисса и ограничения одного локального прогона описаны в `REPORT.md`.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"scenario": "sequential failover of each API replica",
|
||||
"replicas": 2,
|
||||
"results": [
|
||||
{
|
||||
"target_instance_id": "8e081627dfc3",
|
||||
"window_seconds": 15.775,
|
||||
"successful_reads": 3,
|
||||
"successful_writes": 4,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 0.444,
|
||||
"first_recovery_seconds": 5.018
|
||||
},
|
||||
{
|
||||
"target_instance_id": "95b5db854bee",
|
||||
"window_seconds": 16.774,
|
||||
"successful_reads": 3,
|
||||
"successful_writes": 3,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 0.358,
|
||||
"first_recovery_seconds": 6.15
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
# Lab4 evidence — 2026-11-11
|
||||
|
||||
Стенд: Docker Compose, nginx на `127.0.0.1:18084`, API ×2, PostgreSQL, Redis, MinIO. Секреты и session tokens в evidence не сохранялись.
|
||||
|
||||
## Bootstrap и миграции
|
||||
|
||||
- MinIO server image: `bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd…e0cab`.
|
||||
- `minio-init`: bucket `messenger-attachments` создан, anonymous access установлен `private`, пользователь `messenger-api` создан, policy `messenger-attachments-rw` назначена.
|
||||
- Flyway: успешно проверены и применены 4 миграции, итоговая версия schema `v4`.
|
||||
- `docker compose ps`: две здоровые API-реплики; MinIO/PostgreSQL/Redis/nginx healthy; наружу опубликован только nginx.
|
||||
|
||||
## Проверки
|
||||
|
||||
```text
|
||||
AttachmentServiceTest + AttachmentReconcilerTest: BUILD SUCCESSFUL, 10 tests.
|
||||
tests/smoke.py: PASS: lab4 public API scenario.
|
||||
student_tests/lab4_negative.py:
|
||||
PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption,
|
||||
byte limit and filename checks.
|
||||
```
|
||||
|
||||
Persistence hook останавливал nginx, обе API, Redis, PostgreSQL и MinIO, затем запускал те же named volumes и повторял idempotent bucket bootstrap:
|
||||
|
||||
```text
|
||||
PASS: persistence preserved confirmed data and existing session.
|
||||
```
|
||||
|
||||
Failover hook остановил replica из `X-Instance-Id`, обслужившую тестовый запрос:
|
||||
|
||||
```json
|
||||
{"observed_instances_before":2,"successful_writes":102,"failed_attempts":0,"first_recovery_seconds":1.386}
|
||||
```
|
||||
|
||||
```text
|
||||
PASS: failover preserved confirmed data and existing session.
|
||||
```
|
||||
|
||||
При остановленном MinIO:
|
||||
|
||||
```text
|
||||
PASS: unavailable S3 returns contract JSON 503
|
||||
```
|
||||
|
||||
После запуска того же MinIO volume публичный smoke повторно завершился PASS, checksum скачанного original совпал.
|
||||
|
||||
## Level 4: consistency, memory and private S3
|
||||
|
||||
Unit tests inject an interrupted request stream before S3 PUT and a repository
|
||||
failure after S3 PUT. The first leaves storage empty; the second verifies the
|
||||
compensating DELETE. Reconciler tests prove that an orphan younger than the
|
||||
one-hour grace period survives, an older orphan is deleted, dry-run never
|
||||
deletes, and a missing object is reported.
|
||||
|
||||
Bootstrap was executed again against the existing bucket and completed with
|
||||
exit code 0. The direct S3 probe (without application API) produced:
|
||||
|
||||
```text
|
||||
PASS: bucket is private and API credentials are non-admin
|
||||
```
|
||||
|
||||
Full Gradle suite using the live Compose PostgreSQL/Redis/MinIO network:
|
||||
|
||||
```text
|
||||
BUILD SUCCESSFUL in 11m 47s
|
||||
```
|
||||
|
||||
Concurrent memory experiment (`evidence/upload-memory-2026-11-11.json`):
|
||||
|
||||
```json
|
||||
{"concurrency":12,"successful_uploads":12,"payload_bytes_each":6835833,
|
||||
"duration_seconds":10.156,"api_replicas":2,"memory_samples":5,
|
||||
"baseline_api_memory_bytes":668467200,"peak_api_memory_bytes":839909375,
|
||||
"peak_increase_bytes":171442175,"configured_max_concurrent_uploads_per_replica":3}
|
||||
```
|
||||
|
||||
The memory values are the aggregate of both API containers from `docker stats`.
|
||||
Sampling can miss a shorter peak, so this is an operational bound observation,
|
||||
not a JVM heap proof. The code-level bounds are 10 MiB/request, 40M pixels and
|
||||
three active uploads per replica.
|
||||
@@ -0,0 +1,41 @@
|
||||
# Lab5 verification — 2026-11-18
|
||||
|
||||
Среда: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23, Spring Boot 4.1.1, RabbitMQ 4.1.4.
|
||||
|
||||
## Статические и unit-проверки
|
||||
|
||||
```text
|
||||
docker compose --env-file .env.example config --quiet
|
||||
PASS (exit 0)
|
||||
|
||||
gradlew test --tests '*AttachmentServiceTest' --tests '*ThumbnailProcessorTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'
|
||||
BUILD SUCCESSFUL
|
||||
```
|
||||
|
||||
## Публичный сценарий
|
||||
|
||||
```text
|
||||
python tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
```
|
||||
|
||||
В момент проверки отдельные services `api`, `worker`, `rabbitmq`, `postgres`, `redis`, `minio` и `nginx` были запущены; RabbitMQ queue `attachments.process.v1` имела `durable=true`.
|
||||
|
||||
## Остановка worker, restart и redelivery
|
||||
|
||||
```text
|
||||
python student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
```
|
||||
|
||||
Сценарий теста:
|
||||
|
||||
1. останавливает worker;
|
||||
2. получает upload `202 queued`, проверяет `409` content и привязку к сообщению;
|
||||
3. перезапускает API и RabbitMQ без удаления volumes;
|
||||
4. запускает worker и ждёт `ready`;
|
||||
5. сверяет original bytes и единственный thumbnail;
|
||||
6. публикует тот же `job_id` повторно;
|
||||
7. проверяет terminal `ready` и ровно одну запись thumbnail.
|
||||
|
||||
Синтетические пароли и токены тест не печатает и в evidence не сохраняет.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Lab5 level 4 verification — 2026-11-19
|
||||
|
||||
Среда: Windows, Docker Compose, Java 23, Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, MinIO.
|
||||
|
||||
## Статические и unit-проверки
|
||||
|
||||
```text
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
docker compose --env-file .env.example config --quiet
|
||||
exit 0
|
||||
|
||||
gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' \
|
||||
--tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' \
|
||||
--tests '*StagingCleanerTest'
|
||||
BUILD SUCCESSFUL
|
||||
```
|
||||
|
||||
`ImageVariantProcessorTest` проверяет реальные JPEG/WebP signatures, ImageIO decode и размеры thumbnail/crop/optimized. `RetryPolicyTest` проверяет bounded exponential backoff и jitter.
|
||||
|
||||
## API и recovery
|
||||
|
||||
```text
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
```
|
||||
|
||||
Recovery test останавливает worker, получает `202 queued`, перезапускает API и RabbitMQ без удаления volumes, запускает worker и сверяет original SHA-256, thumbnail 256×160, crop 128×128, WebP signature. Повторная публикация того же `job_id` оставляет `ready` и ровно три variant rows.
|
||||
|
||||
## Доказанные окна отказа
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_fault_injection.py
|
||||
PASS: outbox commit/publish crash and result commit/ack crash recovered
|
||||
```
|
||||
|
||||
1. API завершается после COMMIT attachment/outbox до publish. Перезапущенный API republisher публикует сохранённое намерение; новый upload не создаётся вручную.
|
||||
2. Worker завершается после записи объектов и atomic ready COMMIT до ack. Redelivery завершает тот же effect; variant rows остаются уникальными.
|
||||
|
||||
## DLQ, replay и lease
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_dlq_replay_lease.py
|
||||
PASS: DLQ, guarded replay and expired lease recovery
|
||||
```
|
||||
|
||||
- malformed persistent JSON был permanent failure и увеличил `attachments.failed.v1` ровно на 1;
|
||||
- failed job replayed один раз, audit count = 1; повтор в queued/ready дал no-op;
|
||||
- после replay получены те же три variant names без второго набора;
|
||||
- искусственно истёкший processing lease был requeued, attempt увеличился, job дошёл до ready.
|
||||
|
||||
Секреты, session tokens и contents сообщений в evidence не сохранялись. Docker volumes в проверках не удалялись.
|
||||
@@ -0,0 +1,213 @@
|
||||
# Lab5, трек 5.2 — доказанная идемпотентность pipeline
|
||||
|
||||
Дата: 2026-11-19. Исходная точка: `75d2f78` + рабочее дерево с изменениями lab5.
|
||||
Стенд: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23 (контейнеры и Gradle
|
||||
toolchain), Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, Redis 7.4,
|
||||
MinIO `bitnamilegacy/minio:2025.4.22-debian-12-r1`.
|
||||
|
||||
Все пользователи, сообщения и изображения синтетические. Секреты, session tokens,
|
||||
содержимое сообщений и signed URL в evidence не сохранялись.
|
||||
|
||||
## Гипотеза и инварианты
|
||||
|
||||
Гипотеза: при versioned pipeline и fencing по `(attachment_id, pipeline_version,
|
||||
lease)` повторная доставка, перестановка сообщений, crash в окнах до/после side
|
||||
effect и устаревшая job не теряют подтверждённую работу и не заменяют результат
|
||||
новой версии старым.
|
||||
|
||||
Проверяемые инварианты:
|
||||
|
||||
1. `attachments.status='ready'` для активной версии означает ровно три variant
|
||||
rows этой версии (`thumbnail`, `crop`, `optimized`) и отсутствие
|
||||
`queued/processing` jobs у вложения.
|
||||
2. Для вложения существует не более одной активной (`queued/processing`) job.
|
||||
3. Ключ variant rows — `(attachment_id, pipeline_version, name)`; повторная
|
||||
запись не создаёт дубль.
|
||||
4. Объектные ключи версионированы: `thumbnail-vN.jpg`, `crop-vN.jpg`,
|
||||
`optimized-vN.webp`; `original` общий и неизменяемый. Поэтому v1 не может
|
||||
перезаписать v2.
|
||||
5. `attachments.pipeline_version` монотонно повышается только аудированной
|
||||
транзакцией `promote-pipeline`; старая `queued/processing` job получает
|
||||
`superseded` и `published=true`.
|
||||
6. Все подтверждённые тестом задачи (`status='ready'`) остаются `ready` после
|
||||
duplicate/reorder/stale доставок.
|
||||
7. В active version нет variant rows с `pipeline_version` больше
|
||||
`attachments.pipeline_version`.
|
||||
8. Одна и та же `pipeline_version` детерминирована; изменение профиля
|
||||
оформляется новой версией, потому что `ON CONFLICT DO NOTHING` не перезаписывает
|
||||
уже зафиксированные variant rows.
|
||||
|
||||
## Команды и результаты
|
||||
|
||||
```text
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
docker compose --env-file .env.example config --quiet
|
||||
exit 0
|
||||
|
||||
.\gradlew.bat test
|
||||
BUILD SUCCESSFUL (30 tests; full suite against live Compose PostgreSQL/Redis/MinIO)
|
||||
```
|
||||
|
||||
Для полного Gradle suite хранилища Compose не публикуют порты наружу; на время
|
||||
прогона использовались временные localhost-туннели к `postgres:5432`,
|
||||
`redis:6379`, `rabbitmq:5672`, `minio:9000`. После прогона туннели удалены.
|
||||
|
||||
### Основной эксперимент трека
|
||||
|
||||
```text
|
||||
python -X utf8 student_tests/lab5_pipeline_idempotency.py
|
||||
PASS: version fencing, superseded, duplicate/reorder and pre/post-side-effect crashes
|
||||
```
|
||||
|
||||
Тест автоматически:
|
||||
|
||||
1. создаёт v1 attachment, дожидается `ready` и снимает snapshot трёх variant rows;
|
||||
2. останавливает worker, повышает версию до v2 через `scripts/promote-pipeline`,
|
||||
публикует `v2, v2, v1` (порядок доставки не совпадает с порядком создания) и
|
||||
запускает worker;
|
||||
3. проверяет, что v2 дал отдельный результат, в БД ровно 6 rows (3 v1 + 3 v2),
|
||||
active version = 2, а повторные stale v1 / duplicate v2 не меняют snapshot;
|
||||
4. отдельно повышает ещё **queued** v1 job: она становится
|
||||
`superseded|published=true|superseded_by_v2`, v1 variant rows не появляются,
|
||||
v2 доходит до `ready` с тремя вариантами;
|
||||
5. вставляет crash до S3 side effect, после S3 side effect и после result
|
||||
COMMIT; каждый раз проверяет восстановление через lease/reaper/redelivery;
|
||||
6. проверяет, что все 5 подтверждённых вложений остались `ready`.
|
||||
|
||||
### Наблюдаемое состояние после прогона
|
||||
|
||||
Пример versioned-attachment (сокращённые ID приведены для читаемости):
|
||||
|
||||
```text
|
||||
attachment A (v1 -> v2, сначала завершён v1)
|
||||
active=2 ready
|
||||
job v1 ready published=t attempt=1
|
||||
job v2 ready published=t attempt=1
|
||||
variant keys: crop-v1, optimized-v1, thumbnail-v1,
|
||||
crop-v2, optimized-v2, thumbnail-v2
|
||||
|
||||
attachment B (queued v1 повышен до v2)
|
||||
active=2 ready
|
||||
job v1 superseded published=t last_error=superseded_by_v2
|
||||
job v2 ready published=t attempt=1
|
||||
variant keys: crop-v2, optimized-v2, thumbnail-v2 (v1 rows отсутствуют)
|
||||
```
|
||||
|
||||
SQL-инварианты по всем attachment, созданным 2026-11-19 в ходе финального
|
||||
прогона и регрессии lab5:
|
||||
|
||||
```text
|
||||
superseded_jobs|1
|
||||
today_promotions|9
|
||||
today_ready|55
|
||||
today_ready_bad_variants|0
|
||||
ready_with_nonterminal_job|0
|
||||
attachments_with_two_active_jobs|0
|
||||
duplicate_variant_keys|0
|
||||
stale_results_for_active_version|0
|
||||
```
|
||||
|
||||
Полный volume содержит также 3 attachment, созданных 2026-11-18 до миграции V7
|
||||
(тогда уровень 3 сохранял только original + thumbnail). Это исторические данные
|
||||
предыдущего стенда; они не являются результатом текущего versioned pipeline.
|
||||
Запросы инвариантов выше ограничены `created_at >= '2026-11-19'`; на чистом
|
||||
volume исторических строк нет.
|
||||
|
||||
### Точки отказа
|
||||
|
||||
| Fault | Где срабатывает | Наблюдение |
|
||||
| --- | --- | --- |
|
||||
| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker, до чтения staging/S3 PUT, `halt(84)` | 0 variant rows, 0 объектов; reaper возвращает job в `queued`, attempt увеличивается, задача доходит до `ready` |
|
||||
| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker, после 4 S3 PUT, до result COMMIT, `halt(85)` | objects=4, variant rows=0; повтор пишет те же deterministic keys и коммитит результат, objects остаётся 4 |
|
||||
| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker, после atomic `ready` COMMIT, до ack, `halt(86)` | redelivery видит `ready` и завершает no-op; 3 variant rows не дублируются |
|
||||
| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API, после COMMIT attachment/outbox, до publish, `halt(87)` | republisher находит `published=false` и публикует сохранённое намерение; задача доходит до `ready` |
|
||||
|
||||
Crash-точки включаются только переменными окружения и по умолчанию выключены;
|
||||
они предназначены для изолированного учебного стенда.
|
||||
|
||||
## Регрессия уровней 3–4
|
||||
|
||||
```text
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 student_tests/lab5_worker_recovery.py
|
||||
PASS: stopped worker, durable restart recovery and duplicate delivery
|
||||
|
||||
python -X utf8 student_tests/lab5_fault_injection.py
|
||||
PASS: outbox commit/publish crash and result commit/ack crash recovered
|
||||
|
||||
python -X utf8 student_tests/lab5_dlq_replay_lease.py
|
||||
PASS: DLQ, guarded replay and expired lease recovery
|
||||
|
||||
python -X utf8 student_tests/idempotency_concurrent.py
|
||||
PASS: 32 concurrent requests used 2 replicas and created one message
|
||||
|
||||
python -X utf8 student_tests/readiness_gate.py
|
||||
PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration
|
||||
|
||||
python -X utf8 student_tests/failover_each.py --action-script scripts/stop-one.cmd \
|
||||
--restore-script scripts/restore-api.cmd --window 15 \
|
||||
--output evidence/failover-each-2026-11-19.json
|
||||
(JSON: 2 replicas sequentially stopped and restored; confirmed writes preserved)
|
||||
|
||||
python -X utf8 student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \
|
||||
--duration 180 --concurrency 4 \
|
||||
--output evidence/rolling-update-2026-11-19.json
|
||||
(JSON: 4 observed replacement instance IDs; 335 successful writes, 3 transitional failures)
|
||||
|
||||
python -X utf8 student_tests/lab4_negative.py
|
||||
PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks
|
||||
|
||||
python -X utf8 student_tests/concurrent_upload_memory.py \
|
||||
--output evidence/upload-memory-2026-11-19.json
|
||||
PASS: concurrent upload memory experiment (12/12 uploads accepted, 2 API replicas)
|
||||
```
|
||||
|
||||
`readiness_gate.py` проверяет Google-подобный gate по обязательной схеме:
|
||||
`ReadinessService` теперь пробует обе таблицы — `message_idempotency` (lab3) и
|
||||
`attachments` (lab4/lab5). `concurrent_upload_memory.py` обновлён с lab4-контракта
|
||||
`201` на lab5-контракт `202`.
|
||||
|
||||
## Почему это at-least-once, а не exactly-once
|
||||
|
||||
RabbitMQ даёт at-least-once: сообщение может быть доставлено повторно после
|
||||
потери соединения или падения worker до ack; порядок между разными сообщениями
|
||||
не является бизнес-порядком. Поэтому duplicate и reorder — нормальные состояния,
|
||||
а не аномалии.
|
||||
|
||||
Наш pipeline делает **эффект** идемпотентным, но не делает доставку exactly-once:
|
||||
|
||||
- publisher confirm подтверждает только приём сообщения брокером, а consumer ack
|
||||
отправляется после PostgreSQL COMMIT; между S3 PUT и COMMIT нет распределённой
|
||||
транзакции;
|
||||
- crash после S3 PUT и до COMMIT оставляет объекты, а crash после COMMIT и до ack
|
||||
вызывает redelivery; оба окна специально инжектируются тестом;
|
||||
- повторная обработка может повторно выполнить S3 PUT, но deterministic keys,
|
||||
`ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` не создают
|
||||
второго набора результатов и не меняют active version;
|
||||
- stale job ограничена `pipeline_version` и lease в каждом переходе состояния,
|
||||
поэтому не может записать результат поверх новой версии.
|
||||
|
||||
Корректная формулировка: **at-least-once delivery + idempotent effect =
|
||||
effectively-once наблюдаемый результат для активной версии**. Exactly-once
|
||||
доставка потребовала бы транзакционной связи RabbitMQ, PostgreSQL и S3 (или
|
||||
эквивалентного распределённого протокола), которой в этой учебной архитектуре
|
||||
нет и которая не нужна для данных инвариантов.
|
||||
|
||||
## Границы
|
||||
|
||||
- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO;
|
||||
эксперимент не доказывает отказоустойчивость к потере host/диска.
|
||||
- Versioned S3 objects старой версии после promotion «в полёте» могут остаться
|
||||
orphan-объектами: active result они не перезаписывают, но отдельного GC для
|
||||
versioned objects нет (чистится только staging).
|
||||
- `superseded`/promotion — операторская SQL-транзакция через скрипт, не публичный
|
||||
HTTP API.
|
||||
- Числа failover/rolling в JSON — проверка сохранности и восстановления на одном
|
||||
Windows-хосте, не benchmark пропускной способности; latency включает
|
||||
особенность localhost dual-stack в Docker Desktop.
|
||||
- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант;
|
||||
на чистом volume и для новых данных действует инвариант трёх вариантов.
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T10:17:21.9038708Z",
|
||||
"queue_depth": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795083444.15,
|
||||
"5"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"oldest_job_age": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795083446.324,
|
||||
"139"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alerts": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"alerts": [
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "AttachmentQueueLagHigh",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "943917ff1db9",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "otel-collector-exported-metrics",
|
||||
"messaging_destination_name": "attachments.process.level4.v1",
|
||||
"otel_scope_name": "messenger-queue",
|
||||
"otel_scope_version": "1.0.0",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity": "warning",
|
||||
"slo": "attachment-time-to-ready"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "The durable worker queue has not drained within the SLO threshold.",
|
||||
"runbook": "infra/otel/runbooks.md#queue-lag",
|
||||
"summary": "Oldest attachment job is over 60 seconds old"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-19T10:16:07.684164819Z",
|
||||
"value": "1.39e+02"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"workload": "synthetic PNG attachment uploads",
|
||||
"requested": 5,
|
||||
"accepted": 5,
|
||||
"errors": 0,
|
||||
"elapsed_seconds": 1.063,
|
||||
"contains_credentials": false
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T10:18:32.6133979Z",
|
||||
"queue_depth": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795083514.639,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"oldest_job_age": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795083516.701,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"jobs": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"outcome": "success"
|
||||
},
|
||||
"value": [
|
||||
1795083518.741,
|
||||
"4.145557430652597"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T10:20:07.2547700Z",
|
||||
"api_5xx": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795083609.264,
|
||||
"0.15208333333333335"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"jdbc_spans_hint": "Grafana Explore -> Tempo: { resource.service.name = \"messenger-api\" && span.db.system.name = \"postgresql\" }",
|
||||
"correlated_logs_hint": "Grafana Explore -> Loki: {service_name=\"messenger-api\"} | json | level=\"ERROR\""
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
[
|
||||
{
|
||||
"at": "2026-11-19T10:19:31.3274597Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4537.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:19:36.9271045Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4098.9
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:19:42.0396820Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 3988.8
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:19:47.0377275Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4011.4
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:19:52.0526176Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4038.2
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:19:57.1033752Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4058.3
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T10:20:02.1668525Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4049.2
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,563 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "error",
|
||||
"exception_message": "Failed to obtain JDBC Connection",
|
||||
"exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j",
|
||||
"exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083580665709480",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "17",
|
||||
"severity_text": "ERROR",
|
||||
"span_id": "4e994b1e2de6ea26",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "c3307b04e636e33828d63bc789e863b8"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083580643209485",
|
||||
"Unexpected error occurred in scheduled task"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "error",
|
||||
"exception_message": "Failed to obtain JDBC Connection",
|
||||
"exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j",
|
||||
"exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083585672714591",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "17",
|
||||
"severity_text": "ERROR",
|
||||
"span_id": "36f0c3d0073685cf",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "8d45b62708d37ee1b98b38733d488365"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083585670851940",
|
||||
"Unexpected error occurred in scheduled task"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "error",
|
||||
"exception_message": "Failed to obtain JDBC Connection",
|
||||
"exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j",
|
||||
"exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083595680182050",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "17",
|
||||
"severity_text": "ERROR",
|
||||
"span_id": "3b18672eedc0d813",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "0e9e710a9decc517e4c19b26ca8976c6"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083595678179181",
|
||||
"Unexpected error occurred in scheduled task"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "error",
|
||||
"exception_message": "Failed to obtain JDBC Connection",
|
||||
"exception_stacktrace": "org.springframework.jdbc.CannotGetJdbcConnectionException: Failed to obtain JDBC Connection\n\tat org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:84)\n\tat org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.j",
|
||||
"exception_type": "org.springframework.jdbc.CannotGetJdbcConnectionException",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083600685613922",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "org.springframework.scheduling.support.TaskUtils$LoggingErrorHandler",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "17",
|
||||
"severity_text": "ERROR",
|
||||
"span_id": "1b7757646a541419",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "93100e12c7a8800d880867d4ffbe4a09"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083600683377175",
|
||||
"Unexpected error occurred in scheduled task"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"event": "queue_metrics_sample_failed",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083575542909551",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "5bb939bc64521236",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "7b47023dcfebb5738de24578b483931a"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083575541296003",
|
||||
"Queue telemetry sample failed: CannotGetJdbcConnectionException"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"event": "queue_metrics_sample_failed",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083590676105708",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "7ceec07d73695135",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "243ff15610e01771b8784440d0255eb5"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083590675921437",
|
||||
"Queue telemetry sample failed: CannotGetJdbcConnectionException"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"event": "queue_metrics_sample_failed",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083605690135533",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "com.example.mtusi_messenger.observability.QueueTelemetry",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "55d69d550d51484a",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "113194c82bf52dc7045c4ddcca64416e"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083605689816022",
|
||||
"Queue telemetry sample failed: CannotGetJdbcConnectionException"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083570548641327",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "com.zaxxer.hikari.pool.PoolBase",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "5bb939bc64521236",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "7b47023dcfebb5738de24578b483931a"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083570548127003",
|
||||
"HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@6908cf44 (This connection has been closed.). Possibly consider using a shorter maxLifetime value."
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"container_id": "943917ff1db9c7c2ee0caaccbec76b2a2a705ac23f4d992b7d50929c4c8feb62",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"flags": "3",
|
||||
"host_arch": "amd64",
|
||||
"host_name": "943917ff1db9",
|
||||
"observed_timestamp": "1795083570555957813",
|
||||
"os_description": "Linux 5.15.167.4-microsoft-standard-WSL2",
|
||||
"os_type": "linux",
|
||||
"os_version": "5.15.167.4-microsoft-standard-WSL2",
|
||||
"process_command_args": "[\"/opt/java/openjdk/bin/java\",\"-javaagent:/opt/opentelemetry-javaagent.jar\",\"-jar\",\"/app/app.jar\"]",
|
||||
"process_executable_path": "/opt/java/openjdk/bin/java",
|
||||
"process_pid": "8",
|
||||
"process_runtime_description": "Eclipse Adoptium OpenJDK 64-Bit Server VM 23.0.2+7",
|
||||
"process_runtime_name": "OpenJDK Runtime Environment",
|
||||
"process_runtime_version": "23.0.2+7",
|
||||
"scope_name": "com.zaxxer.hikari.pool.PoolBase",
|
||||
"service_instance_id": "943917ff1db9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "5bb939bc64521236",
|
||||
"telemetry_distro_name": "opentelemetry-java-instrumentation",
|
||||
"telemetry_distro_version": "2.33.1",
|
||||
"telemetry_sdk_language": "java",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.67.0",
|
||||
"trace_id": "7b47023dcfebb5738de24578b483931a"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795083570555778998",
|
||||
"HikariPool-1 - Failed to validate connection org.postgresql.jdbc.PgConnection@51c569d9 (This connection has been closed.). Possibly consider using a shorter maxLifetime value."
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 40976,
|
||||
"linesProcessedPerSecond": 136,
|
||||
"totalBytesProcessed": 2703,
|
||||
"totalLinesProcessed": 9,
|
||||
"execTime": 0.065965,
|
||||
"queueTime": 0.00129,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 9,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 9,
|
||||
"totalStructuredMetadataBytesProcessed": 1992
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"queryUsedV2Engine": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0,
|
||||
"dataobj": {
|
||||
"prePredicateDecompressedRows": 0,
|
||||
"prePredicateDecompressedBytes": 0,
|
||||
"prePredicateDecompressedStructuredMetadataBytes": 0,
|
||||
"postPredicateRows": 0,
|
||||
"postPredicateDecompressedBytes": 0,
|
||||
"postPredicateStructuredMetadataBytes": 0,
|
||||
"postFilterRows": 0,
|
||||
"pagesScanned": 0,
|
||||
"pagesDownloaded": 0,
|
||||
"pagesDownloadedBytes": 0,
|
||||
"pageBatches": 0,
|
||||
"totalRowsAvailable": 0,
|
||||
"totalPageDownloadTime": 0
|
||||
}
|
||||
},
|
||||
"querierExecTime": 0.047749
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 1,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 9,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"queryUsedV2Engine": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 2703,
|
||||
"headChunkLines": 9,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 9,
|
||||
"headChunkStructuredMetadataBytes": 1992,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 492747,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0,
|
||||
"dataobj": {
|
||||
"prePredicateDecompressedRows": 0,
|
||||
"prePredicateDecompressedBytes": 0,
|
||||
"prePredicateDecompressedStructuredMetadataBytes": 0,
|
||||
"postPredicateRows": 0,
|
||||
"postPredicateDecompressedBytes": 0,
|
||||
"postPredicateStructuredMetadataBytes": 0,
|
||||
"postFilterRows": 0,
|
||||
"pagesScanned": 0,
|
||||
"pagesDownloaded": 0,
|
||||
"pagesDownloadedBytes": 0,
|
||||
"pageBatches": 0,
|
||||
"totalRowsAvailable": 0,
|
||||
"totalPageDownloadTime": 0
|
||||
}
|
||||
},
|
||||
"recvWaitTime": 0.008569
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0,
|
||||
"usedBloomFilters": false,
|
||||
"totalStreams": 0,
|
||||
"chunkRefsLookupTime": 0,
|
||||
"bloomFilterTime": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
recovered
|
||||
@@ -0,0 +1,697 @@
|
||||
{
|
||||
"traces": [
|
||||
{
|
||||
"traceID": "2f7370e029e3f7a16b43878d68ae8e20",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083242196014220",
|
||||
"durationMs": 32
|
||||
},
|
||||
{
|
||||
"traceID": "4896829563bf7a106ee8d143ac84832e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083239923257437",
|
||||
"durationMs": 6
|
||||
},
|
||||
{
|
||||
"traceID": "198b1479aed61af14a84c7b7a2ed9b1e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083239570603967",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "21502c0de7d46dc0269c8ba624a6c23b",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083233700846631",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "3410d7364dba96b2aacfc2aadfea97be",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083229579273036"
|
||||
},
|
||||
{
|
||||
"traceID": "4c35f7b7fc7e631b7b6ba39f98c579cf",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083229565470275",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "20db1bec473cb94e969018b15be1a01f",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083229470223000",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "1552c3687c6de569229b530f24a4e3f1",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083223342807944",
|
||||
"durationMs": 9
|
||||
},
|
||||
{
|
||||
"traceID": "2b55cb82eb4404af16918e5a5d0cdd57",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083219563846030",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "21224d7bdfd6538117436f0d7d894709",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083219401117603",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "1ef30879f867c99a5f768bf2d67de7c4",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083214510024043",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "a223dd910754a9dd2d82e7bdb4424dc",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083214497015475",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "90dae0d5256b8b84bfd2a8339d6be34",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083214398206478",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "4cd4879aceb0549998b84df6f5d778bd",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083210923828917",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "28e809860332b12c9835ccf41dddc01e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083209509201945"
|
||||
},
|
||||
{
|
||||
"traceID": "52d228b9188561777ea6de6c84f60b96",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083204495117245"
|
||||
},
|
||||
{
|
||||
"traceID": "1317c6519ef468e6012b6d2791feea2d",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083202718212905",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "2f77661745383a05a210a31eeb10e9d3",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083194505678372",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "34612903902c205b2c8493baf36d8533",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083194492176788",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "29d3aa01326c8a54ddc239c54c429cff",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083192125513698",
|
||||
"durationMs": 6
|
||||
},
|
||||
{
|
||||
"traceID": "4b88e4110e4db4e24c776b7f446c4e7a",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083190034891922",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "34e785064a8dc9639a8a8f892adbde3c",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083189490866815"
|
||||
},
|
||||
{
|
||||
"traceID": "12e6eb2f7dc204f7cd4faf2dedd0c126",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083187892313453",
|
||||
"durationMs": 13
|
||||
},
|
||||
{
|
||||
"traceID": "235564101cd94c4ee16ce1f6e61e6573",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083184436384728",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "557b661ca4b8a55da2b22488d5d1a76e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083184423130530"
|
||||
},
|
||||
{
|
||||
"traceID": "233215abd278bff45d2750c745563535",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083183754090813",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "7009d65bdcb674eadd9b54f30c40307",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083179425828960",
|
||||
"durationMs": 10
|
||||
},
|
||||
{
|
||||
"traceID": "39ca7029a7af8f73cdbfdc6dd957a638",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083177461848482",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "396d27a55614c7e6cc8e929a8e6642b7",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083174419724867"
|
||||
},
|
||||
{
|
||||
"traceID": "469f72453a60129e164dc2e9e21ed9f2",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083173349996485",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "21524c6bad04c5393ed93d1b32a7ddfb",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083171300590874",
|
||||
"durationMs": 7
|
||||
},
|
||||
{
|
||||
"traceID": "35d793b9eb0d9671db56664143632461",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083169240643784",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "2d189acf2529e1e5070a34283fc27aab",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083167170951206",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "3510522263a30ca9821d3282fa84df6e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083163056985121",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "20e8f9da366714e56573936da9821eee",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083159419911892",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "4684c71727338fddbcc92e81c3b66fde",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083159247657934",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "19b2c69c87cfcbcb7fba87cf79f6e056",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083154350059891",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "4edb69a63d986c07bb80252c1e3756fd",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083154263143826",
|
||||
"durationMs": 6
|
||||
},
|
||||
{
|
||||
"traceID": "47ba154a890c3cb5eadd06e5c2177290",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "messenger",
|
||||
"startTimeUnixNano": "1795083154084280265"
|
||||
},
|
||||
{
|
||||
"traceID": "3c443e3d5e4b067ebe65209271d29870",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083145901013447",
|
||||
"durationMs": 32
|
||||
},
|
||||
{
|
||||
"traceID": "b105a97cc15b8dbd1beab3907e064ec",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083144348529368"
|
||||
},
|
||||
{
|
||||
"traceID": "3d332022d66852a32492b2bb50e4a49c",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083139347599576"
|
||||
},
|
||||
{
|
||||
"traceID": "32aeaf15c5816848ead7dc9cb0d317d2",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083134345037608",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "307377d165adba665b634e5a787f8faa",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083133489684589",
|
||||
"durationMs": 10
|
||||
},
|
||||
{
|
||||
"traceID": "fd53f3fac974e14a9b0bc6b7fb2b3da",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083129344879817",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "1eae20965992f2a3b00292e96763a1b7",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083125000318817",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "52ae20055b7d94742c82e7e84a38e735",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083114655214727",
|
||||
"durationMs": 7
|
||||
},
|
||||
{
|
||||
"traceID": "f9c0c2521f8b128e6ad9267eaea7c31",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083114270828828",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "553567508997161c44dde4db81e9e7a1",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083114156439582",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "52a46cc6567688c8f1a61f4f2da6b7b6",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083112579003220",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "20609217d4e9741a70699a93d47ee262",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083109270416562"
|
||||
},
|
||||
{
|
||||
"traceID": "1a21f5d8b71f0f85cf09f3ea38b9e14e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083108445836642",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "444b20682dddb90d8f199c5706753a9",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083106389151714",
|
||||
"durationMs": 7
|
||||
},
|
||||
{
|
||||
"traceID": "3d802498b048b6c57522373e84d66500",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083104269589692"
|
||||
},
|
||||
{
|
||||
"traceID": "2913d2fde17689ac2914f12dbbd50bc1",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083104151632742",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "228511d2e5d949660507d602f0ddef90",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083102279505914",
|
||||
"durationMs": 9
|
||||
},
|
||||
{
|
||||
"traceID": "1b12ae26787f10f67057fc1015c31917",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083099267378928",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "43062e16f5b379215b34b5bb2fc2e407",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083098068647506",
|
||||
"durationMs": 11
|
||||
},
|
||||
{
|
||||
"traceID": "3d8bbe120b5653959c4dabed68f8cbbc",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083094201070242"
|
||||
},
|
||||
{
|
||||
"traceID": "1feda03af32692e2cb8960e623e387f9",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083094198945116",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "2e0cffc7ec23628b6002ed62b1bfc794",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083093973006592",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "3d443a0cd84aacb6a25f3988c02c0b22",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083087825041036",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "3c05f026d5daa80a273f11e2d1f15a1b",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083084195702312",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "3377fff79205c50913da9523e86cf267",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083081595922893",
|
||||
"durationMs": 9
|
||||
},
|
||||
{
|
||||
"traceID": "26186c174d50642629713908cc0b22f7",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083079195725678"
|
||||
},
|
||||
{
|
||||
"traceID": "fab56b55c29cc155c83aad521a338fe",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083074193776114",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "50e8e14ee65caf9671d55d7436644ad6",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083073356737601",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "2ec40aa182ea167e2f5a21f958c4ef94",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083071308395569",
|
||||
"durationMs": 6
|
||||
},
|
||||
{
|
||||
"traceID": "1ac3d4136a09f7f4bf756f6846bb2ccb",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083069216469144",
|
||||
"durationMs": 8
|
||||
},
|
||||
{
|
||||
"traceID": "1b5e3b8f90874b5011b7b12483238ec2",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083067098088469",
|
||||
"durationMs": 8
|
||||
},
|
||||
{
|
||||
"traceID": "ab2a39ce37c7fd72a20f377cdfb284b",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083065014158531",
|
||||
"durationMs": 8
|
||||
},
|
||||
{
|
||||
"traceID": "1cf32cac103b8da37d0a36b742a4c0ba",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083064125248315",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "18201f914e73079877fe4eaecf08f654",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083064003875969",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "2a7711fe81236abc4a73fe4a6e8ddb40",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083062950592934",
|
||||
"durationMs": 7
|
||||
},
|
||||
{
|
||||
"traceID": "460a4763151dd74b518c962f0ea45435",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083059124764109",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "25c3e39f4b20e8d0d3cb078ff5df2ad7",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083058999411000",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "3ed3706b294ebabc65ef639c6040c067",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083058731005797",
|
||||
"durationMs": 16
|
||||
},
|
||||
{
|
||||
"traceID": "7d9a5a43868eaf91e3e3bc229126d94",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083054490241800",
|
||||
"durationMs": 10
|
||||
},
|
||||
{
|
||||
"traceID": "106606ab7649a3687c746c4522500278",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083054115882944",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "441c134a0bd1992ddfe2e444fa00a396",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083049115458312",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "2166253a92c073ec9134f2bdd629b69",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083049113615406",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "1bf168c1495a1aed48983e09c32ad8fd",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083048185194380",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "271d3c0780a876ba064fb3e4ee5b5277",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083046116869717",
|
||||
"durationMs": 6
|
||||
},
|
||||
{
|
||||
"traceID": "16cb9a3288e21109cff7ecbcc24b985d",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083039109138417",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "2c5c1ca5cc5238040df8864a052850a0",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083033910282803",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "3805ef57104cfb8eb7d4463ea9285ed8",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083033473941705",
|
||||
"durationMs": 14
|
||||
},
|
||||
{
|
||||
"traceID": "56075bd17bad6bf5d7da94e0c27d4f1e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083029041703172",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "480532461c728593952b19efd3ebee14",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083028967837404",
|
||||
"durationMs": 265
|
||||
},
|
||||
{
|
||||
"traceID": "25f19f85951133faaec5ca7de7967394",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083028936172779",
|
||||
"durationMs": 105
|
||||
},
|
||||
{
|
||||
"traceID": "2437cad488ed6ea0b9a8633d82205c4",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083025696649794",
|
||||
"durationMs": 4
|
||||
},
|
||||
{
|
||||
"traceID": "49e629dcff8000c3ffc9c6850b7e3a6d",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083023439629709",
|
||||
"durationMs": 8
|
||||
},
|
||||
{
|
||||
"traceID": "2e684b669291ea59269620d2d49f1939",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083021062422221",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "23daf7d8e1d72a77599a9a4c3ad0a2bd",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083018975339309",
|
||||
"durationMs": 11
|
||||
},
|
||||
{
|
||||
"traceID": "3e2c8a85df9d466ed068290a341eaceb",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083018949869044",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "3a251a6694c7e66a5486f75866ce0b11",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083017702814031",
|
||||
"durationMs": 3
|
||||
},
|
||||
{
|
||||
"traceID": "2f3ffbe6bcba1490803df66b72bb6351",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083009553293092",
|
||||
"durationMs": 5
|
||||
},
|
||||
{
|
||||
"traceID": "3b2652b81e65bceffc7308804b6e5b3d",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "QueueTelemetry.sample",
|
||||
"startTimeUnixNano": "1795083007643550929",
|
||||
"durationMs": 2
|
||||
},
|
||||
{
|
||||
"traceID": "31ddb25055f8bcde77faa3d25717c728",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "GET /health/ready",
|
||||
"startTimeUnixNano": "1795083007413274745",
|
||||
"durationMs": 9
|
||||
},
|
||||
{
|
||||
"traceID": "1863383c2292bd14bbec60f475b2e2c0",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "UnpublishedJobRepublisher.publishPending",
|
||||
"startTimeUnixNano": "1795083003890520986",
|
||||
"durationMs": 1
|
||||
},
|
||||
{
|
||||
"traceID": "30b2935430bc98695edd4770944f602e",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795083003869406925",
|
||||
"durationMs": 2
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"inspectedTraces": 333,
|
||||
"inspectedBytes": "27485",
|
||||
"completedJobs": 1,
|
||||
"totalJobs": 3
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"workload": "synthetic PNG attachment uploads",
|
||||
"requested": 1,
|
||||
"accepted": 1,
|
||||
"errors": 0,
|
||||
"elapsed_seconds": 0.181,
|
||||
"contains_credentials": false
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:19:25.6100693+00:00",
|
||||
"trace_id": "a3dc1f8fabc7dc732fc3f502007f7654",
|
||||
"retry_event": "attachment_job_retry_scheduled",
|
||||
"retry_attempt": "2",
|
||||
"retry_error_code": "processing_failed",
|
||||
"completion_event": "attachment_job_completed",
|
||||
"producer_spans": 3,
|
||||
"consumer_spans": 3,
|
||||
"same_trace": true,
|
||||
"synthetic_data_only": true,
|
||||
"spans": [
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "POST /api/v1/chats/{chatId}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"span_id": "FnoyHc+3lfY=",
|
||||
"parent_span_id": null
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT messenger.sessions",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "AwIhXSCtyas=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT messenger.users",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "HcCKR1DIetM=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "1j1NfZtuv3Q=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT messenger.chats",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "0FENqd5y7Xg=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "SELECT messenger.chat_members",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "fDDLjgAtoiQ=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "NMtDSJHphqU=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "INSERT messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "X6SdQUgT6n8=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "JW9UyNpvDLc=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "oJlbvd5/myM=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "attachments.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "h977yFuU9zE=",
|
||||
"parent_span_id": "FnoyHc+3lfY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "TqQIfXHlizY=",
|
||||
"parent_span_id": "h977yFuU9zE="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "nIg4Oa+5lKY=",
|
||||
"parent_span_id": "csh2DBq5I10="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "attachment.process.v1 process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "Bfy8+s/9Unk=",
|
||||
"parent_span_id": "vcCIFAFUBAM="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "V7Aeg6wlcMA=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "vxPbe/6D3BQ=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "HJI00k8Q2B4=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "4XxYraggMpU=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "AoRNFcpE608=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "J2nUmKPyp8A=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "7WwtV5gNGjs=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "ny8zFeqRljY=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "WpX199HTrxw=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "uUmkZKxHWk0=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "CzRA6Jtx9ek=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "URqheSsGi2c=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "vAsiDivIg8A=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "fxdp+Lm8MRg=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "QMAfoOiUphI=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "ENOwjZ3sLF4=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "1J5W4nRH74U=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "EUzGj0Wk+mM=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "DbH/1dk+6Bs=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "bZfJhnK++2A=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "HKLGaPt22w0=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "2Q87CMlxKlw=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "MnDc4GLrCGY=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "OV/6iY4llYI=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "SELECT messenger",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "EZd5UKV/Kqk=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachment_jobs",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "Oi+6TrQwblY=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT messenger.attachment_variants",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "h7/DjtpoNvk=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT messenger.attachment_variants",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "qqugt2fdBzY=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "INSERT messenger.attachment_variants",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "1CgS88Bdu2A=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "UPDATE messenger.attachments",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "aA89RdGHL5I=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "VX1lxYQCfT4=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "UeAXa0EhHpo=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "E7t66VFyg6Q=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "PUT",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "CvEetnQTWlI=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-worker",
|
||||
"name": "DELETE",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"span_id": "TpQJoEtLY7o=",
|
||||
"parent_span_id": "Bfy8+s/9Unk="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "attachments.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "csh2DBq5I10=",
|
||||
"parent_span_id": "TqQIfXHlizY="
|
||||
},
|
||||
{
|
||||
"service": "messenger-api",
|
||||
"name": "attachments.v1 publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "vcCIFAFUBAM=",
|
||||
"parent_span_id": "nIg4Oa+5lKY="
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:31:32.4454018Z",
|
||||
"queue_depth": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795105892.467,
|
||||
"3"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"oldest_job_age": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795105892.476,
|
||||
"138"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alerts": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"alerts": [
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "AttachmentQueueLagHigh",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "5f24a6dc-7220-429d-a093-d19540a77484",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "otel-collector-exported-metrics",
|
||||
"messaging_destination_name": "attachments.process.level4.v1",
|
||||
"otel_scope_name": "messenger-queue",
|
||||
"otel_scope_version": "1.0.0",
|
||||
"service_instance_id": "5f24a6dc-7220-429d-a093-d19540a77484",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "0.0.1",
|
||||
"severity": "warning",
|
||||
"slo": "attachment-time-to-ready"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "The durable worker queue has not drained within the SLO threshold.",
|
||||
"runbook": "infra/otel/runbooks.md#queue-lag",
|
||||
"summary": "Oldest attachment job is over 60 seconds old"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-19T16:30:27.684164819Z",
|
||||
"value": "1.28e+02"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alert_firing": true
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"workload": "synthetic PNG attachment uploads",
|
||||
"requested": 3,
|
||||
"accepted": 3,
|
||||
"errors": 0,
|
||||
"elapsed_seconds": 0.597,
|
||||
"contains_credentials": false
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:32:43.2643478Z",
|
||||
"queue_depth": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795105963.253,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"oldest_job_age": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795105963.262,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"jobs": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"outcome": "retry"
|
||||
},
|
||||
"value": [
|
||||
1795105963.273,
|
||||
"0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"outcome": "success"
|
||||
},
|
||||
"value": [
|
||||
1795105963.273,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alerts": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"alerts": []
|
||||
}
|
||||
},
|
||||
"alert_resolved": true
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:36:47.0833934Z",
|
||||
"api_5xx": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795106207.063,
|
||||
"0.5846403015440146"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alerts": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"alerts": [
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "MessengerApiHighErrorRate",
|
||||
"severity": "warning",
|
||||
"slo": "api-availability"
|
||||
},
|
||||
"annotations": {
|
||||
"description": "Five-minute 5xx ratio is above 5% for two minutes.",
|
||||
"runbook": "infra/otel/runbooks.md#api-high-error-rate",
|
||||
"summary": "Messenger API 5xx ratio exceeds 5%"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-19T16:34:37.684164819Z",
|
||||
"value": "9.644768265300965e-01"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"alert_firing": true,
|
||||
"jdbc_spans_hint": "Grafana Explore -> Tempo: { resource.service.name = \"messenger-api\" && span.db.system.name = \"postgresql\" }",
|
||||
"correlated_logs_hint": "Grafana Explore -> Loki: {service_name=\"messenger-api\"} | json | level=\"ERROR\""
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
[
|
||||
{
|
||||
"at": "2026-11-19T16:34:00.5253128Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 401,
|
||||
"duration_ms": 4240.2
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:05.8085003Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4040.7
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:10.8602421Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4034.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:15.9061557Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4035.0
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:20.9495283Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4019.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:25.9764025Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4018.2
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:31.0048332Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4024.0
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:36.0394294Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4033.6
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:41.0840646Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4028.6
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:46.1268110Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4076.1
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:51.2134687Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4013.2
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:34:56.2358974Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4022.9
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:01.2714634Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4029.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:06.3066502Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4023.7
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:11.3429193Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4030.3
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:16.3792975Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4031.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:21.4180181Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4013.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:26.4479913Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4025.9
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:31.4865276Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4028.5
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:36.5299328Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4026.3
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:41.5598609Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4026.4
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:46.5956509Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4023.2
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:51.6212244Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4012.7
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:35:56.6416286Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4030.0
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:01.6791804Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4022.7
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:06.7040114Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4021.4
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:11.7306456Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4029.9
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:16.7621800Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4028.0
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:21.8035195Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4014.1
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:26.8302112Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4018.8
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:31.8595958Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4027.3
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:36.8893717Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4029.3
|
||||
},
|
||||
{
|
||||
"at": "2026-11-19T16:36:41.9284299Z",
|
||||
"ready_status": 503,
|
||||
"business_status": 503,
|
||||
"duration_ms": 4032.4
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:41:55.2729486+00:00",
|
||||
"readiness_recovered": true,
|
||||
"alerts": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"alerts": []
|
||||
}
|
||||
},
|
||||
"alert_resolved": true,
|
||||
"api_5xx_ratio": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795106515.309,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,361 @@
|
||||
{
|
||||
"traces": [
|
||||
{
|
||||
"traceID": "71d67516856cb51d3fd6187fdf2817f",
|
||||
"rootServiceName": "messenger-worker",
|
||||
"rootTraceName": "messenger",
|
||||
"startTimeUnixNano": "1795106139953505266",
|
||||
"durationMs": 1,
|
||||
"spanSet": {
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "03631e0ae3ce65a9",
|
||||
"startTimeUnixNano": "1795106139953505266",
|
||||
"durationNanos": "1393486",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
},
|
||||
"spanSets": [
|
||||
{
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "03631e0ae3ce65a9",
|
||||
"startTimeUnixNano": "1795106139953505266",
|
||||
"durationNanos": "1393486",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
}
|
||||
],
|
||||
"serviceStats": {
|
||||
"messenger-worker": {
|
||||
"spanCount": 1,
|
||||
"errorCount": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceID": "cc2248aee1659a6529b09fd076ff8e02",
|
||||
"rootServiceName": "messenger-worker",
|
||||
"rootTraceName": "messenger",
|
||||
"startTimeUnixNano": "1795106052953980568",
|
||||
"durationMs": 1,
|
||||
"spanSet": {
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "7ffc924466fe3b7f",
|
||||
"startTimeUnixNano": "1795106052953980568",
|
||||
"durationNanos": "1763744",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
},
|
||||
"spanSets": [
|
||||
{
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "7ffc924466fe3b7f",
|
||||
"startTimeUnixNano": "1795106052953980568",
|
||||
"durationNanos": "1763744",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
}
|
||||
],
|
||||
"serviceStats": {
|
||||
"messenger-worker": {
|
||||
"spanCount": 1,
|
||||
"errorCount": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceID": "9e389058dfdf137e833785ddbb0d1f6f",
|
||||
"rootServiceName": "messenger-worker",
|
||||
"rootTraceName": "messenger",
|
||||
"startTimeUnixNano": "1795106043478695215",
|
||||
"spanSet": {
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "c89fe13bd4a61fa4",
|
||||
"startTimeUnixNano": "1795106043478695215",
|
||||
"durationNanos": "953569",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
},
|
||||
"spanSets": [
|
||||
{
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "c89fe13bd4a61fa4",
|
||||
"startTimeUnixNano": "1795106043478695215",
|
||||
"durationNanos": "953569",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
}
|
||||
],
|
||||
"serviceStats": {
|
||||
"messenger-worker": {
|
||||
"spanCount": 1,
|
||||
"errorCount": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceID": "6a35aedc38417630096d551d8d504d6e",
|
||||
"rootServiceName": "messenger-worker",
|
||||
"rootTraceName": "messenger",
|
||||
"startTimeUnixNano": "1795106041627390774",
|
||||
"durationMs": 18,
|
||||
"spanSet": {
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "d68a32afe63bf2f2",
|
||||
"startTimeUnixNano": "1795106041627390774",
|
||||
"durationNanos": "18820885",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
},
|
||||
"spanSets": [
|
||||
{
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "d68a32afe63bf2f2",
|
||||
"startTimeUnixNano": "1795106041627390774",
|
||||
"durationNanos": "18820885",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 1
|
||||
}
|
||||
],
|
||||
"serviceStats": {
|
||||
"messenger-worker": {
|
||||
"spanCount": 1,
|
||||
"errorCount": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceID": "696ba634a1655319ddc27bdcd88edb88",
|
||||
"rootServiceName": "messenger-api",
|
||||
"rootTraceName": "ExpiredLeaseReaper.recoverExpired",
|
||||
"startTimeUnixNano": "1795106040066361178",
|
||||
"durationMs": 5017,
|
||||
"spanSet": {
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "bb6cc6ecea134b52",
|
||||
"startTimeUnixNano": "1795106040066669462",
|
||||
"durationNanos": "33951674",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"spanID": "27022a80d9f3f062",
|
||||
"startTimeUnixNano": "1795106040104844852",
|
||||
"durationNanos": "1526278",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 2
|
||||
},
|
||||
"spanSets": [
|
||||
{
|
||||
"spans": [
|
||||
{
|
||||
"spanID": "bb6cc6ecea134b52",
|
||||
"startTimeUnixNano": "1795106040066669462",
|
||||
"durationNanos": "33951674",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"spanID": "27022a80d9f3f062",
|
||||
"startTimeUnixNano": "1795106040104844852",
|
||||
"durationNanos": "1526278",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status",
|
||||
"value": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"matched": 2
|
||||
}
|
||||
],
|
||||
"serviceStats": {
|
||||
"messenger-api": {
|
||||
"spanCount": 3,
|
||||
"errorCount": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"inspectedBytes": "280709",
|
||||
"completedJobs": 3,
|
||||
"totalJobs": 3
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
[
|
||||
{
|
||||
"captured_at": "2026-11-19T16:47:45.9583664+00:00",
|
||||
"collector_memory": "105.9MiB",
|
||||
"collector_memory_bytes": 111044198,
|
||||
"collector_memory_percent": "20.68%",
|
||||
"exporter_queue_size": 0.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:01.7530524+00:00",
|
||||
"collector_memory": "105.9MiB",
|
||||
"collector_memory_bytes": 111044198,
|
||||
"collector_memory_percent": "20.69%",
|
||||
"exporter_queue_size": 2.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:08.4652103+00:00",
|
||||
"collector_memory": "105.9MiB",
|
||||
"collector_memory_bytes": 111044198,
|
||||
"collector_memory_percent": "20.69%",
|
||||
"exporter_queue_size": 4.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:15.1961009+00:00",
|
||||
"collector_memory": "106.1MiB",
|
||||
"collector_memory_bytes": 111253914,
|
||||
"collector_memory_percent": "20.71%",
|
||||
"exporter_queue_size": 11.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:21.8707908+00:00",
|
||||
"collector_memory": "110.7MiB",
|
||||
"collector_memory_bytes": 116077363,
|
||||
"collector_memory_percent": "21.61%",
|
||||
"exporter_queue_size": 11.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:28.5455460+00:00",
|
||||
"collector_memory": "114.7MiB",
|
||||
"collector_memory_bytes": 120271667,
|
||||
"collector_memory_percent": "22.40%",
|
||||
"exporter_queue_size": 20.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:35.2646871+00:00",
|
||||
"collector_memory": "120.8MiB",
|
||||
"collector_memory_bytes": 126667981,
|
||||
"collector_memory_percent": "23.59%",
|
||||
"exporter_queue_size": 30.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:42.0621346+00:00",
|
||||
"collector_memory": "137MiB",
|
||||
"collector_memory_bytes": 143654912,
|
||||
"collector_memory_percent": "26.77%",
|
||||
"exporter_queue_size": 30.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 0.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:48.8023730+00:00",
|
||||
"collector_memory": "139MiB",
|
||||
"collector_memory_bytes": 145752064,
|
||||
"collector_memory_percent": "27.14%",
|
||||
"exporter_queue_size": 37.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 35.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:48:55.5413710+00:00",
|
||||
"collector_memory": "139.3MiB",
|
||||
"collector_memory_bytes": 146066637,
|
||||
"collector_memory_percent": "27.20%",
|
||||
"exporter_queue_size": 47.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 35.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:02.2702013+00:00",
|
||||
"collector_memory": "157.4MiB",
|
||||
"collector_memory_bytes": 165045862,
|
||||
"collector_memory_percent": "30.74%",
|
||||
"exporter_queue_size": 53.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 1805.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:08.9583898+00:00",
|
||||
"collector_memory": "162.5MiB",
|
||||
"collector_memory_bytes": 170393600,
|
||||
"collector_memory_percent": "31.74%",
|
||||
"exporter_queue_size": 53.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 1805.0,
|
||||
"failed_logs_total": 0.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:15.6767766+00:00",
|
||||
"collector_memory": "162.8MiB",
|
||||
"collector_memory_bytes": 170708173,
|
||||
"collector_memory_percent": "31.79%",
|
||||
"exporter_queue_size": 61.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 2852.0,
|
||||
"failed_logs_total": 318.0
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:22.4259651+00:00",
|
||||
"collector_memory": "165.6MiB",
|
||||
"collector_memory_bytes": 173644186,
|
||||
"collector_memory_percent": "32.35%",
|
||||
"exporter_queue_size": 66.0,
|
||||
"exporter_queue_capacity": 2048.0,
|
||||
"failed_spans_total": 3364.0,
|
||||
"failed_logs_total": 926.0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:22.5235902+00:00",
|
||||
"make_test_passed": true,
|
||||
"message_load_exit_code": 0,
|
||||
"successful_message_requests": 3060,
|
||||
"failed_message_requests": 0,
|
||||
"collector_memory_limit_bytes": 536870912,
|
||||
"collector_memory_limiter_bytes": 402653184,
|
||||
"max_collector_memory_bytes": 173644186,
|
||||
"max_exporter_queue_size": 66,
|
||||
"exporter_queue_capacity": 2048,
|
||||
"failed_spans_total": 3364,
|
||||
"failed_log_records_total": 926,
|
||||
"bounded": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python tests/smoke.py --base-url "http://127.0.0.1:18080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 75.0,
|
||||
"elapsed_seconds": 75.186,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 3060,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 40.699,
|
||||
"successful_latency_ms": {
|
||||
"p50": 98.635,
|
||||
"p95": 183.436,
|
||||
"p99": 198.298
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:50:50.5393078+00:00",
|
||||
"loki_ready": true,
|
||||
"loki_http_status": 200,
|
||||
"tempo_ready": true,
|
||||
"tempo_http_status": 200,
|
||||
"exporter_queue_size": 0,
|
||||
"recovery_confirmed": true
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T16:49:54.2198516+00:00",
|
||||
"loki_ready": false,
|
||||
"tempo_ready": true,
|
||||
"exporter_queue_size": 0.0
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# Observability cost — 19.11.2026
|
||||
|
||||
Команда:
|
||||
|
||||
```powershell
|
||||
powershell -File scripts/experiment-observability-cost.ps1 `
|
||||
-BaseUrl http://127.0.0.1:18080 -LoadSeconds 45 -Concurrency 4 `
|
||||
-WarmupSeconds 10 -StatsIntervalSeconds 5
|
||||
```
|
||||
|
||||
Режимы выполнялись последовательно: telemetry off; full 100%/10s; economy 5%/60s; balanced 25%/30s. Перед каждым режимом API и worker пересоздавались, затем выполнялся одинаковый прогрев. `summary.json` и `comparison.csv` содержат итог; `*-result.json`, `*-load.json` и `*-resource-samples.json` — raw данные. Ingest измерен дельтами Collector/Tempo/Loki/Prometheus counters, CPU/RAM — `docker stats`.
|
||||
|
||||
| mode | RPS | p50/p95/p99 ms | trace retention vs full | backend B/request | Collector lost spans |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| off | 40.547 | 97.186 / 198.133 / 275.316 | 0% | 0 | 0 |
|
||||
| full | 22.419 | 185.241 / 315.289 / 405.961 | 100% | 6918.72 | 0 |
|
||||
| economy | 14.622 | 287.141 / 405.639 / 502.152 | 5.52% | 1108.25 | 0 |
|
||||
| balanced | 16.934 | 209.980 / 391.490 / 409.089 | 23.48% | 2200.52 | 0 |
|
||||
|
||||
`balanced-head-sampling-probe.json` — 40 synthetic `GET /health/not-found` responses со status 500: 8 sampled, 32 unsampled. В sampled example Tempo вернул 200; в unsampled — 404. Evidence не содержит credentials, request bodies или message text.
|
||||
|
||||
Ограничение: это один closed-loop прогон на локальном Docker Desktop. Он годится для измерения порядка накладных расходов и ingest, но не для статистического ранжирования latency близких sampling-профилей.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T17:50:37.4885344+00:00",
|
||||
"request_count": 40,
|
||||
"response_status_counts": {
|
||||
"500": 40
|
||||
},
|
||||
"correlated_log_count": 40,
|
||||
"sampled_log_count": 8,
|
||||
"unsampled_log_count": 32,
|
||||
"sampled_ratio": 0.2,
|
||||
"sampled_example": {
|
||||
"trace_id": "00d814f7907510c8066b49c89cefd0ec",
|
||||
"tempo_http_status": 200
|
||||
},
|
||||
"unsampled_example": {
|
||||
"trace_id": "495125068d65af4c622b34ada5196db5",
|
||||
"tempo_http_status": 404
|
||||
},
|
||||
"interpretation": "Head sampling is decided before the HTTP outcome: a non-2xx log can exist even when its trace was not emitted to Tempo."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.352,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 768,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 16.934,
|
||||
"successful_latency_ms": {
|
||||
"p50": 209.98,
|
||||
"p95": 391.49,
|
||||
"p99": 409.089
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python tests/smoke.py --base-url "http://127.0.0.1:18080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,149 @@
|
||||
[
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:40.0143412+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.22,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:40.0143412+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 23.47,
|
||||
"memory_bytes": 396256870,
|
||||
"memory_display": "377.9MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:40.0143412+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.41,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:47.6725368+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 1.0,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:47.6725368+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.02,
|
||||
"memory_bytes": 411461222,
|
||||
"memory_display": "392.4MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:47.6725368+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.26,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:55.3352746+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.58,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:55.3352746+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 82.94,
|
||||
"memory_bytes": 416808960,
|
||||
"memory_display": "397.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:44:55.3352746+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 3.12,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:03.0055136+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.77,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:03.0055136+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 79.98,
|
||||
"memory_bytes": 420059546,
|
||||
"memory_display": "400.6MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:03.0055136+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.28,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:10.7228542+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.56,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:10.7228542+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.02,
|
||||
"memory_bytes": 420478976,
|
||||
"memory_display": "401MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:10.7228542+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.5,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:18.3628330+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.34,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:18.3628330+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 74.77,
|
||||
"memory_bytes": 423205274,
|
||||
"memory_display": "403.6MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:18.3628330+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.47,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:26.0533423+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.33,
|
||||
"memory_bytes": 155713536,
|
||||
"memory_display": "148.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:26.0533423+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 58.16,
|
||||
"memory_bytes": 426351002,
|
||||
"memory_display": "406.6MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:45:26.0533423+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.43,
|
||||
"memory_bytes": 313524224,
|
||||
"memory_display": "299MiB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"mode": {
|
||||
"name": "balanced",
|
||||
"sdk_disabled": false,
|
||||
"sampler_arg": 0.25,
|
||||
"metric_interval_ms": 30000
|
||||
},
|
||||
"load": {
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.352,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 768,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 16.934,
|
||||
"successful_latency_ms": {
|
||||
"p50": 209.98,
|
||||
"p95": 391.49,
|
||||
"p99": 409.089
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"ingest_before": {
|
||||
"captured_at": "2026-11-19T20:44:35.0412998+03:00",
|
||||
"collector_accepted_spans": 74001.0,
|
||||
"collector_accepted_logs": 5524.0,
|
||||
"collector_accepted_metric_points": 97066.0,
|
||||
"collector_sent_spans": 70630.0,
|
||||
"collector_sent_logs": 4389.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 28939480.0,
|
||||
"tempo_ingest_spans": 56863.0,
|
||||
"loki_line_bytes": 1491972.0,
|
||||
"loki_metadata_bytes": 1398836.0,
|
||||
"loki_lines": 4053.0,
|
||||
"prometheus_appended_samples": 1157813.0
|
||||
},
|
||||
"ingest_after": {
|
||||
"captured_at": "2026-11-19T20:45:43.0724595+03:00",
|
||||
"collector_accepted_spans": 76201.0,
|
||||
"collector_accepted_logs": 6299.0,
|
||||
"collector_accepted_metric_points": 97304.0,
|
||||
"collector_sent_spans": 72837.0,
|
||||
"collector_sent_logs": 5164.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 30069342.0,
|
||||
"tempo_ingest_spans": 59070.0,
|
||||
"loki_line_bytes": 1780565.0,
|
||||
"loki_metadata_bytes": 1670379.0,
|
||||
"loki_lines": 4828.0,
|
||||
"prometheus_appended_samples": 1164131.0
|
||||
},
|
||||
"ingest_delta": {
|
||||
"collector_accepted_spans": 2200.0,
|
||||
"collector_accepted_logs": 775.0,
|
||||
"collector_accepted_metric_points": 238.0,
|
||||
"collector_sent_spans": 2207.0,
|
||||
"collector_sent_logs": 775.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_failed_logs": 0.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 1129862.0,
|
||||
"tempo_ingest_spans": 2207.0,
|
||||
"loki_line_bytes": 288593.0,
|
||||
"loki_metadata_bytes": 271543.0,
|
||||
"loki_lines": 775.0,
|
||||
"prometheus_appended_samples": 6318.0,
|
||||
"loki_total_bytes": 560136.0,
|
||||
"backend_total_bytes": 1689998.0
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 67.623,
|
||||
"cpu_peak_percent": 82.94,
|
||||
"ram_avg_bytes": 416374550,
|
||||
"ram_peak_bytes": 426351002
|
||||
},
|
||||
"worker": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.781,
|
||||
"cpu_peak_percent": 3.12,
|
||||
"ram_avg_bytes": 313524224,
|
||||
"ram_peak_bytes": 313524224
|
||||
},
|
||||
"collector": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.543,
|
||||
"cpu_peak_percent": 1.0,
|
||||
"ram_avg_bytes": 155713536,
|
||||
"ram_peak_bytes": 155713536
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 10.0,
|
||||
"elapsed_seconds": 10.228,
|
||||
"concurrency": 2,
|
||||
"successful_requests": 99,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 9.679,
|
||||
"successful_latency_ms": {
|
||||
"p50": 188.687,
|
||||
"p95": 391.506,
|
||||
"p99": 1458.041
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
"mode","sdk_disabled","trace_sample_ratio","metric_interval_ms","successful_requests","failed_requests","successful_rps","rps_change_vs_off_percent","latency_p50_ms","latency_p95_ms","latency_p99_ms","p95_change_vs_off_percent","api_cpu_avg_percent","api_ram_peak_bytes","worker_cpu_avg_percent","worker_ram_peak_bytes","collector_cpu_avg_percent","collector_ram_peak_bytes","accepted_spans","accepted_logs","accepted_metric_points","spans_per_successful_request","trace_retained_percent_of_full_normalized","estimated_spans_not_emitted_by_sdk","collector_transport_lost_spans","tempo_ingest_bytes","loki_ingest_bytes","backend_trace_and_log_ingest_bytes","tempo_bytes_per_successful_request","loki_bytes_per_successful_request","backend_bytes_per_successful_request","prometheus_appended_samples"
|
||||
"off","True","0","60000","1840","0","40.547","0","97.186","198.133","275.316","0","66.496","380633088","0.33","208876339","0.054","151519232","0","0","0","0","0","22447","0","0","0","0","0","0","0","6643"
|
||||
"full","False","1","10000","1012","0","22.419","-44.71","185.241","315.289","405.961","59.13","69.176","425407283","0.544","323171123","2.169","157496115","12346","1019","711","12.1996","100","0","0","6286284","715458","7001742","6211.74","706.97","6918.72","7479"
|
||||
"economy","False","0.05","60000","664","0","14.622","-63.94","287.141","405.639","502.152","104.73","68.564","431803597","0.517","293601280","0.877","159593267","447","671","115","0.6732","5.52","7654","0","241260","494616","735876","363.34","744.9","1108.25","6008"
|
||||
"balanced","False","0.25","30000","768","0","16.934","-58.24","209.98","391.49","409.089","97.59","67.623","426351002","0.781","313524224","0.543","155713536","2200","775","238","2.8646","23.48","7169","0","1129862","560136","1689998","1471.17","729.34","2200.52","6318"
|
||||
|
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.411,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 664,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 14.622,
|
||||
"successful_latency_ms": {
|
||||
"p50": 287.141,
|
||||
"p95": 405.639,
|
||||
"p99": 502.152
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
[
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:39.6397582+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.11,
|
||||
"memory_bytes": 159593267,
|
||||
"memory_display": "152.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:39.6397582+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 19.43,
|
||||
"memory_bytes": 416599245,
|
||||
"memory_display": "397.3MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:39.6397582+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.35,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:47.3615591+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.41,
|
||||
"memory_bytes": 159593267,
|
||||
"memory_display": "152.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:47.3615591+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 80.42,
|
||||
"memory_bytes": 418801254,
|
||||
"memory_display": "399.4MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:47.3615591+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.57,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:55.0298538+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.03,
|
||||
"memory_bytes": 159593267,
|
||||
"memory_display": "152.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:55.0298538+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 72.47,
|
||||
"memory_bytes": 427714150,
|
||||
"memory_display": "407.9MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:40:55.0298538+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.66,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:02.7166071+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.87,
|
||||
"memory_bytes": 159593267,
|
||||
"memory_display": "152.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:02.7166071+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.82,
|
||||
"memory_bytes": 431803597,
|
||||
"memory_display": "411.8MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:02.7166071+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.47,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:10.3853303+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.49,
|
||||
"memory_bytes": 159593267,
|
||||
"memory_display": "152.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:10.3853303+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 76.06,
|
||||
"memory_bytes": 428343296,
|
||||
"memory_display": "408.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:10.3853303+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.5,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:18.0435483+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.52,
|
||||
"memory_bytes": 157810688,
|
||||
"memory_display": "150.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:18.0435483+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 76.23,
|
||||
"memory_bytes": 428238438,
|
||||
"memory_display": "408.4MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:18.0435483+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.42,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:25.6861454+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 3.71,
|
||||
"memory_bytes": 157810688,
|
||||
"memory_display": "150.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:25.6861454+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.52,
|
||||
"memory_bytes": 431174451,
|
||||
"memory_display": "411.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:41:25.6861454+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.65,
|
||||
"memory_bytes": 293601280,
|
||||
"memory_display": "280MiB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"mode": {
|
||||
"name": "economy",
|
||||
"sdk_disabled": false,
|
||||
"sampler_arg": 0.05,
|
||||
"metric_interval_ms": 60000
|
||||
},
|
||||
"load": {
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.411,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 664,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 14.622,
|
||||
"successful_latency_ms": {
|
||||
"p50": 287.141,
|
||||
"p95": 405.639,
|
||||
"p99": 502.152
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"ingest_before": {
|
||||
"captured_at": "2026-11-19T20:40:34.9641917+03:00",
|
||||
"collector_accepted_spans": 73201.0,
|
||||
"collector_accepted_logs": 4719.0,
|
||||
"collector_accepted_metric_points": 96428.0,
|
||||
"collector_sent_spans": 69837.0,
|
||||
"collector_sent_logs": 3584.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 28514676.0,
|
||||
"tempo_ingest_spans": 56063.0,
|
||||
"loki_line_bytes": 1173945.0,
|
||||
"loki_metadata_bytes": 1099842.0,
|
||||
"loki_lines": 3248.0,
|
||||
"prometheus_appended_samples": 1133674.0
|
||||
},
|
||||
"ingest_after": {
|
||||
"captured_at": "2026-11-19T20:41:42.7120231+03:00",
|
||||
"collector_accepted_spans": 73648.0,
|
||||
"collector_accepted_logs": 5390.0,
|
||||
"collector_accepted_metric_points": 96543.0,
|
||||
"collector_sent_spans": 70284.0,
|
||||
"collector_sent_logs": 4255.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 28755936.0,
|
||||
"tempo_ingest_spans": 56510.0,
|
||||
"loki_line_bytes": 1428634.0,
|
||||
"loki_metadata_bytes": 1339769.0,
|
||||
"loki_lines": 3919.0,
|
||||
"prometheus_appended_samples": 1139682.0
|
||||
},
|
||||
"ingest_delta": {
|
||||
"collector_accepted_spans": 447.0,
|
||||
"collector_accepted_logs": 671.0,
|
||||
"collector_accepted_metric_points": 115.0,
|
||||
"collector_sent_spans": 447.0,
|
||||
"collector_sent_logs": 671.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_failed_logs": 0.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 241260.0,
|
||||
"tempo_ingest_spans": 447.0,
|
||||
"loki_line_bytes": 254689.0,
|
||||
"loki_metadata_bytes": 239927.0,
|
||||
"loki_lines": 671.0,
|
||||
"prometheus_appended_samples": 6008.0,
|
||||
"loki_total_bytes": 494616.0,
|
||||
"backend_total_bytes": 735876.0
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 68.564,
|
||||
"cpu_peak_percent": 80.42,
|
||||
"ram_avg_bytes": 426096347,
|
||||
"ram_peak_bytes": 431803597
|
||||
},
|
||||
"worker": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.517,
|
||||
"cpu_peak_percent": 0.66,
|
||||
"ram_avg_bytes": 293601280,
|
||||
"ram_peak_bytes": 293601280
|
||||
},
|
||||
"collector": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.877,
|
||||
"cpu_peak_percent": 3.71,
|
||||
"ram_avg_bytes": 159083959,
|
||||
"ram_peak_bytes": 159593267
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 10.0,
|
||||
"elapsed_seconds": 10.398,
|
||||
"concurrency": 2,
|
||||
"successful_requests": 77,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 7.405,
|
||||
"successful_latency_ms": {
|
||||
"p50": 273.64,
|
||||
"p95": 396.707,
|
||||
"p99": 872.673
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.14,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1012,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 22.419,
|
||||
"successful_latency_ms": {
|
||||
"p50": 185.241,
|
||||
"p95": 315.289,
|
||||
"p99": 405.961
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
[
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:50.6934872+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.68,
|
||||
"memory_bytes": 155084390,
|
||||
"memory_display": "147.9MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:50.6934872+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 43.03,
|
||||
"memory_bytes": 396781158,
|
||||
"memory_display": "378.4MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:50.6934872+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.34,
|
||||
"memory_bytes": 323066266,
|
||||
"memory_display": "308.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:58.3607881+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.6,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:58.3607881+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 81.27,
|
||||
"memory_bytes": 409573786,
|
||||
"memory_display": "390.6MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:36:58.3607881+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 2.09,
|
||||
"memory_bytes": 323066266,
|
||||
"memory_display": "308.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:06.0106951+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 6.64,
|
||||
"memory_bytes": 154874675,
|
||||
"memory_display": "147.7MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:06.0106951+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 79.67,
|
||||
"memory_bytes": 409783501,
|
||||
"memory_display": "390.8MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:06.0106951+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.31,
|
||||
"memory_bytes": 323066266,
|
||||
"memory_display": "308.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:13.6552262+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.27,
|
||||
"memory_bytes": 155294106,
|
||||
"memory_display": "148.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:13.6552262+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 74.69,
|
||||
"memory_bytes": 418801254,
|
||||
"memory_display": "399.4MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:13.6552262+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.28,
|
||||
"memory_bytes": 323171123,
|
||||
"memory_display": "308.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:21.2593950+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.26,
|
||||
"memory_bytes": 155294106,
|
||||
"memory_display": "148.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:21.2593950+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.01,
|
||||
"memory_bytes": 421632410,
|
||||
"memory_display": "402.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:21.2593950+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.24,
|
||||
"memory_bytes": 323171123,
|
||||
"memory_display": "308.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:28.9314370+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 6.11,
|
||||
"memory_bytes": 157076685,
|
||||
"memory_display": "149.8MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:28.9314370+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 80.22,
|
||||
"memory_bytes": 424358707,
|
||||
"memory_display": "404.7MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:28.9314370+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.33,
|
||||
"memory_bytes": 323171123,
|
||||
"memory_display": "308.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:36.5264368+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.62,
|
||||
"memory_bytes": 157496115,
|
||||
"memory_display": "150.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:36.5264368+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 48.34,
|
||||
"memory_bytes": 425407283,
|
||||
"memory_display": "405.7MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:37:36.5264368+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.22,
|
||||
"memory_bytes": 323171123,
|
||||
"memory_display": "308.2MiB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"mode": {
|
||||
"name": "full",
|
||||
"sdk_disabled": false,
|
||||
"sampler_arg": 1.0,
|
||||
"metric_interval_ms": 10000
|
||||
},
|
||||
"load": {
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.14,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1012,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 22.419,
|
||||
"successful_latency_ms": {
|
||||
"p50": 185.241,
|
||||
"p95": 315.289,
|
||||
"p99": 405.961
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"ingest_before": {
|
||||
"captured_at": "2026-11-19T20:36:45.972962+03:00",
|
||||
"collector_accepted_spans": 60758.0,
|
||||
"collector_accepted_logs": 3588.0,
|
||||
"collector_accepted_metric_points": 95359.0,
|
||||
"collector_sent_spans": 57378.0,
|
||||
"collector_sent_logs": 2453.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 22180655.0,
|
||||
"tempo_ingest_spans": 43604.0,
|
||||
"loki_line_bytes": 747186.0,
|
||||
"loki_metadata_bytes": 699288.0,
|
||||
"loki_lines": 2117.0,
|
||||
"prometheus_appended_samples": 1109213.0
|
||||
},
|
||||
"ingest_after": {
|
||||
"captured_at": "2026-11-19T20:37:53.5546074+03:00",
|
||||
"collector_accepted_spans": 73104.0,
|
||||
"collector_accepted_logs": 4607.0,
|
||||
"collector_accepted_metric_points": 96070.0,
|
||||
"collector_sent_spans": 69432.0,
|
||||
"collector_sent_logs": 3472.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 28466939.0,
|
||||
"tempo_ingest_spans": 55981.0,
|
||||
"loki_line_bytes": 1116124.0,
|
||||
"loki_metadata_bytes": 1045808.0,
|
||||
"loki_lines": 3136.0,
|
||||
"prometheus_appended_samples": 1116692.0
|
||||
},
|
||||
"ingest_delta": {
|
||||
"collector_accepted_spans": 12346.0,
|
||||
"collector_accepted_logs": 1019.0,
|
||||
"collector_accepted_metric_points": 711.0,
|
||||
"collector_sent_spans": 12054.0,
|
||||
"collector_sent_logs": 1019.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_failed_logs": 0.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 6286284.0,
|
||||
"tempo_ingest_spans": 12377.0,
|
||||
"loki_line_bytes": 368938.0,
|
||||
"loki_metadata_bytes": 346520.0,
|
||||
"loki_lines": 1019.0,
|
||||
"prometheus_appended_samples": 7479.0,
|
||||
"loki_total_bytes": 715458.0,
|
||||
"backend_total_bytes": 7001742.0
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 69.176,
|
||||
"cpu_peak_percent": 81.27,
|
||||
"ram_avg_bytes": 415191157,
|
||||
"ram_peak_bytes": 425407283
|
||||
},
|
||||
"worker": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.544,
|
||||
"cpu_peak_percent": 2.09,
|
||||
"ram_avg_bytes": 323126184,
|
||||
"ram_peak_bytes": 323171123
|
||||
},
|
||||
"collector": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 2.169,
|
||||
"cpu_peak_percent": 6.64,
|
||||
"ram_avg_bytes": 155234187,
|
||||
"ram_peak_bytes": 157496115
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 10.0,
|
||||
"elapsed_seconds": 10.178,
|
||||
"concurrency": 2,
|
||||
"successful_requests": 83,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 8.155,
|
||||
"successful_latency_ms": {
|
||||
"p50": 210.318,
|
||||
"p95": 382.103,
|
||||
"p99": 1033.558
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.379,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1840,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 40.547,
|
||||
"successful_latency_ms": {
|
||||
"p50": 97.186,
|
||||
"p95": 198.133,
|
||||
"p99": 275.316
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
[
|
||||
{
|
||||
"captured_at": "2026-11-19T17:32:57.3534028+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.05,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:32:57.3534028+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 7.72,
|
||||
"memory_bytes": 304296755,
|
||||
"memory_display": "290.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:32:57.3534028+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.5,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:05.0940138+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.05,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:05.0940138+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 75.89,
|
||||
"memory_bytes": 369203610,
|
||||
"memory_display": "352.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:05.0940138+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.27,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:12.7649354+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.04,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:12.7649354+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 76.82,
|
||||
"memory_bytes": 368259891,
|
||||
"memory_display": "351.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:12.7649354+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.3,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:20.3812382+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.04,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:20.3812382+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 78.72,
|
||||
"memory_bytes": 374446490,
|
||||
"memory_display": "357.1MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:20.3812382+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.24,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:28.0386363+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.1,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:28.0386363+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 77.11,
|
||||
"memory_bytes": 377067930,
|
||||
"memory_display": "359.6MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:28.0386363+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.42,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:35.6857265+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.05,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:35.6857265+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 75.04,
|
||||
"memory_bytes": 380633088,
|
||||
"memory_display": "363MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:35.6857265+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.24,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:43.3529739+00:00",
|
||||
"service": "otel-collector",
|
||||
"cpu_percent": 0.05,
|
||||
"memory_bytes": 151519232,
|
||||
"memory_display": "144.5MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:43.3529739+00:00",
|
||||
"service": "api",
|
||||
"cpu_percent": 74.17,
|
||||
"memory_bytes": 379794227,
|
||||
"memory_display": "362.2MiB"
|
||||
},
|
||||
{
|
||||
"captured_at": "2026-11-19T17:33:43.3529739+00:00",
|
||||
"service": "worker",
|
||||
"cpu_percent": 0.34,
|
||||
"memory_bytes": 208876339,
|
||||
"memory_display": "199.2MiB"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"mode": {
|
||||
"name": "off",
|
||||
"sdk_disabled": true,
|
||||
"sampler_arg": 0.0,
|
||||
"metric_interval_ms": 60000
|
||||
},
|
||||
"load": {
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 45.0,
|
||||
"elapsed_seconds": 45.379,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1840,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 40.547,
|
||||
"successful_latency_ms": {
|
||||
"p50": 97.186,
|
||||
"p95": 198.133,
|
||||
"p99": 275.316
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"ingest_before": {
|
||||
"captured_at": "2026-11-19T20:32:50.4901468+03:00",
|
||||
"collector_accepted_spans": 59600.0,
|
||||
"collector_accepted_logs": 3470.0,
|
||||
"collector_accepted_metric_points": 94238.0,
|
||||
"collector_sent_spans": 56236.0,
|
||||
"collector_sent_logs": 2335.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 21604219.0,
|
||||
"tempo_ingest_spans": 42462.0,
|
||||
"loki_line_bytes": 691475.0,
|
||||
"loki_metadata_bytes": 647495.0,
|
||||
"loki_lines": 1999.0,
|
||||
"prometheus_appended_samples": 1084710.0
|
||||
},
|
||||
"ingest_after": {
|
||||
"captured_at": "2026-11-19T20:34:00.39937+03:00",
|
||||
"collector_accepted_spans": 59600.0,
|
||||
"collector_accepted_logs": 3470.0,
|
||||
"collector_accepted_metric_points": 94238.0,
|
||||
"collector_sent_spans": 56236.0,
|
||||
"collector_sent_logs": 2335.0,
|
||||
"collector_failed_spans": 3364.0,
|
||||
"collector_failed_logs": 1135.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 21604219.0,
|
||||
"tempo_ingest_spans": 42462.0,
|
||||
"loki_line_bytes": 691475.0,
|
||||
"loki_metadata_bytes": 647495.0,
|
||||
"loki_lines": 1999.0,
|
||||
"prometheus_appended_samples": 1091353.0
|
||||
},
|
||||
"ingest_delta": {
|
||||
"collector_accepted_spans": 0.0,
|
||||
"collector_accepted_logs": 0.0,
|
||||
"collector_accepted_metric_points": 0.0,
|
||||
"collector_sent_spans": 0.0,
|
||||
"collector_sent_logs": 0.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_failed_logs": 0.0,
|
||||
"collector_enqueue_failed_spans": 0.0,
|
||||
"collector_refused_spans": 0.0,
|
||||
"tempo_ingest_bytes": 0.0,
|
||||
"tempo_ingest_spans": 0.0,
|
||||
"loki_line_bytes": 0.0,
|
||||
"loki_metadata_bytes": 0.0,
|
||||
"loki_lines": 0.0,
|
||||
"prometheus_appended_samples": 6643.0,
|
||||
"loki_total_bytes": 0.0,
|
||||
"backend_total_bytes": 0.0
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 66.496,
|
||||
"cpu_peak_percent": 78.72,
|
||||
"ram_avg_bytes": 364814570,
|
||||
"ram_peak_bytes": 380633088
|
||||
},
|
||||
"worker": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.33,
|
||||
"cpu_peak_percent": 0.5,
|
||||
"ram_avg_bytes": 208876339,
|
||||
"ram_peak_bytes": 208876339
|
||||
},
|
||||
"collector": {
|
||||
"sample_count": 7,
|
||||
"cpu_avg_percent": 0.054,
|
||||
"cpu_peak_percent": 0.1,
|
||||
"ram_avg_bytes": 151519232,
|
||||
"ram_peak_bytes": 151519232
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 10.0,
|
||||
"elapsed_seconds": 10.225,
|
||||
"concurrency": 2,
|
||||
"successful_requests": 126,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 12.323,
|
||||
"successful_latency_ms": {
|
||||
"p50": 122.017,
|
||||
"p95": 277.867,
|
||||
"p99": 964.446
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"captured_at": "2026-11-19T20:45:43.8506063+03:00",
|
||||
"methodology": {
|
||||
"load_seconds": 45,
|
||||
"concurrency": 4,
|
||||
"warmup_seconds": 10,
|
||||
"stats_interval_seconds": 5,
|
||||
"workload": "closed-loop POST messages; identical profile per mode; API and worker recreated before each mode",
|
||||
"sampling_loss_definition": "estimated against full-mode accepted spans per successful request; it is SDK non-emission, not Collector transport loss",
|
||||
"ingest_units": "Collector items; Tempo/Loki uncompressed backend bytes; Prometheus appended samples"
|
||||
},
|
||||
"comparison": [
|
||||
{
|
||||
"mode": "off",
|
||||
"sdk_disabled": true,
|
||||
"trace_sample_ratio": 0.0,
|
||||
"metric_interval_ms": 60000,
|
||||
"successful_requests": 1840,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 40.547,
|
||||
"rps_change_vs_off_percent": 0.0,
|
||||
"latency_p50_ms": 97.186,
|
||||
"latency_p95_ms": 198.133,
|
||||
"latency_p99_ms": 275.316,
|
||||
"p95_change_vs_off_percent": 0.0,
|
||||
"api_cpu_avg_percent": 66.496,
|
||||
"api_ram_peak_bytes": 380633088,
|
||||
"worker_cpu_avg_percent": 0.33,
|
||||
"worker_ram_peak_bytes": 208876339,
|
||||
"collector_cpu_avg_percent": 0.054,
|
||||
"collector_ram_peak_bytes": 151519232,
|
||||
"accepted_spans": 0.0,
|
||||
"accepted_logs": 0.0,
|
||||
"accepted_metric_points": 0.0,
|
||||
"spans_per_successful_request": 0.0,
|
||||
"trace_retained_percent_of_full_normalized": 0.0,
|
||||
"estimated_spans_not_emitted_by_sdk": 22447.0,
|
||||
"collector_transport_lost_spans": 0.0,
|
||||
"tempo_ingest_bytes": 0.0,
|
||||
"loki_ingest_bytes": 0.0,
|
||||
"backend_trace_and_log_ingest_bytes": 0.0,
|
||||
"tempo_bytes_per_successful_request": 0.0,
|
||||
"loki_bytes_per_successful_request": 0.0,
|
||||
"backend_bytes_per_successful_request": 0.0,
|
||||
"prometheus_appended_samples": 6643.0
|
||||
},
|
||||
{
|
||||
"mode": "full",
|
||||
"sdk_disabled": false,
|
||||
"trace_sample_ratio": 1.0,
|
||||
"metric_interval_ms": 10000,
|
||||
"successful_requests": 1012,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 22.419,
|
||||
"rps_change_vs_off_percent": -44.71,
|
||||
"latency_p50_ms": 185.241,
|
||||
"latency_p95_ms": 315.289,
|
||||
"latency_p99_ms": 405.961,
|
||||
"p95_change_vs_off_percent": 59.13,
|
||||
"api_cpu_avg_percent": 69.176,
|
||||
"api_ram_peak_bytes": 425407283,
|
||||
"worker_cpu_avg_percent": 0.544,
|
||||
"worker_ram_peak_bytes": 323171123,
|
||||
"collector_cpu_avg_percent": 2.169,
|
||||
"collector_ram_peak_bytes": 157496115,
|
||||
"accepted_spans": 12346.0,
|
||||
"accepted_logs": 1019.0,
|
||||
"accepted_metric_points": 711.0,
|
||||
"spans_per_successful_request": 12.1996,
|
||||
"trace_retained_percent_of_full_normalized": 100.0,
|
||||
"estimated_spans_not_emitted_by_sdk": 0.0,
|
||||
"collector_transport_lost_spans": 0.0,
|
||||
"tempo_ingest_bytes": 6286284.0,
|
||||
"loki_ingest_bytes": 715458.0,
|
||||
"backend_trace_and_log_ingest_bytes": 7001742.0,
|
||||
"tempo_bytes_per_successful_request": 6211.74,
|
||||
"loki_bytes_per_successful_request": 706.97,
|
||||
"backend_bytes_per_successful_request": 6918.72,
|
||||
"prometheus_appended_samples": 7479.0
|
||||
},
|
||||
{
|
||||
"mode": "economy",
|
||||
"sdk_disabled": false,
|
||||
"trace_sample_ratio": 0.05,
|
||||
"metric_interval_ms": 60000,
|
||||
"successful_requests": 664,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 14.622,
|
||||
"rps_change_vs_off_percent": -63.94,
|
||||
"latency_p50_ms": 287.141,
|
||||
"latency_p95_ms": 405.639,
|
||||
"latency_p99_ms": 502.152,
|
||||
"p95_change_vs_off_percent": 104.73,
|
||||
"api_cpu_avg_percent": 68.564,
|
||||
"api_ram_peak_bytes": 431803597,
|
||||
"worker_cpu_avg_percent": 0.517,
|
||||
"worker_ram_peak_bytes": 293601280,
|
||||
"collector_cpu_avg_percent": 0.877,
|
||||
"collector_ram_peak_bytes": 159593267,
|
||||
"accepted_spans": 447.0,
|
||||
"accepted_logs": 671.0,
|
||||
"accepted_metric_points": 115.0,
|
||||
"spans_per_successful_request": 0.6732,
|
||||
"trace_retained_percent_of_full_normalized": 5.52,
|
||||
"estimated_spans_not_emitted_by_sdk": 7654.0,
|
||||
"collector_transport_lost_spans": 0.0,
|
||||
"tempo_ingest_bytes": 241260.0,
|
||||
"loki_ingest_bytes": 494616.0,
|
||||
"backend_trace_and_log_ingest_bytes": 735876.0,
|
||||
"tempo_bytes_per_successful_request": 363.34,
|
||||
"loki_bytes_per_successful_request": 744.9,
|
||||
"backend_bytes_per_successful_request": 1108.25,
|
||||
"prometheus_appended_samples": 6008.0
|
||||
},
|
||||
{
|
||||
"mode": "balanced",
|
||||
"sdk_disabled": false,
|
||||
"trace_sample_ratio": 0.25,
|
||||
"metric_interval_ms": 30000,
|
||||
"successful_requests": 768,
|
||||
"failed_requests": 0,
|
||||
"successful_rps": 16.934,
|
||||
"rps_change_vs_off_percent": -58.24,
|
||||
"latency_p50_ms": 209.98,
|
||||
"latency_p95_ms": 391.49,
|
||||
"latency_p99_ms": 409.089,
|
||||
"p95_change_vs_off_percent": 97.59,
|
||||
"api_cpu_avg_percent": 67.623,
|
||||
"api_ram_peak_bytes": 426351002,
|
||||
"worker_cpu_avg_percent": 0.781,
|
||||
"worker_ram_peak_bytes": 313524224,
|
||||
"collector_cpu_avg_percent": 0.543,
|
||||
"collector_ram_peak_bytes": 155713536,
|
||||
"accepted_spans": 2200.0,
|
||||
"accepted_logs": 775.0,
|
||||
"accepted_metric_points": 238.0,
|
||||
"spans_per_successful_request": 2.8646,
|
||||
"trace_retained_percent_of_full_normalized": 23.48,
|
||||
"estimated_spans_not_emitted_by_sdk": 7169.0,
|
||||
"collector_transport_lost_spans": 0.0,
|
||||
"tempo_ingest_bytes": 1129862.0,
|
||||
"loki_ingest_bytes": 560136.0,
|
||||
"backend_trace_and_log_ingest_bytes": 1689998.0,
|
||||
"tempo_bytes_per_successful_request": 1471.17,
|
||||
"loki_bytes_per_successful_request": 729.34,
|
||||
"backend_bytes_per_successful_request": 2200.52,
|
||||
"prometheus_appended_samples": 6318.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"scenario": "sequential API replacement under closed-loop writes",
|
||||
"requested_duration_seconds": 180.0,
|
||||
"elapsed_seconds": 182.022,
|
||||
"rolling_action_seconds": 125.275,
|
||||
"concurrency": 4,
|
||||
"observed_instance_ids": [
|
||||
"431acab3a50c",
|
||||
"8e081627dfc3",
|
||||
"d67820ebd09d",
|
||||
"fdff9a67741c"
|
||||
],
|
||||
"successful_requests": 335,
|
||||
"failed_requests": 3,
|
||||
"error_ratio": 0.008876,
|
||||
"successful_rps": 1.84,
|
||||
"successful_latency_ms": {
|
||||
"p50": 2092.164,
|
||||
"p95": 2252.301,
|
||||
"p99": 3064.667
|
||||
},
|
||||
"note": "Closed-loop result on one host; success latency excludes failures."
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"concurrency": 12,
|
||||
"successful_uploads": 12,
|
||||
"payload_bytes_each": 6835833,
|
||||
"duration_seconds": 10.156,
|
||||
"api_replicas": 2,
|
||||
"memory_samples": 5,
|
||||
"baseline_api_memory_bytes": 668467200,
|
||||
"peak_api_memory_bytes": 839909375,
|
||||
"peak_increase_bytes": 171442175,
|
||||
"configured_max_concurrent_uploads_per_replica": 3,
|
||||
"configured_max_upload_bytes": 10485760,
|
||||
"configured_max_image_pixels": 40000000
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"concurrency": 12,
|
||||
"successful_uploads": 12,
|
||||
"payload_bytes_each": 6835833,
|
||||
"duration_seconds": 31.241,
|
||||
"api_replicas": 2,
|
||||
"memory_samples": 10,
|
||||
"baseline_api_memory_bytes": 665321471,
|
||||
"peak_api_memory_bytes": 833198489,
|
||||
"peak_increase_bytes": 167877018,
|
||||
"configured_max_concurrent_uploads_per_replica": 3,
|
||||
"configured_max_upload_bytes": 10485760,
|
||||
"configured_max_image_pixels": 40000000
|
||||
}
|
||||
Vendored
BIN
Binary file not shown.
+9
@@ -0,0 +1,9 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip
|
||||
networkTimeout=10000
|
||||
retries=0
|
||||
retryBackOffMs=500
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# gradlew start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh gradlew
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
@rem SPDX-License-Identifier: Apache-2.0
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem gradlew startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables, and ensure extensions are enabled
|
||||
setlocal EnableExtensions
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
"%COMSPEC%" /c exit 1
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
|
||||
|
||||
@rem Execute gradlew
|
||||
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
|
||||
@rem which allows us to clear the local environment before executing the java command
|
||||
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
|
||||
|
||||
:exitWithErrorLevel
|
||||
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
|
||||
"%COMSPEC%" /c exit %ERRORLEVEL%
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__/attachments/*",
|
||||
"arn:aws:s3:::__S3_BUCKET__/staging/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
worker_processes auto;
|
||||
worker_shutdown_timeout 10s;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# Let the application enforce the exact 10 MiB contract and return JSON.
|
||||
client_max_body_size 11m;
|
||||
client_body_timeout 10s;
|
||||
access_log /var/log/nginx/access.log;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
|
||||
# Docker embedded DNS returns current Compose replicas. Nginx OSS 1.27.3+
|
||||
# refreshes these addresses without a configuration reload.
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
least_conn;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
proxy_intercept_errors on;
|
||||
error_page 502 503 504 = @no_ready_api;
|
||||
|
||||
location = /nginx-health {
|
||||
access_log off;
|
||||
default_type application/json;
|
||||
return 200 '{"status":"ok"}';
|
||||
}
|
||||
|
||||
# After both peers have refused traffic or become unreachable, Nginx
|
||||
# would normally emit its own HTML 502. Keep the public API contract:
|
||||
# no ready backend is a JSON 503 with the required tracing headers.
|
||||
location @no_ready_api {
|
||||
internal;
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
add_header X-Instance-Id nginx always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}';
|
||||
}
|
||||
|
||||
# Message POST is the only non-idempotent HTTP method that is safe to
|
||||
# replay here: the API requires Idempotency-Key and commits the message
|
||||
# together with its shared PostgreSQL idempotency record.
|
||||
location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_request_buffering on;
|
||||
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 4s;
|
||||
proxy_read_timeout 4s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_request_buffering on;
|
||||
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
# Upload is not replayed: it has no idempotency key. The client may retry explicitly.
|
||||
proxy_next_upstream off;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 4s;
|
||||
proxy_read_timeout 4s;
|
||||
|
||||
# No sticky sessions. Failed/unready peers are retried elsewhere.
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
+27
-4
@@ -1,7 +1,30 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Стек наблюдаемости
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
Поток данных:
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
`messenger-api / messenger-worker -> OTLP/HTTP -> otel-collector -> Prometheus / Loki / Tempo -> Grafana`.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
Версии закреплены в `compose.yaml`: Java agent 2.33.1 (SDK 1.67.0), Collector Contrib 0.165.0, Prometheus 3.14.0, Loki 3.7.8, Tempo 3.1.1 и Grafana 13.3.0.
|
||||
|
||||
## Файлы
|
||||
|
||||
- `collector.yaml` — OTLP receivers, memory limiter, redaction, batch и bounded retry queues;
|
||||
- `prometheus.yaml` и `prometheus-alerts.yaml` — scrape, SLO/queue/worker/export alerts;
|
||||
- `loki.yaml` — локальное хранилище структурных OTLP logs;
|
||||
- `tempo.yaml` — локальное хранилище traces;
|
||||
- `grafana/provisioning` — три datasources и dashboard provisioning;
|
||||
- `grafana/dashboards/messenger-overview.json` — импортируемый dashboard;
|
||||
- `runbooks.md` — локальные действия для каждого alert.
|
||||
|
||||
Grafana доступна только на `127.0.0.1:3000`, Prometheus — на `127.0.0.1:9090`. Для автоматизированных incident snapshots Loki и Tempo доступны только на loopback-портах 3100 и 3200. OTLP Collector не публикуется на интерфейсы хоста.
|
||||
|
||||
## Проверка корреляции
|
||||
|
||||
1. Выполнить `python tests/smoke.py --base-url http://localhost:8080`.
|
||||
2. В Tempo найти server span upload и перейти к RabbitMQ producer/consumer span.
|
||||
3. Скопировать `trace_id` и выполнить в Loki `{service_name=~"messenger-(api|worker)"} | json | trace_id="..."`.
|
||||
4. Убедиться, что у API и worker разные `service.name`, но общий trace context.
|
||||
|
||||
Для durable retry `traceparent/tracestate` сохраняются рядом с job в PostgreSQL и восстанавливаются перед outbox republish. Проверяемый сценарий: `scripts/incident-retry-trace.ps1`; он требует минимум два producer/consumer span в одной trace и сохраняет их parent IDs.
|
||||
|
||||
Collector удаляет потенциально чувствительные HTTP/message/body/DB parameter attributes. Java agent не настроен на захват request/response headers или bodies. Метрики используют только route template, status/method, job type/outcome и queue name.
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 384
|
||||
spike_limit_mib: 96
|
||||
attributes/redact:
|
||||
actions:
|
||||
- key: http.request.header.authorization
|
||||
action: delete
|
||||
- key: http.request.body
|
||||
action: delete
|
||||
- key: http.response.body
|
||||
action: delete
|
||||
- key: messaging.message.body
|
||||
action: delete
|
||||
- key: db.statement.parameters
|
||||
action: delete
|
||||
- key: url.query
|
||||
action: delete
|
||||
batch:
|
||||
send_batch_size: 512
|
||||
timeout: 5s
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
send_timestamps: true
|
||||
resource_constant_labels:
|
||||
included: ["service.*", "deployment.environment.name"]
|
||||
otlp_http/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 2048
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlp_http/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 2048
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
without_type_suffix: false
|
||||
without_units: false
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if [ "${OTEL_SDK_DISABLED:-false}" = "true" ]; then
|
||||
exec java -jar /app/app.jar "$@"
|
||||
fi
|
||||
|
||||
exec java -javaagent:/opt/opentelemetry-javaagent.jar -jar /app/app.jar "$@"
|
||||
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API requests / second",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "sum by (http_request_method) (rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_request_method}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API 5xx ratio",
|
||||
"description": "5xx business responses divided by all business responses over 5 minutes; health routes are excluded. Displayed as percent (0..100%).",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_response_status_code=~\"5..\",http_route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])), 0.001)", "legendFormat": "5xx / all"}],
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API duration p50 / p95 / p99",
|
||||
"description": "Seconds. histogram_quantile is calculated from the summed Prometheus histogram buckets of all API replicas; replica percentiles are never averaged.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95"},
|
||||
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Job outcomes / second",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[5m]))", "legendFormat": "{{outcome}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Job duration p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[5m])))", "legendFormat": "{{outcome}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Time to ready p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[10m])))", "legendFormat": "p95"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Queue depth and oldest age",
|
||||
"description": "Left axis: durable queued jobs (count). Right axis: age of the oldest unfinished job (seconds). Backlog age is not processing latency.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max(messenger_queue_depth{service_name=\"messenger-api\"})", "legendFormat": "depth"},
|
||||
{"refId": "B", "expr": "max(messenger_queue_oldest_job_age_seconds{service_name=\"messenger-api\"})", "legendFormat": "oldest age (s)"}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "depth"}, "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "left"}]},
|
||||
{"matcher": {"id": "byName", "options": "oldest age (s)"}, "properties": [{"id": "unit", "value": "s"}, {"id": "custom.axisPlacement", "value": "right"}]}
|
||||
]
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Worker saturation",
|
||||
"description": "busy workers / configured worker capacity. Dimensionless ratio displayed as percent.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "messenger_worker_busy{service_name=\"messenger-worker\"} / clamp_min(messenger_worker_capacity{service_name=\"messenger-worker\"}, 1)", "legendFormat": "{{service_instance_id}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "DB pool used / waiting",
|
||||
"description": "Number of HikariCP connections by state plus pending acquisition requests. Unit: connections/requests (count).",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum by (service_name, state) (db_client_connections_usage{service_name=~\"messenger-(api|worker)\"})", "legendFormat": "{{service_name}} {{state}}"},
|
||||
{"refId": "B", "expr": "sum by (service_name) (db_client_connections_pending_requests{service_name=~\"messenger-(api|worker)\"})", "legendFormat": "{{service_name}} waiting"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Correlated application logs",
|
||||
"type": "logs",
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"targets": [{"refId": "A", "expr": "{service_name=~\"messenger-(api|worker)\"} | json", "queryType": "range"}],
|
||||
"options": {"showTime": true, "showLabels": false, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "sortOrder": "Descending", "dedupStrategy": "none"},
|
||||
"gridPos": {"h": 10, "w": 16, "x": 0, "y": 24}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Recent traces",
|
||||
"type": "traces",
|
||||
"datasource": {"type": "tempo", "uid": "tempo"},
|
||||
"targets": [{"refId": "A", "queryType": "search", "serviceName": "messenger-api", "limit": 20}],
|
||||
"gridPos": {"h": 10, "w": 8, "x": 16, "y": 24}
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 41,
|
||||
"tags": ["hla", "lab6", "opentelemetry"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-30m", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger observability",
|
||||
"uid": "messenger-observability",
|
||||
"version": 3,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Messenger Lab 6
|
||||
orgId: 1
|
||||
folder: HLA Lab 6
|
||||
type: file
|
||||
disableDeletion: true
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,59 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 10s
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":"([a-f0-9]{32})"'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: GET
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"'
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
search:
|
||||
hide: false
|
||||
@@ -0,0 +1,37 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 72h
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,51 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
rules:
|
||||
- alert: MessengerApiHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5..",http_route!~"/health/.*"}[5m])) or vector(0))
|
||||
/
|
||||
clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_route!~"/health/.*"}[5m])), 0.001)
|
||||
) > 0.05
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: Messenger API 5xx ratio exceeds 5%
|
||||
description: Five-minute 5xx ratio is above 5% for two minutes.
|
||||
runbook: infra/otel/runbooks.md#api-high-error-rate
|
||||
|
||||
- alert: AttachmentQueueLagHigh
|
||||
expr: messenger_queue_oldest_job_age_seconds{service_name="messenger-api"} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
slo: attachment-time-to-ready
|
||||
annotations:
|
||||
summary: Oldest attachment job is over 60 seconds old
|
||||
description: The durable worker queue has not drained within the SLO threshold.
|
||||
runbook: infra/otel/runbooks.md#queue-lag
|
||||
|
||||
- alert: AttachmentWorkerSaturated
|
||||
expr: |
|
||||
messenger_worker_busy{service_name="messenger-worker"}
|
||||
/ clamp_min(messenger_worker_capacity{service_name="messenger-worker"}, 1) >= 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: All attachment workers are busy
|
||||
description: Worker busy/capacity has remained at 100% for two minutes.
|
||||
runbook: infra/otel/runbooks.md#worker-saturation
|
||||
|
||||
- alert: TelemetryExportFailures
|
||||
expr: sum(rate(otelcol_exporter_send_failed_spans_total[5m])) + sum(rate(otelcol_exporter_send_failed_log_records_total[5m])) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: Collector is dropping traces or logs
|
||||
description: A telemetry backend is unavailable or rejecting data.
|
||||
runbook: infra/otel/runbooks.md#telemetry-backend
|
||||
@@ -0,0 +1,19 @@
|
||||
global:
|
||||
scrape_interval: 10s
|
||||
evaluation_interval: 10s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/prometheus-alerts.yaml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["prometheus:9090"]
|
||||
|
||||
- job_name: otel-collector-exported-metrics
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
|
||||
- job_name: otel-collector-internal
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Runbooks
|
||||
|
||||
## API high error rate
|
||||
|
||||
1. In Grafana, compare the 5xx ratio with `db_client_*` pool metrics.
|
||||
2. Open a failed server span in Tempo and inspect child JDBC/Redis/S3 spans.
|
||||
3. Search Loki by the trace ID. Do not search by a user, chat or message ID.
|
||||
4. Restore the failing dependency and confirm the alert resolves after the five-minute window.
|
||||
|
||||
## Queue lag
|
||||
|
||||
1. Check `messenger_queue_depth` and `messenger_queue_oldest_job_age_seconds`.
|
||||
2. Check worker busy/capacity and the rate of `messenger_jobs_processed_total` by outcome.
|
||||
3. Search worker errors in Loki, then open the linked consumer trace in Tempo.
|
||||
4. Restore or scale the worker and confirm depth and age return to zero.
|
||||
|
||||
## Worker saturation
|
||||
|
||||
1. Compare busy/capacity with job duration p95 and container CPU/RAM.
|
||||
2. Inspect slow consumer spans and their S3/JDBC children.
|
||||
3. Remove the bottleneck before increasing concurrency; memory is explicitly bounded.
|
||||
|
||||
## Telemetry backend
|
||||
|
||||
1. Inspect Collector exporter failure and queue metrics on port 8888 through Prometheus.
|
||||
2. Restore Loki or Tempo. Collector retries for at most 60 seconds with a 2048-item queue.
|
||||
3. Confirm the application remained available; telemetry loss is allowed and bounded.
|
||||
@@ -0,0 +1,31 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
live_store:
|
||||
max_trace_idle: 10s
|
||||
max_block_duration: 5m
|
||||
wal:
|
||||
path: /var/tempo/live-store/traces
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions
|
||||
cd /d "%~dp0\.."
|
||||
if not defined ENV_FILE set "ENV_FILE=.env"
|
||||
|
||||
docker compose --env-file "%ENV_FILE%" --profile verification run --rm private-bucket-check
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
|
||||
cd "$PROJECT_DIR"
|
||||
: "${ENV_FILE:=.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" --profile verification run --rm private-bucket-check
|
||||
@@ -0,0 +1,278 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://127.0.0.1:18080",
|
||||
[string]$EnvFile = ".env",
|
||||
[int]$LoadSeconds = 45,
|
||||
[int]$Concurrency = 4,
|
||||
[int]$WarmupSeconds = 10,
|
||||
[int]$StatsIntervalSeconds = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-observability-cost"
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
||||
|
||||
$modes = @(
|
||||
[ordered]@{ name = "off"; sdk_disabled = $true; sampler_arg = 0.0; metric_interval_ms = 60000 },
|
||||
[ordered]@{ name = "full"; sdk_disabled = $false; sampler_arg = 1.0; metric_interval_ms = 10000 },
|
||||
[ordered]@{ name = "economy"; sdk_disabled = $false; sampler_arg = 0.05; metric_interval_ms = 60000 },
|
||||
[ordered]@{ name = "balanced"; sdk_disabled = $false; sampler_arg = 0.25; metric_interval_ms = 30000 }
|
||||
)
|
||||
|
||||
function Query-PrometheusScalar([string]$expression) {
|
||||
$encoded = [uri]::EscapeDataString($expression)
|
||||
$response = Invoke-RestMethod "http://127.0.0.1:9090/api/v1/query?query=$encoded"
|
||||
if (@($response.data.result).Count -eq 0) { return 0.0 }
|
||||
return [double]$response.data.result[0].value[1]
|
||||
}
|
||||
|
||||
function Query-TextMetric([string]$url, [string]$metricName) {
|
||||
$content = (Invoke-WebRequest -UseBasicParsing $url).Content
|
||||
$sum = 0.0
|
||||
foreach ($line in ($content -split "`n")) {
|
||||
if ($line -match ('^' + [regex]::Escape($metricName) + '(?:\{[^}]*\})?\s+([0-9.eE+\-]+)\s*$')) {
|
||||
$sum += [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture)
|
||||
}
|
||||
}
|
||||
return $sum
|
||||
}
|
||||
|
||||
function Capture-IngestCounters {
|
||||
return [ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
collector_accepted_spans = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_spans_total) or vector(0)'
|
||||
collector_accepted_logs = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_log_records_total) or vector(0)'
|
||||
collector_accepted_metric_points = Query-PrometheusScalar 'sum(otelcol_receiver_accepted_metric_points_total) or vector(0)'
|
||||
collector_sent_spans = Query-PrometheusScalar 'sum(otelcol_exporter_sent_spans_total) or vector(0)'
|
||||
collector_sent_logs = Query-PrometheusScalar 'sum(otelcol_exporter_sent_log_records_total) or vector(0)'
|
||||
collector_failed_spans = Query-PrometheusScalar 'sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
|
||||
collector_failed_logs = Query-PrometheusScalar 'sum(otelcol_exporter_send_failed_log_records_total) or vector(0)'
|
||||
collector_enqueue_failed_spans = Query-PrometheusScalar 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)'
|
||||
collector_refused_spans = Query-PrometheusScalar 'sum(otelcol_receiver_refused_spans_total) or vector(0)'
|
||||
tempo_ingest_bytes = Query-TextMetric 'http://127.0.0.1:3200/metrics' 'tempo_distributor_bytes_received_total'
|
||||
tempo_ingest_spans = Query-TextMetric 'http://127.0.0.1:3200/metrics' 'tempo_distributor_spans_received_total'
|
||||
loki_line_bytes = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_bytes_received_total'
|
||||
loki_metadata_bytes = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_structured_metadata_bytes_received_total'
|
||||
loki_lines = Query-TextMetric 'http://127.0.0.1:3100/metrics' 'loki_distributor_lines_received_total'
|
||||
prometheus_appended_samples = Query-PrometheusScalar 'sum(prometheus_tsdb_head_samples_appended_total) or vector(0)'
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-MemoryToBytes([string]$value) {
|
||||
if ($value -notmatch '^([0-9.]+)\s*([KMG]iB)$') { return 0L }
|
||||
$number = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture)
|
||||
$factor = switch ($Matches[2]) {
|
||||
'KiB' { 1KB }
|
||||
'MiB' { 1MB }
|
||||
'GiB' { 1GB }
|
||||
}
|
||||
return [long]($number * $factor)
|
||||
}
|
||||
|
||||
function Wait-ApiReady {
|
||||
$deadline = (Get-Date).AddMinutes(4)
|
||||
do {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 3 "$BaseUrl/health/ready"
|
||||
if ($response.StatusCode -eq 200) { return }
|
||||
}
|
||||
catch { Start-Sleep -Seconds 2 }
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "API did not become ready at $BaseUrl"
|
||||
}
|
||||
|
||||
function Capture-ContainerStats([hashtable]$containerNames) {
|
||||
$names = @($containerNames.Values)
|
||||
$rows = @(docker stats --no-stream --format '{{json .}}' $names | ForEach-Object { $_ | ConvertFrom-Json })
|
||||
$captured = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
$result = @()
|
||||
foreach ($service in $containerNames.Keys) {
|
||||
$name = $containerNames[$service]
|
||||
$row = @($rows | Where-Object { $_.Name -eq $name })[0]
|
||||
if ($null -eq $row) { continue }
|
||||
$used = ($row.MemUsage -split '/')[0].Trim()
|
||||
$cpu = [double]::Parse(($row.CPUPerc -replace '%','').Trim(), [Globalization.CultureInfo]::InvariantCulture)
|
||||
$result += [pscustomobject][ordered]@{
|
||||
captured_at = $captured
|
||||
service = $service
|
||||
cpu_percent = $cpu
|
||||
memory_bytes = Convert-MemoryToBytes $used
|
||||
memory_display = $used
|
||||
}
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
function Delta([object]$before, [object]$after, [string]$property) {
|
||||
return [double]$after.$property - [double]$before.$property
|
||||
}
|
||||
|
||||
function Summarize-Stats([object[]]$samples, [string]$service) {
|
||||
$selected = @($samples | Where-Object { $_.service -eq $service })
|
||||
if ($selected.Count -eq 0) {
|
||||
return [ordered]@{ sample_count = 0; cpu_avg_percent = 0; cpu_peak_percent = 0; ram_avg_bytes = 0; ram_peak_bytes = 0 }
|
||||
}
|
||||
$cpu = $selected | Measure-Object -Property cpu_percent -Average -Maximum
|
||||
$ram = $selected | Measure-Object -Property memory_bytes -Average -Maximum
|
||||
return [ordered]@{
|
||||
sample_count = $selected.Count
|
||||
cpu_avg_percent = [Math]::Round($cpu.Average, 3)
|
||||
cpu_peak_percent = [Math]::Round($cpu.Maximum, 3)
|
||||
ram_avg_bytes = [long]$ram.Average
|
||||
ram_peak_bytes = [long]$ram.Maximum
|
||||
}
|
||||
}
|
||||
|
||||
$modeResults = @()
|
||||
$loadJob = $null
|
||||
Push-Location $project
|
||||
try {
|
||||
foreach ($mode in $modes) {
|
||||
Write-Host "=== mode=$($mode.name) sdk_disabled=$($mode.sdk_disabled) sampler=$($mode.sampler_arg) metrics_ms=$($mode.metric_interval_ms) ==="
|
||||
$env:OTEL_SDK_DISABLED = $mode.sdk_disabled.ToString().ToLowerInvariant()
|
||||
$env:OTEL_TRACES_SAMPLER = 'parentbased_traceidratio'
|
||||
$env:OTEL_TRACES_SAMPLER_ARG = [string]$mode.sampler_arg
|
||||
$env:OTEL_METRIC_EXPORT_INTERVAL = [string]$mode.metric_interval_ms
|
||||
|
||||
docker compose --env-file $EnvFile up -d --no-deps --force-recreate api worker | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) { throw "compose recreate failed for $($mode.name)" }
|
||||
Wait-ApiReady
|
||||
|
||||
$warmupFile = Join-Path $evidenceDir "$($mode.name)-warmup.json"
|
||||
& python tests/load.py --base-url $BaseUrl --duration $WarmupSeconds --concurrency 2 --output $warmupFile | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) { throw "warm-up failed for $($mode.name)" }
|
||||
Start-Sleep -Seconds 12
|
||||
|
||||
$before = Capture-IngestCounters
|
||||
$containerNames = @{}
|
||||
foreach ($service in @('api','worker','otel-collector')) {
|
||||
$id = docker compose --env-file $EnvFile ps -q $service
|
||||
if (-not $id) { throw "container is missing for $service" }
|
||||
$containerNames[$service] = (docker inspect --format '{{.Name}}' $id).TrimStart('/')
|
||||
}
|
||||
|
||||
$loadFile = Join-Path $evidenceDir "$($mode.name)-load.json"
|
||||
$loadJob = Start-Job -ArgumentList $project,$BaseUrl,$LoadSeconds,$Concurrency,$loadFile -ScriptBlock {
|
||||
param($projectPath,$url,$duration,$workers,$outputPath)
|
||||
Set-Location $projectPath
|
||||
$env:PYTHONUTF8 = "1"
|
||||
& python tests/load.py --base-url $url --duration $duration --concurrency $workers --output $outputPath
|
||||
[pscustomobject]@{ exit_code = $LASTEXITCODE }
|
||||
}
|
||||
|
||||
$samples = @()
|
||||
while ($loadJob.State -in @('NotStarted','Running')) {
|
||||
$samples += Capture-ContainerStats $containerNames
|
||||
Start-Sleep -Seconds $StatsIntervalSeconds
|
||||
}
|
||||
$jobOutput = @(Receive-Job $loadJob)
|
||||
$exitRecord = @($jobOutput | Where-Object { $_.PSObject.Properties.Name -contains 'exit_code' })[-1]
|
||||
if ($null -eq $exitRecord -or $exitRecord.exit_code -ne 0) { throw "load failed for $($mode.name)" }
|
||||
Remove-Job $loadJob
|
||||
$loadJob = $null
|
||||
|
||||
Start-Sleep -Seconds 12
|
||||
$after = Capture-IngestCounters
|
||||
$load = Get-Content $loadFile -Raw | ConvertFrom-Json
|
||||
$delta = [ordered]@{}
|
||||
foreach ($property in @(
|
||||
'collector_accepted_spans','collector_accepted_logs','collector_accepted_metric_points',
|
||||
'collector_sent_spans','collector_sent_logs','collector_failed_spans','collector_failed_logs',
|
||||
'collector_enqueue_failed_spans','collector_refused_spans','tempo_ingest_bytes','tempo_ingest_spans',
|
||||
'loki_line_bytes','loki_metadata_bytes','loki_lines','prometheus_appended_samples')) {
|
||||
$delta[$property] = Delta $before $after $property
|
||||
}
|
||||
$delta['loki_total_bytes'] = $delta.loki_line_bytes + $delta.loki_metadata_bytes
|
||||
$delta['backend_total_bytes'] = $delta.tempo_ingest_bytes + $delta.loki_total_bytes
|
||||
|
||||
$modeResult = [ordered]@{
|
||||
mode = $mode
|
||||
load = $load
|
||||
ingest_before = $before
|
||||
ingest_after = $after
|
||||
ingest_delta = $delta
|
||||
resources = [ordered]@{
|
||||
api = Summarize-Stats $samples 'api'
|
||||
worker = Summarize-Stats $samples 'worker'
|
||||
collector = Summarize-Stats $samples 'otel-collector'
|
||||
}
|
||||
}
|
||||
$modeResult | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "$($mode.name)-result.json")
|
||||
$samples | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "$($mode.name)-resource-samples.json")
|
||||
$modeResults += $modeResult
|
||||
}
|
||||
|
||||
$full = @($modeResults | Where-Object { $_.mode.name -eq 'full' })[0]
|
||||
$off = @($modeResults | Where-Object { $_.mode.name -eq 'off' })[0]
|
||||
$fullSpansPerRequest = [double]$full.ingest_delta.collector_accepted_spans / [Math]::Max(1, [double]$full.load.successful_requests)
|
||||
$comparison = @()
|
||||
foreach ($result in $modeResults) {
|
||||
$requests = [double]$result.load.successful_requests
|
||||
$spansPerRequest = [double]$result.ingest_delta.collector_accepted_spans / [Math]::Max(1, $requests)
|
||||
$expectedAtFull = $fullSpansPerRequest * $requests
|
||||
$notEmitted = [Math]::Max(0, $expectedAtFull - [double]$result.ingest_delta.collector_accepted_spans)
|
||||
$transportLost = [double]$result.ingest_delta.collector_failed_spans + [double]$result.ingest_delta.collector_enqueue_failed_spans + [double]$result.ingest_delta.collector_refused_spans
|
||||
$comparison += [pscustomobject][ordered]@{
|
||||
mode = $result.mode.name
|
||||
sdk_disabled = $result.mode.sdk_disabled
|
||||
trace_sample_ratio = $result.mode.sampler_arg
|
||||
metric_interval_ms = $result.mode.metric_interval_ms
|
||||
successful_requests = $result.load.successful_requests
|
||||
failed_requests = $result.load.failed_requests
|
||||
successful_rps = $result.load.successful_rps
|
||||
rps_change_vs_off_percent = [Math]::Round(100 * ([double]$result.load.successful_rps / [double]$off.load.successful_rps - 1), 2)
|
||||
latency_p50_ms = $result.load.successful_latency_ms.p50
|
||||
latency_p95_ms = $result.load.successful_latency_ms.p95
|
||||
latency_p99_ms = $result.load.successful_latency_ms.p99
|
||||
p95_change_vs_off_percent = [Math]::Round(100 * ([double]$result.load.successful_latency_ms.p95 / [double]$off.load.successful_latency_ms.p95 - 1), 2)
|
||||
api_cpu_avg_percent = $result.resources.api.cpu_avg_percent
|
||||
api_ram_peak_bytes = $result.resources.api.ram_peak_bytes
|
||||
worker_cpu_avg_percent = $result.resources.worker.cpu_avg_percent
|
||||
worker_ram_peak_bytes = $result.resources.worker.ram_peak_bytes
|
||||
collector_cpu_avg_percent = $result.resources.collector.cpu_avg_percent
|
||||
collector_ram_peak_bytes = $result.resources.collector.ram_peak_bytes
|
||||
accepted_spans = $result.ingest_delta.collector_accepted_spans
|
||||
accepted_logs = $result.ingest_delta.collector_accepted_logs
|
||||
accepted_metric_points = $result.ingest_delta.collector_accepted_metric_points
|
||||
spans_per_successful_request = [Math]::Round($spansPerRequest, 4)
|
||||
trace_retained_percent_of_full_normalized = if ($fullSpansPerRequest -gt 0) { [Math]::Round(100 * $spansPerRequest / $fullSpansPerRequest, 2) } else { 0 }
|
||||
estimated_spans_not_emitted_by_sdk = [Math]::Round($notEmitted, 0)
|
||||
collector_transport_lost_spans = $transportLost
|
||||
tempo_ingest_bytes = $result.ingest_delta.tempo_ingest_bytes
|
||||
loki_ingest_bytes = $result.ingest_delta.loki_total_bytes
|
||||
backend_trace_and_log_ingest_bytes = $result.ingest_delta.backend_total_bytes
|
||||
tempo_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.tempo_ingest_bytes / [Math]::Max(1, $requests), 2)
|
||||
loki_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.loki_total_bytes / [Math]::Max(1, $requests), 2)
|
||||
backend_bytes_per_successful_request = [Math]::Round([double]$result.ingest_delta.backend_total_bytes / [Math]::Max(1, $requests), 2)
|
||||
prometheus_appended_samples = $result.ingest_delta.prometheus_appended_samples
|
||||
}
|
||||
}
|
||||
|
||||
$comparison | Export-Csv -NoTypeInformation -Encoding utf8 (Join-Path $evidenceDir 'comparison.csv')
|
||||
[ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
methodology = [ordered]@{
|
||||
load_seconds = $LoadSeconds
|
||||
concurrency = $Concurrency
|
||||
warmup_seconds = $WarmupSeconds
|
||||
stats_interval_seconds = $StatsIntervalSeconds
|
||||
workload = 'closed-loop POST messages; identical profile per mode; API and worker recreated before each mode'
|
||||
sampling_loss_definition = 'estimated against full-mode accepted spans per successful request; it is SDK non-emission, not Collector transport loss'
|
||||
ingest_units = 'Collector items; Tempo/Loki uncompressed backend bytes; Prometheus appended samples'
|
||||
}
|
||||
comparison = $comparison
|
||||
} | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 (Join-Path $evidenceDir 'summary.json')
|
||||
|
||||
& make test BASE_URL=$BaseUrl PYTHON=python 2>&1 | Tee-Object -FilePath (Join-Path $evidenceDir 'balanced-make-test.txt') | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) { throw 'make test failed with balanced telemetry' }
|
||||
Write-Host "Evidence: $evidenceDir"
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $loadJob) { Stop-Job $loadJob -ErrorAction SilentlyContinue; Remove-Job $loadJob -Force -ErrorAction SilentlyContinue }
|
||||
Remove-Item Env:OTEL_SDK_DISABLED -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OTEL_TRACES_SAMPLER -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OTEL_TRACES_SAMPLER_ARG -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OTEL_METRIC_EXPORT_INTERVAL -ErrorAction SilentlyContinue
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
param(
|
||||
[int]$DurationSeconds = 165,
|
||||
[string]$BaseUrl = "http://localhost:8080",
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-database-stop"
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
||||
|
||||
Push-Location $project
|
||||
try {
|
||||
docker compose --env-file $EnvFile stop postgres
|
||||
$samples = @()
|
||||
$deadline = (Get-Date).AddSeconds($DurationSeconds)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$started = Get-Date
|
||||
try {
|
||||
$response = Invoke-WebRequest -Uri "$BaseUrl/health/ready" `
|
||||
-SkipHttpErrorCheck -TimeoutSec 5
|
||||
$readyStatus = [int]$response.StatusCode
|
||||
}
|
||||
catch {
|
||||
$readyStatus = 0
|
||||
}
|
||||
try {
|
||||
$response = Invoke-WebRequest -Uri "$BaseUrl/api/v1/users/me" `
|
||||
-Headers @{ Authorization = "Bearer incident-synthetic-token" } `
|
||||
-SkipHttpErrorCheck -TimeoutSec 5
|
||||
$businessStatus = [int]$response.StatusCode
|
||||
}
|
||||
catch {
|
||||
$businessStatus = 0
|
||||
}
|
||||
$samples += [ordered]@{
|
||||
at = $started.ToUniversalTime().ToString("o")
|
||||
ready_status = $readyStatus
|
||||
business_status = $businessStatus
|
||||
duration_ms = [math]::Round(((Get-Date) - $started).TotalMilliseconds, 1)
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
$samples | ConvertTo-Json -Depth 5 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "http-samples.json")
|
||||
|
||||
$alertsSnapshot = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
||||
$highErrorAlert = @($alertsSnapshot.data.alerts | Where-Object {
|
||||
$_.labels.alertname -eq "MessengerApiHighErrorRate" -and $_.state -eq "firing"
|
||||
})
|
||||
$observations = [ordered]@{
|
||||
captured_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
api_5xx = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=sum(rate(http_server_request_duration_seconds_count%7Bhttp_response_status_code%3D~%225..%22%7D%5B2m%5D))"
|
||||
alerts = $alertsSnapshot
|
||||
alert_firing = ($highErrorAlert.Count -gt 0)
|
||||
jdbc_spans_hint = "Grafana Explore -> Tempo: { resource.service.name = `"messenger-api`" && span.db.system.name = `"postgresql`" }"
|
||||
correlated_logs_hint = "Grafana Explore -> Loki: {service_name=`"messenger-api`"} | json | level=`"ERROR`""
|
||||
}
|
||||
$observations | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "failure-observation.json")
|
||||
if ($highErrorAlert.Count -eq 0) {
|
||||
throw "MessengerApiHighErrorRate did not reach firing state"
|
||||
}
|
||||
|
||||
$end = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() * 1000000
|
||||
$start = [DateTimeOffset]::UtcNow.AddMinutes(-10).ToUnixTimeMilliseconds() * 1000000
|
||||
$logQuery = [uri]::EscapeDataString('{service_name="messenger-api"}')
|
||||
Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$logQuery&start=$start&end=$end&limit=100&direction=backward" |
|
||||
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "loki-api-logs.json")
|
||||
$dbTraceQuery = [uri]::EscapeDataString('{ span.db.system = "postgresql" && status = error }')
|
||||
Invoke-RestMethod "http://localhost:3200/api/search?q=$dbTraceQuery&limit=100" |
|
||||
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-postgres-errors.json")
|
||||
$httpTraceQuery = [uri]::EscapeDataString('{ name = "GET /health/ready" && status = error }')
|
||||
Invoke-RestMethod "http://localhost:3200/api/search?q=$httpTraceQuery&limit=100" |
|
||||
ConvertTo-Json -Depth 30 | Set-Content -Encoding utf8 (Join-Path $evidenceDir "tempo-http-errors.json")
|
||||
}
|
||||
finally {
|
||||
docker compose --env-file $EnvFile start postgres
|
||||
try {
|
||||
$readyDeadline = (Get-Date).AddMinutes(2)
|
||||
do {
|
||||
Start-Sleep -Seconds 5
|
||||
try {
|
||||
$readyResponse = Invoke-WebRequest -Uri "$BaseUrl/health/ready" `
|
||||
-SkipHttpErrorCheck -TimeoutSec 10
|
||||
$ready = ([int]$readyResponse.StatusCode -eq 200)
|
||||
}
|
||||
catch {
|
||||
$ready = $false
|
||||
}
|
||||
} while (-not $ready -and (Get-Date) -lt $readyDeadline)
|
||||
|
||||
$alertDeadline = (Get-Date).AddMinutes(6)
|
||||
do {
|
||||
Start-Sleep -Seconds 10
|
||||
$recoveryAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
||||
$activeHighError = @($recoveryAlerts.data.alerts | Where-Object {
|
||||
$_.labels.alertname -eq "MessengerApiHighErrorRate"
|
||||
})
|
||||
} while ($activeHighError.Count -gt 0 -and (Get-Date) -lt $alertDeadline)
|
||||
|
||||
[ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
readiness_recovered = $ready
|
||||
alerts = $recoveryAlerts
|
||||
alert_resolved = ($activeHighError.Count -eq 0)
|
||||
api_5xx_ratio = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_response_status_code%3D~%225..%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%20or%20vector%280%29%29%20%2F%20clamp_min%28sum%28rate%28http_server_request_duration_seconds_count%7Bservice_name%3D%22messenger-api%22%2Chttp_route%21~%22%2Fhealth%2F.%2A%22%7D%5B5m%5D%29%29%2C%200.001%29"
|
||||
} | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "recovery-observation.json")
|
||||
if (-not $ready -or $activeHighError.Count -gt 0) {
|
||||
throw "database or MessengerApiHighErrorRate did not recover before the deadline"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Evidence saved to $evidenceDir"
|
||||
@@ -0,0 +1,117 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://localhost:8080",
|
||||
[string]$EnvFile = ".env",
|
||||
[int]$RecoveryWaitSeconds = 60
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-retry-trace"
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
||||
|
||||
function Ensure-Running([string]$container) {
|
||||
$state = docker inspect --format '{{.State.Status}}' $container
|
||||
if ($state -ne 'running') {
|
||||
docker start $container | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Push-Location $project
|
||||
try {
|
||||
$worker = docker compose --env-file $EnvFile ps -q worker
|
||||
$minio = docker compose --env-file $EnvFile ps -q minio
|
||||
if (-not $worker -or -not $minio) {
|
||||
throw "worker/minio containers do not exist; start the stack first"
|
||||
}
|
||||
|
||||
docker stop $worker | Out-Null
|
||||
$env:PYTHONUTF8 = "1"
|
||||
python tests/attachment_load.py --base-url $BaseUrl --count 1 --interval 0 `
|
||||
--output (Join-Path $evidenceDir "load.json")
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "synthetic upload failed"
|
||||
}
|
||||
|
||||
docker stop $minio | Out-Null
|
||||
$startedAt = [DateTimeOffset]::UtcNow
|
||||
$dockerSince = $startedAt.ToString("yyyy-MM-ddTHH:mm:ssZ")
|
||||
docker start $worker | Out-Null
|
||||
|
||||
$retrySeen = $false
|
||||
$deadline = (Get-Date).AddMinutes(4)
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$workerLogs = docker logs --since $dockerSince $worker 2>&1 | Out-String
|
||||
$retrySeen = $workerLogs.Contains("attachment_job_retry_scheduled")
|
||||
} while (-not $retrySeen -and (Get-Date) -lt $deadline)
|
||||
|
||||
if (-not $retrySeen) {
|
||||
throw "worker did not schedule a retry before the deadline"
|
||||
}
|
||||
Ensure-Running $minio
|
||||
Start-Sleep -Seconds $RecoveryWaitSeconds
|
||||
|
||||
$startNanos = $startedAt.ToUnixTimeMilliseconds() * 1000000
|
||||
$retryQuery = [uri]::EscapeDataString('{service_name="messenger-worker"} | event="attachment_job_retry_scheduled"')
|
||||
$retryResult = Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$retryQuery&start=$startNanos&limit=20&direction=forward"
|
||||
$retryStream = $retryResult.data.result | Where-Object { $_.stream.trace_id } | Select-Object -First 1
|
||||
if (-not $retryStream) {
|
||||
throw "retry event was not found in Loki"
|
||||
}
|
||||
$traceId = $retryStream.stream.trace_id
|
||||
|
||||
$successQueryText = '{service_name="messenger-worker"} | trace_id="' + $traceId + '" | event="attachment_job_completed"'
|
||||
$successQuery = [uri]::EscapeDataString($successQueryText)
|
||||
$successResult = Invoke-RestMethod "http://localhost:3100/loki/api/v1/query_range?query=$successQuery&start=$startNanos&limit=20&direction=forward"
|
||||
if (-not $successResult.data.result) {
|
||||
throw "successful retry completion was not found in Loki"
|
||||
}
|
||||
|
||||
$trace = Invoke-RestMethod "http://localhost:3200/api/traces/$traceId"
|
||||
$spans = @()
|
||||
foreach ($batch in $trace.batches) {
|
||||
$service = ($batch.resource.attributes | Where-Object key -eq 'service.name').value.stringValue
|
||||
foreach ($scope in $batch.scopeSpans) {
|
||||
foreach ($span in $scope.spans) {
|
||||
if ($span.name -match 'attachments|publish|process|PUT|DELETE|INSERT|UPDATE|SELECT') {
|
||||
$spans += [ordered]@{
|
||||
service = $service
|
||||
name = $span.name
|
||||
kind = $span.kind
|
||||
span_id = $span.spanId
|
||||
parent_span_id = $span.parentSpanId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$producerCount = @($spans | Where-Object kind -eq 'SPAN_KIND_PRODUCER').Count
|
||||
$consumerCount = @($spans | Where-Object kind -eq 'SPAN_KIND_CONSUMER').Count
|
||||
if ($producerCount -lt 2 -or $consumerCount -lt 2) {
|
||||
throw "trace does not contain both initial and retry producer/consumer spans"
|
||||
}
|
||||
|
||||
[ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
trace_id = $traceId
|
||||
retry_event = $retryStream.stream.event
|
||||
retry_attempt = $retryStream.stream.attempt
|
||||
retry_error_code = $retryStream.stream.error_code
|
||||
completion_event = "attachment_job_completed"
|
||||
producer_spans = $producerCount
|
||||
consumer_spans = $consumerCount
|
||||
same_trace = $true
|
||||
synthetic_data_only = $true
|
||||
spans = $spans
|
||||
} | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "retry-trace-summary.json")
|
||||
}
|
||||
finally {
|
||||
if ($minio) { Ensure-Running $minio }
|
||||
if ($worker) { Ensure-Running $worker }
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
Write-Host "Evidence saved to $evidenceDir"
|
||||
@@ -0,0 +1,149 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://localhost:8080",
|
||||
[string]$EnvFile = ".env",
|
||||
[int]$LoadSeconds = 75,
|
||||
[int]$Concurrency = 4
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-telemetry-backend"
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
||||
|
||||
function Query-Scalar([string]$expression) {
|
||||
try {
|
||||
$encoded = [uri]::EscapeDataString($expression)
|
||||
$response = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=$encoded"
|
||||
if (@($response.data.result).Count -eq 0) { return 0.0 }
|
||||
return [double]$response.data.result[0].value[1]
|
||||
}
|
||||
catch { return 0.0 }
|
||||
}
|
||||
|
||||
function Convert-MemoryToBytes([string]$value) {
|
||||
if ($value -notmatch '^([0-9.]+)([KMG]iB)$') { return 0 }
|
||||
$number = [double]$Matches[1]
|
||||
$factor = switch ($Matches[2]) {
|
||||
'KiB' { 1KB }
|
||||
'MiB' { 1MB }
|
||||
'GiB' { 1GB }
|
||||
}
|
||||
return [long]($number * $factor)
|
||||
}
|
||||
|
||||
function Capture-Sample {
|
||||
$stats = docker stats --no-stream --format '{{json .}}' $collector | ConvertFrom-Json
|
||||
$used = ($stats.MemUsage -split '/')[0].Trim()
|
||||
return [ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
collector_memory = $used
|
||||
collector_memory_bytes = Convert-MemoryToBytes $used
|
||||
collector_memory_percent = $stats.MemPerc
|
||||
exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)'
|
||||
exporter_queue_capacity = Query-Scalar 'max(otelcol_exporter_queue_capacity)'
|
||||
failed_spans_total = Query-Scalar 'sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
|
||||
failed_logs_total = Query-Scalar 'sum(otelcol_exporter_send_failed_log_records_total) or vector(0)'
|
||||
}
|
||||
}
|
||||
|
||||
function Ensure-Running([string]$container) {
|
||||
if ($container -and (docker inspect --format '{{.State.Status}}' $container) -ne 'running') {
|
||||
docker start $container | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$samples = @()
|
||||
$loadJob = $null
|
||||
Push-Location $project
|
||||
try {
|
||||
$collector = docker compose --env-file $EnvFile ps -q otel-collector
|
||||
$loki = docker compose --env-file $EnvFile ps -q loki
|
||||
$tempo = docker compose --env-file $EnvFile ps -q tempo
|
||||
if (-not $collector -or -not $loki -or -not $tempo) {
|
||||
throw "collector/loki/tempo containers do not exist; start the stack first"
|
||||
}
|
||||
|
||||
$samples += Capture-Sample
|
||||
docker stop $loki $tempo | Out-Null
|
||||
|
||||
$makeOutput = & make test BASE_URL=$BaseUrl PYTHON=python 2>&1
|
||||
$makeExit = $LASTEXITCODE
|
||||
$makeOutput | Set-Content -Encoding utf8 (Join-Path $evidenceDir "make-test.txt")
|
||||
|
||||
$loadOutput = Join-Path $evidenceDir "message-load.json"
|
||||
$loadJob = Start-Job -ArgumentList $project,$BaseUrl,$LoadSeconds,$Concurrency,$loadOutput -ScriptBlock {
|
||||
param($projectPath,$url,$duration,$workers,$outputPath)
|
||||
Set-Location $projectPath
|
||||
$env:PYTHONUTF8 = "1"
|
||||
& python tests/load.py --base-url $url --duration $duration `
|
||||
--concurrency $workers --output $outputPath
|
||||
[pscustomobject]@{ exit_code = $LASTEXITCODE }
|
||||
}
|
||||
|
||||
while ($loadJob.State -in @('NotStarted','Running')) {
|
||||
$samples += Capture-Sample
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
$loadJobResult = Receive-Job $loadJob
|
||||
$loadExit = @($loadJobResult | Where-Object { $_.PSObject.Properties.Name -contains 'exit_code' })[-1].exit_code
|
||||
$samples += Capture-Sample
|
||||
|
||||
$maxMemory = (($samples | ForEach-Object { [long]$_['collector_memory_bytes'] }) |
|
||||
Measure-Object -Maximum).Maximum
|
||||
$maxQueue = (($samples | ForEach-Object { [double]$_['exporter_queue_size'] }) |
|
||||
Measure-Object -Maximum).Maximum
|
||||
$maxFailedSpans = (($samples | ForEach-Object { [double]$_['failed_spans_total'] }) |
|
||||
Measure-Object -Maximum).Maximum
|
||||
$maxFailedLogs = (($samples | ForEach-Object { [double]$_['failed_logs_total'] }) |
|
||||
Measure-Object -Maximum).Maximum
|
||||
$loadReport = Get-Content $loadOutput -Raw | ConvertFrom-Json
|
||||
|
||||
$samples | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "collector-samples.json")
|
||||
[ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
make_test_passed = ($makeExit -eq 0 -and ($makeOutput -join "`n") -match 'PASS:')
|
||||
message_load_exit_code = $loadExit
|
||||
successful_message_requests = $loadReport.successful_requests
|
||||
failed_message_requests = $loadReport.failed_requests
|
||||
collector_memory_limit_bytes = 536870912
|
||||
collector_memory_limiter_bytes = 402653184
|
||||
max_collector_memory_bytes = $maxMemory
|
||||
max_exporter_queue_size = $maxQueue
|
||||
exporter_queue_capacity = 2048
|
||||
failed_spans_total = $maxFailedSpans
|
||||
failed_log_records_total = $maxFailedLogs
|
||||
bounded = ($maxMemory -lt 536870912 -and $maxQueue -le 2048)
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "failure-observation.json")
|
||||
|
||||
if ($makeExit -ne 0 -or $loadExit -ne 0 -or $loadReport.successful_requests -lt 1) {
|
||||
throw "business traffic failed while telemetry backends were unavailable"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($loadJob) { Remove-Job $loadJob -Force -ErrorAction SilentlyContinue }
|
||||
if ($loki) { Ensure-Running $loki }
|
||||
if ($tempo) { Ensure-Running $tempo }
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$recoveryDeadline = (Get-Date).AddMinutes(2)
|
||||
$recovery = [ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString("o")
|
||||
loki_ready = $false
|
||||
tempo_ready = $false
|
||||
exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)'
|
||||
}
|
||||
do {
|
||||
Start-Sleep -Seconds 5
|
||||
try { $recovery.loki_ready = ((Invoke-WebRequest http://localhost:3100/ready -TimeoutSec 5).StatusCode -eq 200) } catch {}
|
||||
try { $recovery.tempo_ready = ((Invoke-WebRequest http://localhost:3200/ready -TimeoutSec 5).StatusCode -eq 200) } catch {}
|
||||
} while ((-not $recovery.loki_ready -or -not $recovery.tempo_ready) -and
|
||||
(Get-Date) -lt $recoveryDeadline)
|
||||
$recovery.exporter_queue_size = Query-Scalar 'max(otelcol_exporter_queue_size)'
|
||||
$recovery | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "recovery-observation.json")
|
||||
|
||||
Write-Host "Evidence saved to $evidenceDir"
|
||||
@@ -0,0 +1,85 @@
|
||||
param(
|
||||
[int]$DurationSeconds = 150,
|
||||
[int]$JobCount = 10,
|
||||
[string]$BaseUrl = "http://localhost:8080",
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$evidenceDir = Join-Path $project "evidence\lab6-$stamp-worker-stop"
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir | Out-Null
|
||||
|
||||
Push-Location $project
|
||||
try {
|
||||
$workerContainer = docker compose --env-file $EnvFile ps -q worker
|
||||
if (-not $workerContainer) {
|
||||
throw "worker container does not exist; start the stack first"
|
||||
}
|
||||
docker stop $workerContainer
|
||||
python tests/attachment_load.py --base-url $BaseUrl --count $JobCount `
|
||||
--output (Join-Path $evidenceDir "load.json")
|
||||
Start-Sleep -Seconds $DurationSeconds
|
||||
|
||||
$failureAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
||||
$firingQueueAlert = @($failureAlerts.data.alerts | Where-Object {
|
||||
$_.labels.alertname -eq "AttachmentQueueLagHigh" -and $_.state -eq "firing"
|
||||
})
|
||||
$queries = [ordered]@{
|
||||
captured_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
queue_depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)"
|
||||
oldest_job_age = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_oldest_job_age_seconds)"
|
||||
alerts = $failureAlerts
|
||||
alert_firing = ($firingQueueAlert.Count -gt 0)
|
||||
}
|
||||
$queries | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "failure-observation.json")
|
||||
if ($firingQueueAlert.Count -eq 0) {
|
||||
throw "AttachmentQueueLagHigh did not reach firing state"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($workerContainer) {
|
||||
docker start $workerContainer
|
||||
}
|
||||
$deadline = (Get-Date).AddMinutes(4)
|
||||
do {
|
||||
Start-Sleep -Seconds 5
|
||||
try {
|
||||
$depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)"
|
||||
$remaining = [double]$depth.data.result[0].value[1]
|
||||
}
|
||||
catch {
|
||||
$remaining = [double]::PositiveInfinity
|
||||
}
|
||||
} while ($remaining -gt 0 -and (Get-Date) -lt $deadline)
|
||||
$alertDeadline = (Get-Date).AddMinutes(2)
|
||||
do {
|
||||
Start-Sleep -Seconds 5
|
||||
$recoveryAlerts = Invoke-RestMethod "http://localhost:9090/api/v1/alerts"
|
||||
$queueAlert = @($recoveryAlerts.data.alerts | Where-Object {
|
||||
$_.labels.alertname -eq "AttachmentQueueLagHigh"
|
||||
})
|
||||
} while ($queueAlert.Count -gt 0 -and (Get-Date) -lt $alertDeadline)
|
||||
try {
|
||||
$recovery = [ordered]@{
|
||||
captured_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
queue_depth = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_depth)"
|
||||
oldest_job_age = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=max(messenger_queue_oldest_job_age_seconds)"
|
||||
jobs = Invoke-RestMethod "http://localhost:9090/api/v1/query?query=sum%20by%20(outcome)%20(increase(messenger_jobs_processed_total%5B10m%5D))"
|
||||
alerts = $recoveryAlerts
|
||||
alert_resolved = ($queueAlert.Count -eq 0)
|
||||
}
|
||||
$recovery | ConvertTo-Json -Depth 20 | Set-Content -Encoding utf8 `
|
||||
(Join-Path $evidenceDir "recovery-observation.json")
|
||||
if ($queueAlert.Count -gt 0) {
|
||||
throw "AttachmentQueueLagHigh did not resolve before the deadline"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Evidence saved to $evidenceDir"
|
||||
@@ -0,0 +1,92 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://127.0.0.1:18080",
|
||||
[string]$EnvFile = ".env",
|
||||
[int]$RequestCount = 40,
|
||||
[string]$OutputFile = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$project = Split-Path -Parent $PSScriptRoot
|
||||
if (-not $OutputFile) {
|
||||
$OutputFile = Join-Path $project "evidence\head-sampling-probe.json"
|
||||
}
|
||||
|
||||
function Get-TempoStatus([string]$traceId) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 "http://127.0.0.1:3200/api/traces/$traceId"
|
||||
return [int]$response.StatusCode
|
||||
}
|
||||
catch {
|
||||
if ($_.Exception.Response) { return [int]$_.Exception.Response.StatusCode }
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
Push-Location $project
|
||||
try {
|
||||
$apiId = docker compose --env-file $EnvFile ps -q api
|
||||
if (-not $apiId) { throw 'API container is not running' }
|
||||
$since = [DateTimeOffset]::UtcNow.AddSeconds(-2)
|
||||
$statuses = @{}
|
||||
for ($i = 0; $i -lt $RequestCount; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 "$BaseUrl/health/not-found"
|
||||
$status = [int]$response.StatusCode
|
||||
}
|
||||
catch {
|
||||
$status = if ($_.Exception.Response) { [int]$_.Exception.Response.StatusCode } else { 0 }
|
||||
}
|
||||
$key = [string]$status
|
||||
$statuses[$key] = 1 + [int]$statuses[$key]
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 15
|
||||
$rawLogs = @(docker logs --since $since.ToString('o') $apiId 2>&1)
|
||||
$events = @()
|
||||
foreach ($line in $rawLogs) {
|
||||
$text = [string]$line
|
||||
$jsonStart = $text.IndexOf('{')
|
||||
if ($jsonStart -lt 0) { continue }
|
||||
try { $entry = $text.Substring($jsonStart) | ConvertFrom-Json } catch { continue }
|
||||
if ($entry.event -eq 'http_request_completed' -and [int]$entry.'http.response.status_code' -eq 500) {
|
||||
$flags = [Convert]::ToInt32([string]$entry.trace_flags, 16)
|
||||
$events += [pscustomobject]@{
|
||||
trace_id = [string]$entry.trace_id
|
||||
sampled = (($flags -band 1) -eq 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$sampled = @($events | Where-Object sampled)
|
||||
$unsampled = @($events | Where-Object { -not $_.sampled })
|
||||
if ($sampled.Count -eq 0 -or $unsampled.Count -eq 0) {
|
||||
throw "Expected both sampled and unsampled 500 events; sampled=$($sampled.Count), unsampled=$($unsampled.Count)"
|
||||
}
|
||||
$sampledId = $sampled[0].trace_id
|
||||
$unsampledId = $unsampled[0].trace_id
|
||||
$result = [ordered]@{
|
||||
captured_at = [DateTimeOffset]::UtcNow.ToString('o')
|
||||
request_count = $RequestCount
|
||||
response_status_counts = $statuses
|
||||
correlated_log_count = $events.Count
|
||||
sampled_log_count = $sampled.Count
|
||||
unsampled_log_count = $unsampled.Count
|
||||
sampled_ratio = [Math]::Round($sampled.Count / [double]$events.Count, 4)
|
||||
sampled_example = [ordered]@{
|
||||
trace_id = $sampledId
|
||||
tempo_http_status = Get-TempoStatus $sampledId
|
||||
}
|
||||
unsampled_example = [ordered]@{
|
||||
trace_id = $unsampledId
|
||||
tempo_http_status = Get-TempoStatus $unsampledId
|
||||
}
|
||||
interpretation = 'Head sampling is decided before the HTTP outcome: a non-2xx log can exist even when its trace was not emitted to Tempo.'
|
||||
}
|
||||
$parent = Split-Path -Parent $OutputFile
|
||||
New-Item -ItemType Directory -Force -Path $parent | Out-Null
|
||||
$result | ConvertTo-Json -Depth 10 | Set-Content -Encoding utf8 $OutputFile
|
||||
$result | ConvertTo-Json -Depth 10
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$AttachmentId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2)]
|
||||
[int]$PipelineVersion,
|
||||
[string]$RequestedBy = $env:USERNAME,
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$parsedAttachmentId = [guid]::Empty
|
||||
if (-not [guid]::TryParse($AttachmentId, [ref]$parsedAttachmentId)) {
|
||||
throw "AttachmentId must be a UUID"
|
||||
}
|
||||
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
|
||||
throw "RequestedBy contains unsupported characters"
|
||||
}
|
||||
|
||||
$sql = @"
|
||||
WITH candidate AS (
|
||||
SELECT id, pipeline_version
|
||||
FROM attachments
|
||||
WHERE id = '$parsedAttachmentId'::uuid AND pipeline_version < $PipelineVersion
|
||||
FOR UPDATE
|
||||
), superseded AS (
|
||||
UPDATE attachment_jobs j
|
||||
SET status = 'superseded', published = true,
|
||||
last_error_code = 'superseded_by_v$PipelineVersion',
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c
|
||||
WHERE j.attachment_id = c.id AND j.pipeline_version < $PipelineVersion
|
||||
AND j.status IN ('queued', 'processing')
|
||||
RETURNING j.id
|
||||
), created AS (
|
||||
INSERT INTO attachment_jobs
|
||||
(id, attachment_id, pipeline_version, attempt, status, published,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), id, $PipelineVersion, 1, 'queued', false,
|
||||
clock_timestamp(), clock_timestamp(), clock_timestamp()
|
||||
FROM candidate
|
||||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||||
RETURNING id, attachment_id
|
||||
), promoted AS (
|
||||
UPDATE attachments a
|
||||
SET pipeline_version = $PipelineVersion, status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM candidate c, created n
|
||||
WHERE a.id = c.id AND n.attachment_id = c.id
|
||||
RETURNING a.id
|
||||
), audited AS (
|
||||
INSERT INTO attachment_pipeline_promotions
|
||||
(attachment_id, from_version, to_version, job_id, requested_by)
|
||||
SELECT c.id, c.pipeline_version, $PipelineVersion, n.id, '$RequestedBy'
|
||||
FROM candidate c
|
||||
JOIN created n ON n.attachment_id = c.id
|
||||
JOIN promoted p ON p.id = c.id
|
||||
RETURNING job_id
|
||||
)
|
||||
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');
|
||||
"@
|
||||
|
||||
$result = docker compose --env-file $EnvFile exec -T postgres `
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Pipeline promotion transaction failed"
|
||||
}
|
||||
|
||||
$last = ($result | Select-Object -Last 1).Trim()
|
||||
if ($last -eq "no-op") {
|
||||
Write-Output "No-op: attachment is missing, already at v$PipelineVersion, or that version already exists"
|
||||
} else {
|
||||
Write-Output "Promoted attachment $parsedAttachmentId to pipeline v$PipelineVersion with job $last"
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
attachment_id="${1:?usage: promote-pipeline.sh ATTACHMENT_UUID PIPELINE_VERSION [REQUESTED_BY] [ENV_FILE]}"
|
||||
pipeline_version="${2:?pipeline version is required}"
|
||||
requested_by="${3:-operator}"
|
||||
env_file="${4:-.env}"
|
||||
|
||||
case "$attachment_id" in
|
||||
????????-????-????-????-????????????) ;;
|
||||
*) echo "AttachmentId must be a UUID" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$pipeline_version" in
|
||||
1|2) ;;
|
||||
*) echo "PipelineVersion must be 1 or 2" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$requested_by" in
|
||||
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
sql="WITH candidate AS (
|
||||
SELECT id, pipeline_version FROM attachments
|
||||
WHERE id = '$attachment_id'::uuid AND pipeline_version < $pipeline_version FOR UPDATE
|
||||
), superseded AS (
|
||||
UPDATE attachment_jobs j SET status = 'superseded', published = true,
|
||||
last_error_code = 'superseded_by_v$pipeline_version',
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c WHERE j.attachment_id = c.id
|
||||
AND j.pipeline_version < $pipeline_version AND j.status IN ('queued', 'processing')
|
||||
RETURNING j.id
|
||||
), created AS (
|
||||
INSERT INTO attachment_jobs
|
||||
(id, attachment_id, pipeline_version, attempt, status, published,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), id, $pipeline_version, 1, 'queued', false,
|
||||
clock_timestamp(), clock_timestamp(), clock_timestamp() FROM candidate
|
||||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||||
RETURNING id, attachment_id
|
||||
), promoted AS (
|
||||
UPDATE attachments a SET pipeline_version = $pipeline_version,
|
||||
status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, created n WHERE a.id = c.id AND n.attachment_id = c.id
|
||||
RETURNING a.id
|
||||
), audited AS (
|
||||
INSERT INTO attachment_pipeline_promotions
|
||||
(attachment_id, from_version, to_version, job_id, requested_by)
|
||||
SELECT c.id, c.pipeline_version, $pipeline_version, n.id, '$requested_by'
|
||||
FROM candidate c JOIN created n ON n.attachment_id = c.id
|
||||
JOIN promoted p ON p.id = c.id RETURNING job_id
|
||||
)
|
||||
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');"
|
||||
|
||||
result="$(docker compose --env-file "$env_file" exec -T postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
|
||||
if [ "$result" = "no-op" ]; then
|
||||
echo "No-op: attachment is missing, already at v$pipeline_version, or that version already exists"
|
||||
else
|
||||
echo "Promoted attachment $attachment_id to pipeline v$pipeline_version with job $result"
|
||||
fi
|
||||
@@ -0,0 +1,58 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobId,
|
||||
[string]$RequestedBy = $env:USERNAME,
|
||||
[string]$EnvFile = ".env"
|
||||
)
|
||||
|
||||
$parsedJobId = [guid]::Empty
|
||||
if (-not [guid]::TryParse($JobId, [ref]$parsedJobId)) {
|
||||
throw "JobId must be a UUID"
|
||||
}
|
||||
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
|
||||
throw "RequestedBy contains unsupported characters"
|
||||
}
|
||||
|
||||
$sql = @"
|
||||
WITH candidate AS (
|
||||
SELECT j.id, j.attachment_id, j.attempt
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$parsedJobId'::uuid
|
||||
AND j.status = 'failed' AND a.status = 'failed'
|
||||
AND j.pipeline_version = a.pipeline_version
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
SELECT id, '$RequestedBy', attempt FROM candidate
|
||||
RETURNING job_id
|
||||
), reset_job AS (
|
||||
UPDATE attachment_jobs j
|
||||
SET status = 'queued', attempt = 1, published = false,
|
||||
last_error_code = NULL, next_attempt_at = clock_timestamp(),
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, audited a
|
||||
WHERE j.id = c.id AND a.job_id = c.id
|
||||
RETURNING j.attachment_id
|
||||
), reset_attachment AS (
|
||||
UPDATE attachments a
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
FROM reset_job r
|
||||
WHERE a.id = r.attachment_id AND a.status = 'failed'
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
|
||||
THEN 'replayed' ELSE 'no-op' END;
|
||||
"@
|
||||
|
||||
$result = docker compose --env-file $EnvFile exec -T postgres `
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Replay transaction failed"
|
||||
}
|
||||
|
||||
if (($result | Select-Object -Last 1).Trim() -eq "replayed") {
|
||||
Write-Output "Replay requested for failed job $parsedJobId by $RequestedBy"
|
||||
} else {
|
||||
Write-Output "No-op: job $parsedJobId is missing, not failed, or already recovered"
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
job_id="${1:?usage: replay-failed.sh JOB_UUID [REQUESTED_BY] [ENV_FILE]}"
|
||||
requested_by="${2:-operator}"
|
||||
env_file="${3:-.env}"
|
||||
|
||||
case "$job_id" in
|
||||
????????-????-????-????-????????????) ;;
|
||||
*) echo "JobId must be a UUID" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$requested_by" in
|
||||
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
sql="WITH candidate AS (
|
||||
SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed'
|
||||
AND j.pipeline_version = a.pipeline_version
|
||||
FOR UPDATE OF j, a
|
||||
), audited AS (
|
||||
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
|
||||
SELECT id, '$requested_by', attempt FROM candidate RETURNING job_id
|
||||
), reset_job AS (
|
||||
UPDATE attachment_jobs j SET status = 'queued', attempt = 1, published = false,
|
||||
last_error_code = NULL, next_attempt_at = clock_timestamp(),
|
||||
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
|
||||
FROM candidate c, audited a WHERE j.id = c.id AND a.job_id = c.id
|
||||
RETURNING j.attachment_id
|
||||
), reset_attachment AS (
|
||||
UPDATE attachments a SET status = 'queued', error_code = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM reset_job r WHERE a.id = r.attachment_id AND a.status = 'failed'
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
|
||||
THEN 'replayed' ELSE 'no-op' END;"
|
||||
|
||||
result="$(docker compose --env-file "$env_file" exec -T postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
|
||||
if [ "$result" = "replayed" ]; then
|
||||
echo "Replay requested for failed job $job_id by $requested_by"
|
||||
else
|
||||
echo "No-op: job $job_id is missing, not failed, or already recovered"
|
||||
fi
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions
|
||||
cd /d "%~dp0\.."
|
||||
if not defined ENV_FILE set "ENV_FILE=.env"
|
||||
|
||||
docker compose --env-file "%ENV_FILE%" stop nginx api redis postgres minio || exit /b 1
|
||||
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 postgres redis minio || exit /b 1
|
||||
docker compose --env-file "%ENV_FILE%" run --rm minio-init || exit /b 1
|
||||
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api || exit /b 1
|
||||
docker compose --env-file "%ENV_FILE%" up -d --no-deps nginx || exit /b 1
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
|
||||
cd "$PROJECT_DIR"
|
||||
: "${ENV_FILE:=.env}"
|
||||
|
||||
# Preserve named volumes and the network. The migration job is intentionally
|
||||
# not re-run during a same-version restart.
|
||||
docker compose --env-file "$ENV_FILE" stop nginx api redis postgres minio
|
||||
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 postgres redis minio
|
||||
docker compose --env-file "$ENV_FILE" run --rm minio-init
|
||||
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api
|
||||
docker compose --env-file "$ENV_FILE" up -d --no-deps nginx
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions
|
||||
cd /d "%~dp0\.."
|
||||
if not defined ENV_FILE set "ENV_FILE=.env"
|
||||
|
||||
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
|
||||
cd "$PROJECT_DIR"
|
||||
: "${ENV_FILE:=.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 \
|
||||
--no-deps --scale api=2 api
|
||||
@@ -0,0 +1,16 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions
|
||||
cd /d "%~dp0\.."
|
||||
|
||||
set /a REPLICA_COUNT=0
|
||||
for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do set /a REPLICA_COUNT+=1
|
||||
if %REPLICA_COUNT% LSS 2 exit /b 2
|
||||
|
||||
for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do call :replace_one %%C
|
||||
exit /b 0
|
||||
|
||||
:replace_one
|
||||
docker stop --time 10 %1 >nul || exit /b 1
|
||||
docker rm %1 >nul || exit /b 1
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api >nul || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
original_ids=$(docker compose --env-file .env ps -q api)
|
||||
count=$(printf '%s\n' "$original_ids" | sed '/^$/d' | wc -l | tr -d ' ')
|
||||
if [ "$count" -lt 2 ]; then
|
||||
printf '%s\n' 'Rolling update requires at least two API replicas' >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
for container_id in $original_ids; do
|
||||
# Replace exactly one original replica and wait for the replacement before
|
||||
# touching the next one. Named volumes and state stores are not changed.
|
||||
docker stop --time 10 "$container_id" >/dev/null
|
||||
docker rm "$container_id" >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 90 \
|
||||
--no-deps --scale api=2 api >/dev/null
|
||||
done
|
||||
@@ -0,0 +1,26 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions EnableDelayedExpansion
|
||||
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
cd /d "%~dp0\.."
|
||||
if not defined ENV_FILE set "ENV_FILE=.env"
|
||||
|
||||
set "TARGET_CONTAINER="
|
||||
set /a MATCHES=0
|
||||
for /f "usebackq delims=" %%C in (`docker compose --env-file "%ENV_FILE%" ps -q api`) do (
|
||||
set "CONTAINER_ID=%%C"
|
||||
set "SHORT_ID=!CONTAINER_ID:~0,12!"
|
||||
set "HOSTNAME_VALUE="
|
||||
for /f "usebackq delims=" %%H in (`docker inspect --format "{{.Config.Hostname}}" %%C`) do set "HOSTNAME_VALUE=%%H"
|
||||
if "!HOSTNAME_VALUE!"=="%TARGET_INSTANCE_ID%" (
|
||||
set "TARGET_CONTAINER=%%C"
|
||||
set /a MATCHES+=1
|
||||
) else if "!SHORT_ID!"=="%TARGET_INSTANCE_ID%" (
|
||||
set "TARGET_CONTAINER=%%C"
|
||||
set /a MATCHES+=1
|
||||
)
|
||||
)
|
||||
|
||||
if not "!MATCHES!"=="1" exit /b 2
|
||||
docker stop --time 5 "!TARGET_CONTAINER!" >nul
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
|
||||
cd "$PROJECT_DIR"
|
||||
: "${ENV_FILE:=.env}"
|
||||
|
||||
target=""
|
||||
matches=0
|
||||
for container_id in $(docker compose --env-file "$ENV_FILE" ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
short_id=$(printf '%s' "$container_id" | cut -c1-12)
|
||||
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$short_id" ]; then
|
||||
target=$container_id
|
||||
matches=$((matches + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$matches" -ne 1 ]; then
|
||||
printf '%s\n' "Expected one API replica for X-Instance-Id, found $matches" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Five seconds keeps the operator hook below the test's ten-second budget while
|
||||
# still delivering SIGTERM and allowing Spring's graceful shutdown phase.
|
||||
docker stop --time 5 "$target" >/dev/null
|
||||
@@ -0,0 +1,19 @@
|
||||
$ErrorActionPreference = 'Continue'
|
||||
if (Test-Path variable:PSNativeCommandUseErrorActionPreference) {
|
||||
$PSNativeCommandUseErrorActionPreference = $false
|
||||
}
|
||||
|
||||
$nodes = @('hla-swarm-manager', 'hla-swarm-worker1', 'hla-swarm-worker2', 'hla-swarm-registry')
|
||||
foreach ($node in $nodes) {
|
||||
$existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}')
|
||||
if ($existing -eq $node) {
|
||||
docker rm --force $node | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Cannot remove $node" }
|
||||
}
|
||||
}
|
||||
$existingNetwork = (& docker network ls --filter 'name=^hla-swarm-underlay$' --format '{{.Name}}')
|
||||
if ($existingNetwork -eq 'hla-swarm-underlay') {
|
||||
docker network rm hla-swarm-underlay | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Cannot remove hla-swarm-underlay' }
|
||||
}
|
||||
Write-Output 'Removed the disposable three-node Swarm lab and its nested volumes.'
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user