From 4ab9b0ef9a163da763478f9867246d827dc9e4dd Mon Sep 17 00:00:00 2001 From: Khakimov Dmitrij Date: Wed, 18 Nov 2026 21:37:25 +0300 Subject: [PATCH] lab6 done --- .dockerignore | 20 +- .github/pull_request_template.md | 49 +- Dockerfile | 67 + Dockerfile.storage | 34 + INCIDENTS.md | 67 + Makefile | 24 +- REPORT.md | 221 +- RESEARCH.md | 154 + RUNBOOK.md | 58 + cmd/main.go | 121 + cmd/reconcile/main.go | 57 + cmd/worker/main.go | 123 + compose.yaml | 250 ++ contracts/embed.go | 9 + dto/create_chat.go | 13 + dto/create_message.go | 16 + dto/create_message_json.go | 33 + dto/create_message_json_test.go | 21 + dto/create_user.go | 14 + dto/current_user.go | 10 + dto/error.go | 11 + dto/get_chat.go | 12 + dto/list_chats.go | 18 + dto/list_messages.go | 21 + dto/liveness.go | 7 + dto/readiness.go | 7 + dto/session.go | 8 + dto/timestamp.go | 11 + dto/timestamp_test.go | 15 + evidence/README.md | 25 +- evidence/lab6-backend-outage.json | 220 ++ evidence/lab6-backend-run.txt | 1 + evidence/lab6-container-events.txt | 60 + evidence/lab6-environment.json | 13 + evidence/lab6-failover-containers.json | 40 + evidence/lab6-failover-initial.txt | 7 + evidence/lab6-failover.txt | 7 + evidence/lab6-faults-initial.json | 1402 +++++++ evidence/lab6-faults-initial.txt | 13 + evidence/lab6-faults-recovery-initial.json | 46 + evidence/lab6-faults-recovery-initial.txt | 13 + evidence/lab6-faults-run.txt | 4 + evidence/lab6-faults.json | 1221 ++++++ evidence/lab6-final-validation.txt | 9 + evidence/lab6-incident-metrics.json | 2518 ++++++++++++ evidence/lab6-incidents-summary.json | 40 + evidence/lab6-persistence.txt | 6 + evidence/lab6-provisioning-run.txt | 4 + evidence/lab6-provisioning.json | 120 + evidence/lab6-research-run.txt | 16 + evidence/lab6-research-summary.json | 70 + evidence/lab6-research.csv | 9 + evidence/lab6-research.json | 3456 +++++++++++++++++ evidence/lab6-runtime-state.json | 145 + evidence/lab6-smoke.txt | 5 + evidence/lab6-validation.txt | 12 + ...logs-3c112fa1787d40898e457423046b2e97.json | 349 ++ ...logs-dbc9b2dc91404a1fa7fae889b013bb34.json | 181 + ...race-3c112fa1787d40898e457423046b2e97.json | 2109 ++++++++++ ...race-dbc9b2dc91404a1fa7fae889b013bb34.json | 239 ++ go.mod | 63 + go.sum | 143 + infra/README.md | 24 +- infra/grafana/dashboards/messenger.json | 519 +++ .../provisioning/dashboards/dashboards.yaml | 6 + .../provisioning/datasources/datasources.yaml | 29 + infra/haproxy.cfg | 36 + infra/loki.yaml | 27 + infra/migrate.sh | 8 + infra/minio-entrypoint.sh | 5 + infra/otel/README.md | 24 +- infra/otel/collector.yaml | 41 + infra/prometheus/alerts.yaml | 19 + infra/prometheus/prometheus.yaml | 15 + infra/redis.conf | 9 + infra/s3-bootstrap.sh | 11 + infra/s3-policy.json | 21 + infra/tempo.yaml | 17 + internal/app/chat/access.go | 31 + internal/app/chat/attachments.go | 75 + internal/app/chat/create_chat.go | 33 + internal/app/chat/create_message.go | 58 + internal/app/chat/create_user.go | 32 + internal/app/chat/current_user.go | 21 + internal/app/chat/cursor.go | 75 + internal/app/chat/errors.go | 98 + internal/app/chat/get_chat.go | 32 + internal/app/chat/list_chats.go | 53 + internal/app/chat/list_messages.go | 70 + internal/app/chat/liveness.go | 13 + internal/app/chat/middleware.go | 43 + internal/app/chat/readiness.go | 25 + internal/app/chat/request.go | 74 + internal/app/chat/service.go | 76 + internal/app/chat/service_test.go | 353 ++ internal/app/chat/session.go | 45 + internal/app/chat/swagger.go | 38 + internal/app/chat/usecase.go | 21 + internal/attachment/async.go | 158 + internal/attachment/image.go | 128 + internal/attachment/image_test.go | 139 + internal/attachment/json.go | 18 + internal/attachment/json_test.go | 37 + internal/attachment/reconcile.go | 225 ++ internal/attachment/render.go | 267 ++ internal/attachment/render_test.go | 81 + internal/attachment/s3.go | 45 + internal/attachment/service.go | 138 + internal/attachment/worker.go | 658 ++++ .../attachment/worker_integration_test.go | 99 + internal/config/config.go | 119 + internal/observability/db.go | 41 + internal/observability/http.go | 26 + internal/observability/telemetry.go | 205 + internal/observability/telemetry_test.go | 99 + internal/pkg/domain/chat.go | 10 + internal/pkg/domain/errors.go | 32 + internal/pkg/domain/message.go | 13 + internal/pkg/domain/pagination.go | 23 + internal/pkg/domain/session.go | 14 + internal/pkg/domain/user.go | 10 + internal/pkg/password/password.go | 65 + internal/pkg/password/password_test.go | 35 + internal/pkg/repository/memory.go | 202 + internal/pkg/repository/postgres/chats.go | 78 + internal/pkg/repository/postgres/lab4_test.go | 346 ++ internal/pkg/repository/postgres/messages.go | 96 + internal/pkg/repository/postgres/storage.go | 51 + .../pkg/repository/postgres/storage_test.go | 292 ++ internal/pkg/repository/postgres/users.go | 46 + internal/pkg/session/redis.go | 71 + internal/usecase/chat/repository.go | 28 + internal/usecase/chat/service.go | 182 + internal/usecase/chat/session.go | 102 + internal/usecase/chat/session_test.go | 19 + internal/usecase/chat/validation.go | 37 + migrations/001_initial.sql | 54 + migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 20 + migrations/004_message_attachments.sql | 13 + migrations/005_async_attachments.sql | 18 + migrations/006_attachment_jobs.sql | 18 + migrations/007_attachment_outbox.sql | 10 + migrations/008_attachment_replays.sql | 12 + migrations/009_backfill_pipeline.sql | 12 + migrations/010_storage_cleanup.sql | 5 + migrations/011_job_trace_context.sql | 5 + scripts/bootstrap.sh | 20 + scripts/compose.sh | 7 + scripts/observability_backend.py | 79 + scripts/observability_common.py | 58 + scripts/observability_experiment.py | 189 + scripts/observability_faults.py | 149 + scripts/restart.sh | 4 + scripts/stop-one.sh | 13 + scripts/summarize_observability.py | 60 + scripts/verify_observability.py | 45 + scripts/wait_ready.py | 17 + 158 files changed, 21115 insertions(+), 52 deletions(-) create mode 100644 Dockerfile create mode 100644 Dockerfile.storage create mode 100644 INCIDENTS.md create mode 100644 RESEARCH.md create mode 100644 RUNBOOK.md create mode 100644 cmd/main.go create mode 100644 cmd/reconcile/main.go create mode 100644 cmd/worker/main.go create mode 100644 compose.yaml create mode 100644 contracts/embed.go create mode 100644 dto/create_chat.go create mode 100644 dto/create_message.go create mode 100644 dto/create_message_json.go create mode 100644 dto/create_message_json_test.go create mode 100644 dto/create_user.go create mode 100644 dto/current_user.go create mode 100644 dto/error.go create mode 100644 dto/get_chat.go create mode 100644 dto/list_chats.go create mode 100644 dto/list_messages.go create mode 100644 dto/liveness.go create mode 100644 dto/readiness.go create mode 100644 dto/session.go create mode 100644 dto/timestamp.go create mode 100644 dto/timestamp_test.go create mode 100644 evidence/lab6-backend-outage.json create mode 100644 evidence/lab6-backend-run.txt create mode 100644 evidence/lab6-container-events.txt create mode 100644 evidence/lab6-environment.json create mode 100644 evidence/lab6-failover-containers.json create mode 100644 evidence/lab6-failover-initial.txt create mode 100644 evidence/lab6-failover.txt create mode 100644 evidence/lab6-faults-initial.json create mode 100644 evidence/lab6-faults-initial.txt create mode 100644 evidence/lab6-faults-recovery-initial.json create mode 100644 evidence/lab6-faults-recovery-initial.txt create mode 100644 evidence/lab6-faults-run.txt create mode 100644 evidence/lab6-faults.json create mode 100644 evidence/lab6-final-validation.txt create mode 100644 evidence/lab6-incident-metrics.json create mode 100644 evidence/lab6-incidents-summary.json create mode 100644 evidence/lab6-persistence.txt create mode 100644 evidence/lab6-provisioning-run.txt create mode 100644 evidence/lab6-provisioning.json create mode 100644 evidence/lab6-research-run.txt create mode 100644 evidence/lab6-research-summary.json create mode 100644 evidence/lab6-research.csv create mode 100644 evidence/lab6-research.json create mode 100644 evidence/lab6-runtime-state.json create mode 100644 evidence/lab6-smoke.txt create mode 100644 evidence/lab6-validation.txt create mode 100644 evidence/logs-3c112fa1787d40898e457423046b2e97.json create mode 100644 evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json create mode 100644 evidence/trace-3c112fa1787d40898e457423046b2e97.json create mode 100644 evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/loki.yaml create mode 100755 infra/migrate.sh create mode 100755 infra/minio-entrypoint.sh create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/alerts.yaml create mode 100644 infra/prometheus/prometheus.yaml create mode 100644 infra/redis.conf create mode 100755 infra/s3-bootstrap.sh create mode 100644 infra/s3-policy.json create mode 100644 infra/tempo.yaml create mode 100644 internal/app/chat/access.go create mode 100644 internal/app/chat/attachments.go create mode 100644 internal/app/chat/create_chat.go create mode 100644 internal/app/chat/create_message.go create mode 100644 internal/app/chat/create_user.go create mode 100644 internal/app/chat/current_user.go create mode 100644 internal/app/chat/cursor.go create mode 100644 internal/app/chat/errors.go create mode 100644 internal/app/chat/get_chat.go create mode 100644 internal/app/chat/list_chats.go create mode 100644 internal/app/chat/list_messages.go create mode 100644 internal/app/chat/liveness.go create mode 100644 internal/app/chat/middleware.go create mode 100644 internal/app/chat/readiness.go create mode 100644 internal/app/chat/request.go create mode 100644 internal/app/chat/service.go create mode 100644 internal/app/chat/service_test.go create mode 100644 internal/app/chat/session.go create mode 100644 internal/app/chat/swagger.go create mode 100644 internal/app/chat/usecase.go create mode 100644 internal/attachment/async.go create mode 100644 internal/attachment/image.go create mode 100644 internal/attachment/image_test.go create mode 100644 internal/attachment/json.go create mode 100644 internal/attachment/json_test.go create mode 100644 internal/attachment/reconcile.go create mode 100644 internal/attachment/render.go create mode 100644 internal/attachment/render_test.go create mode 100644 internal/attachment/s3.go create mode 100644 internal/attachment/service.go create mode 100644 internal/attachment/worker.go create mode 100644 internal/attachment/worker_integration_test.go create mode 100644 internal/config/config.go create mode 100644 internal/observability/db.go create mode 100644 internal/observability/http.go create mode 100644 internal/observability/telemetry.go create mode 100644 internal/observability/telemetry_test.go create mode 100644 internal/pkg/domain/chat.go create mode 100644 internal/pkg/domain/errors.go create mode 100644 internal/pkg/domain/message.go create mode 100644 internal/pkg/domain/pagination.go create mode 100644 internal/pkg/domain/session.go create mode 100644 internal/pkg/domain/user.go create mode 100644 internal/pkg/password/password.go create mode 100644 internal/pkg/password/password_test.go create mode 100644 internal/pkg/repository/memory.go create mode 100644 internal/pkg/repository/postgres/chats.go create mode 100644 internal/pkg/repository/postgres/lab4_test.go create mode 100644 internal/pkg/repository/postgres/messages.go create mode 100644 internal/pkg/repository/postgres/storage.go create mode 100644 internal/pkg/repository/postgres/storage_test.go create mode 100644 internal/pkg/repository/postgres/users.go create mode 100644 internal/pkg/session/redis.go create mode 100644 internal/usecase/chat/repository.go create mode 100644 internal/usecase/chat/service.go create mode 100644 internal/usecase/chat/session.go create mode 100644 internal/usecase/chat/session_test.go create mode 100644 internal/usecase/chat/validation.go create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_message_attachments.sql create mode 100644 migrations/005_async_attachments.sql create mode 100644 migrations/006_attachment_jobs.sql create mode 100644 migrations/007_attachment_outbox.sql create mode 100644 migrations/008_attachment_replays.sql create mode 100644 migrations/009_backfill_pipeline.sql create mode 100644 migrations/010_storage_cleanup.sql create mode 100644 migrations/011_job_trace_context.sql create mode 100755 scripts/bootstrap.sh create mode 100755 scripts/compose.sh create mode 100644 scripts/observability_backend.py create mode 100644 scripts/observability_common.py create mode 100644 scripts/observability_experiment.py create mode 100644 scripts/observability_faults.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/summarize_observability.py create mode 100644 scripts/verify_observability.py create mode 100644 scripts/wait_ready.py diff --git a/.dockerignore b/.dockerignore index bc16e55..4fc15f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,8 @@ .git .github +.gitignore .env .env.* -!.env.example secrets certs volumes @@ -17,4 +17,22 @@ backups __pycache__ node_modules evidence +examples +scripts +tests +*.md +Dockerfile* +Makefile +requirements-dev.txt +lab.json .DS_Store + +.idea +.vscode +.agents +.codex + +**/.env +**/.env.* +**/.idea +**/.vscode diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..54d2206 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,37 @@ -## Лабораторная 6 +## Лабораторная 6: OpenTelemetry и Grafana -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Хакимов Дмитрий БВТ2301 +Целевая оценка: **5**. Трек 1: **цена наблюдаемости**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Из lab5 перенесены API, PostgreSQL, Redis Streams, S3 и worker с durable outbox/retry. +Добавлены SDK traces/metrics/logs, Collector, Prometheus, Loki, Tempo и Grafana. +W3C context сохраняется вместе с job; API, publish, consume, S3/DB и retry видны в одной трассе. +Dashboard, datasources и два alerts provisioned из Git. Назначение компонентов и сигналы описаны в REPORT.md. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск из чистого clone: `make up`, `make wait`, `make test`. +API: http://localhost:18086. Grafana: http://localhost:13000/d/messenger/messenger-observability. +Подробные prerequisites, bootstrap, остановка и удаление учебных данных: [REPORT.md](../REPORT.md). + +Проверки: OpenAPI/contract, Go tests с race detector, public smoke, persistence и API failover прошли. +Полный fault suite подтвердил worker backlog, retry при отказе S3, DB timeout и работу API без Tempo. +Отдельный backend outage: 35/35 записей успешны, потеряны 3 request traces, queue27/256→0, OOM нет. + +Исследование: 8 опытов,960 запросов,48 изображений ready,0 неожиданных ошибок. +Sampling50% уменьшил trace ingest на48,10%; сохранены125/240 request traces. +Методика, latency/CPU/RAM/ingest и ограничения: [RESEARCH.md](../RESEARCH.md). +Runbook: [RUNBOOK.md](../RUNBOOK.md); разбор отказов: [INCIDENTS.md](../INCIDENTS.md). +Все доказательства находятся в evidence/; результаты прежней lab5 не выдаются за результаты lab6. + +Ограничения: один host, head sampling теряет часть ошибок и медленных запросов, +короткая нагрузка не доказывает улучшение latency или предельный throughput. +Начальный OOM Tempo исправлен увеличением cgroup и ограничением Go heap; неуспешные журналы сохранены. +Основа реализации — проект lab5; источники SDK и документации указаны в REPORT.md. + +- [x] Обязательный минимум lab5 и lab6 сохранён. +- [x] Контракт и публичные tests не ослаблены. +- [x] make check, validate и make test выполнены; реальный вывод приложен. +- [x] Сценарии отказов и исследование воспроизведены. +- [x] Credentials находятся вне Git; evidence не содержит session tokens или тел сообщений. +- [x] README задания сохранён, REPORT.md описывает это решение. + +🤖 Generated with [O3 Agent Codex](https://o3-agent.o3t.ru) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..06df7a9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,67 @@ +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder +RUN GOMAXPROCS=2 GOFLAGS=-p=1 GOBIN=/out CGO_ENABLED=0 go install \ + -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' \ + github.com/pressly/goose/v3/cmd/goose@v3.27.1 + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations +COPY --from=goose-builder /out/goose /usr/local/bin/goose +COPY migrations /migrations +COPY infra/migrate.sh /usr/local/bin/migrate +USER 10001:10001 +ENTRYPOINT ["/usr/local/bin/migrate"] +CMD ["up"] + +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + +WORKDIR /src + +RUN apk add --no-cache gcc musl-dev + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY contracts ./contracts +COPY dto ./dto +COPY internal ./internal +COPY migrations ./migrations +ENV GOFLAGS=-p=1 GOMAXPROCS=1 GOMEMLIMIT=192MiB + +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath \ + -ldflags="-s -w" \ + -o /out/messenger ./cmd +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker + +FROM builder AS tests +COPY --from=goose-builder /out/goose /usr/local/bin/goose +ENV GOOSE_MIGRATION_DIR=/src/migrations +CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + +RUN apk add --no-cache libwebp-tools=1.6.0-r0 + +RUN addgroup -S messenger \ + && adduser -S -D -H -u 10001 -G messenger messenger + +COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + +COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + +COPY --from=builder --chown=messenger:messenger /out/worker /usr/local/bin/worker + +ENV PORT=8080 + +EXPOSE 8080 + +USER messenger:messenger + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/Dockerfile.storage b/Dockerfile.storage new file mode 100644 index 0000000..8b7f7d0 --- /dev/null +++ b/Dockerfile.storage @@ -0,0 +1,34 @@ +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder +ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . +RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder +ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . +RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" \ + org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" \ + org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +RUN mkdir /data && chown 10001:10001 /data +COPY --from=minio-builder /out/minio /usr/local/bin/minio +COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio +USER 10001:10001 +EXPOSE 9000 +ENTRYPOINT ["/usr/local/bin/start-minio"] + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" \ + org.opencontainers.image.revision="d6541ea280b73a834b64d4097e21f2be77676104" \ + org.opencontainers.image.version="RELEASE.2025-08-13T08-35-41Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +COPY --from=mc-builder /out/mc /usr/local/bin/mc +USER 10001:10001 +ENTRYPOINT ["mc"] diff --git a/INCIDENTS.md b/INCIDENTS.md new file mode 100644 index 0000000..a2fe56b --- /dev/null +++ b/INCIDENTS.md @@ -0,0 +1,67 @@ +# Разбор контролируемых инцидентов + +Команда: `python3 scripts/observability_faults.py`. +Успешный полный прогон: evidence/lab6-faults-run.txt, итог PASS в evidence/lab6-faults.json. +Агрегаты: evidence/lab6-incidents-summary.json. История метрик: evidence/lab6-incident-metrics.json. +Времена в JSON записаны в UTC. Все данные синтетические. + +## Очередь растёт, HTTP-приём работает + +Наблюдения:6 загрузок получили 202, возраст незавершённых заданий растёт, +QueueLagGrowing впервые наблюдался firing через 20,18s. Максимальный возраст в серии до восстановления —26,18s. +Worker capacity/busy перестают обновляться. Логи publish уже есть, consumer completion ещё нет. +Это указывает на отсутствие обработки после приёма, а не на отказ HTTP-handler или потерю принятой job. + +После запуска worker временно недоступен S3. Ошибочные S3 GET spans и outcomes storage_unavailable +отделяют этот второй этап от остановки worker. Контекст исходной загрузки сохраняется при retry. +В trace `3c112fa1787d40898e457423046b2e97` видны 4 producer и 4 consumer spans, +ошибки S3, затем успешные PUT, UPDATE/COMMIT результата. +Сохранены 80 реальных spans и 9 потоков связанных логов. +Все 6 заданий стали ready, depth вернулся к 0, QueueLagGrowing снялся. +Подтверждённые загрузки не потеряны.202 само по себе этого не доказывало: проверялось конечное ready. + +Файлы: [trace](evidence/trace-3c112fa1787d40898e457423046b2e97.json), +[logs](evidence/logs-3c112fa1787d40898e457423046b2e97.json). + +## API отвечает 503 из-за ожидания БД + +Наблюдения:28 запросов получили 503; длительность каждого около 2s, максимум 2,051s. +APIHighErrorRate перешёл в firing. В server trace находится ошибочный PostgreSQL SELECT span +примерно той же длительности. HTTP span в этой трассе длится 2,008s. +Задержка локализуется в обращении к PostgreSQL, а не в отдельной медленной обработке HTTP после БД. + +Воздействие:45-секундная ACCESS EXCLUSIVE блокировка таблицы messages. +API ограничивает ожидание запроса примерно 2s. Это объясняет 503 до окончания блокировки. +После снятия блокировки новые записи получили 201. Alert очистился на 10-й проверке с шагом 5s +после первой успешной записи; восстановление самого сервиса и очистка окна alert — разные моменты. +История состояний firing/resolved сохранена в recovery_alerts; финальный список alerts пуст. + +Файлы: [trace](evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json), +[logs](evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json). + +## Недоступно хранилище telemetry + +В основном fault suite Tempo остановлен, но 30 записей сообщений завершились успешно. +Очередь Collector выросла и после восстановления опустела. Память остаётся ограниченной cgroup. +Отдельный опыт `python3 scripts/observability_backend.py` измеряет реальные отсутствующие request traces +после более длительного отказа; результаты находятся в evidence/lab6-backend-outage.json. +Он отделяет отсутствие telemetry от потери подтверждённых бизнес-записей. + +Отдельный опыт завершился PASS: 35 успешных записей, 32 найденные request traces и 3 отсутствующие. +Максимум очереди — 27 из 256 batches; после восстановления — 0. +Максимальный наблюдавшийся RSS Collector — 81666048 bytes при cgroup201326592 bytes, OOM отсутствует. +Счётчик failed spans вырос на 312, refused spans — на 0. +312 — значение счётчика экспорта, а не число уникальных потерянных request traces. +Список 35 trace ID и результаты повторного поиска сохранены в JSON; ID сопоставлены с HTTP201 logs. + +Первый прогон выявил OOM Tempo при лимите 256MiB. Это было настоящей ошибкой стенда: +она мешала получить доказательства корреляции. Лимит увеличен до 512MiB, Go heap ограничен 320MiB. +Повторный полный fault suite успешно получил обе трассы и соответствующие логи из backend. +Неуспешные начальные журналы сохранены как evidence/lab6-faults-initial.* и *-recovery-initial.*. +Они не считаются успешным итоговым прогоном. + +## Повторение диагностики + +Пользуйтесь [RUNBOOK.md](RUNBOOK.md): сначала alert/метрики, затем структурный лог и trace. +Для вывода о причине достаточно сохранённых наблюдений. Команды внедрения отказа дают независимую проверку диагноза. +Задержки в таблицах измерены с шагом 5s и включают задержки SDK/Collector/scrape/evaluation. diff --git a/Makefile b/Makefile index 5d86d17..2efef1f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:18086 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -28,3 +28,25 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: bootstrap up down wait test-own test-observability research verify-observability test-backend +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + COMPOSE_PARALLEL_LIMIT=1 ./scripts/compose.sh up -d --build + $(MAKE) wait +wait: + $(PYTHON) scripts/wait_ready.py --base-url "$(BASE_URL)" +down: + ./scripts/compose.sh down +test-own: + go test ./... +test-observability: + $(PYTHON) scripts/observability_faults.py +research: + $(PYTHON) scripts/observability_experiment.py + $(PYTHON) scripts/summarize_observability.py +verify-observability: + $(PYTHON) scripts/verify_observability.py +test-backend: + $(PYTHON) scripts/observability_backend.py diff --git a/REPORT.md b/REPORT.md index b97ae42..99fba59 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,210 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6: наблюдаемость мессенджера -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка — **5**, трек 1: **цена наблюдаемости**. +Автор и группа заполняются студентом перед сдачей; commit/PR — после публикации. +README задания, обязательный API и публичные проверки сохранены. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine, Compose v2 либо `docker-compose`, Python 3.9+. +Для локальных Go-тестов требуется Go не старее версии в `go.mod`. +Соседняя папка lab5 после переноса не нужна. -## Архитектура и решения +```sh +make up +make wait +make check +make test +make test-own +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Полная проверка OpenAPI требует отдельной Python-зависимости: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +``` + +Bootstrap создаёт случайные credentials в `secrets/`, goose применяет migrations, +S3 bootstrap создаёт приватный bucket и ограниченного пользователя. +Повторный запуск сохраняет существующие credentials и данные. +Секреты передаются через `/run/secrets`; каталог исключён из Git и Docker context. +`scripts/compose.sh` использует `--env-file /dev/null`: локальный файл окружения не требуется. + +| Интерфейс | Адрес | +| --- | --- | +| API / Swagger | http://localhost:18086 / http://localhost:18086/swagger/ | +| Grafana dashboard | http://localhost:13000/d/messenger/messenger-observability | +| Prometheus alerts | http://localhost:19090/alerts | +| Loki HTTP | http://localhost:13100 | +| Tempo HTTP | http://localhost:13200 | + +Порты опубликованы только на loopback. OTLP, DB, Redis и S3 доступны внутри сети Compose. +Grafana разрешает локальный анонимный просмотр. Datasources/dashboard загружаются автоматически. +Для отдельного импорта используйте `infra/grafana/dashboards/messenger.json`. + +Безопасная остановка: `make down`; named volumes сохраняются. +Намеренное удаление **всех учебных данных lab6**: `./scripts/compose.sh down -v`. +Не удаляйте `secrets/` при сохранённых volumes: credentials перестанут соответствовать данным. + +## Что перенесено и добавлено + +Из lab5 перенесены `cmd/`, `dto/`, `internal/`, migrations 001–010, Dockerfile приложения/MinIO, +конфигурации S3, Redis AOF, HAProxy и мигратора. Сохранены сессии, две API-реплики, +идемпотентность сообщений, приватные вложения, durable Redis Streams, отдельный worker, +transactional outbox, lease/fence и ограниченные retry. +HTTP 202 означает устойчивый приём задания, обработка происходит в worker. + +Добавлены SDK traces/metrics/logs, Collector, три хранилища и Grafana. +Migration011 сохраняет W3C traceparent/tracestate в job вместе с durable acceptance. +Publisher создаёт producer span и передаёт контекст через Redis Streams. +Worker создаёт consumer span; retry продолжает исходную причинную связь. +PostgreSQL spans содержат операцию, но не SQL, аргументы или строку подключения. + +```mermaid +flowchart LR + Client --> HAProxy --> API[API x2] + API --> DB[(PostgreSQL)] + API --> S3[(MinIO)] + API --> Broker[(Redis Streams)] + Broker --> Worker + Worker --> DB + Worker --> S3 + API -->|OTLP| Collector + Worker -->|OTLP| Collector + Collector --> Prometheus + Collector --> Loki + Collector --> Tempo + Prometheus --> Grafana + Loki --> Grafana + Tempo --> Grafana +``` + +PostgreSQL — источник истины бизнес-состояния. S3 хранит bytes изображений. +Redis хранит сессии и очередь; AOF обеспечивает её устойчивость. +Collector принимает/обрабатывает/пересылает сигналы, но не хранит историю долговременно. +Prometheus хранит временные ряды, Loki — логи, Tempo — spans. Grafana запрашивает эти источники. + +## Компоненты и хранение + +| Service | Версия | Volume / готовность | +| --- | --- | --- | +| api x2, worker | Go из Dockerfile, `mtusi-messenger:lab6` | API `/health/live`, `/health/ready`; worker отдельно | +| gateway | HAProxy 3.2.4-alpine | Без состояния; проверяет readiness обеих реплик | +| postgres | 17.6-alpine3.22 | postgres_data; pg_isready | +| redis | 7.4.5-alpine3.21 | redis_data; redis-cli ping | +| minio | RELEASE.2025-09-07T16-13-09Z | s3_data; /minio/health/live | +| s3_bootstrap | mc RELEASE.2025-08-13T08-35-41Z | Идемпотентный bootstrap; exit0 | +| migrations | goose 3.27.1 | Журнал в PostgreSQL; exit0 | +| collector | contrib 0.137.0 | Ограниченные очереди RAM; метрики :8888 | +| prometheus | 3.6.0 | prometheus_data; /-/ready; retention7d/1GB | +| loki | 3.5.5 | loki_data; /ready; retention7d | +| tempo | 2.8.2 | tempo_data, WAL/blocks; /ready; retention7d | +| grafana | 12.2.0 | grafana_data; /api/health; provisioning из Git | + +Все компоненты находятся в отдельной сети проекта `mtusi2026lab6`. +Лимиты CPU/RAM заданы в compose.yaml. Внешние хранилища или готовый сервер не используются. +OTel Go traces/metrics 1.38.0, logs 0.14.0. Имена инструментов зафиксированы явно. +Schema URL не заявляется: это собственные сигналы контракта, а не полная реализация semantic conventions. + +## Сигналы и единицы + +| Требование | Метрика | Единица / агрегация | +| --- | --- | --- | +| HTTP throughput/errors | messenger_http_requests_total | count; labels method/route/status_class | +| HTTP latency | messenger_http_duration_seconds_bucket | seconds; histogram_quantile после sum buckets реплик | +| Jobs outcomes | messenger_jobs_total | count; job_type/outcome | +| Worker duration | messenger_job_duration_seconds_bucket | seconds на попытку | +| End-to-end ready | messenger_attachment_ready_seconds_bucket | seconds от durable создания job, включая queue/retry | +| Queue | messenger_queue_depth / messenger_queue_oldest_age_seconds | jobs/seconds; max повторных наблюдений общей очереди | +| DB pool | messenger_db_pool_used / messenger_db_pool_waiting / messenger_db_pool_max | connections / acquisition calls / connections | +| Worker saturation | messenger_worker_busy / messenger_worker_capacity | active jobs / concurrency; отношение busy/capacity | +| Go runtime | messenger_process_memory_bytes / messenger_process_cpu_seconds_total | Sys bytes / cumulative CPU seconds | +| Контейнеры | docker stats в research JSON | cgroup memory / CPU percent | + +Histogram buckets: .005,.01,.025,.05,.1,.25,.5,1,2,5,10,30,60,120 секунд. +p50/p95/p99 вычисляются из объединённых buckets, а не из средних или percentile каждой реплики. +Resource attributes: service.name=messenger-api/messenger-worker, service.version=lab6, +service.instance.id=hostname, deployment.environment.name=lab. +Логи содержат timestamp/severity/service.name/event/trace_id/span_id. +X-Request-Id связывает HTTP-ошибку с логом; X-Trace-Id позволяет открыть трассу. +UUID, request ID, raw URL и filename отсутствуют в metric labels. +Authorization, токены, тела сообщений и изображений не экспортируются. + +## SLI/SLO и alerts + +Учебный SLI API: `1 - sum(rate(messenger_http_requests_total{status_class="5xx"}[5m])) / sum(rate(messenger_http_requests_total[5m]))`. +SLO: **не менее 99% бизнес-запросов без 5xx за скользящие 5 минут**, при наличии трафика. +Health/Swagger исключены. Неверный пароль не считается серверным отказом;4xx остаются в знаменателе. +SLI attachments: доля ready за 10 секунд по end-to-end histogram, дополнительно к возрасту незавершённой очереди. + +APIHighErrorRate: доля 5xx >5% за 1m, for: 5s. +QueueLagGrowing: oldest age>10s и depth>0, for: 5s. +Пороги сокращены для учебного опыта; трек 2 с burn-rate alerts не заявляется. +Правила: infra/prometheus/alerts.yaml; диагностика: [RUNBOOK.md](RUNBOOK.md). +Внешняя доставка не требуется; проверяется локальное firing/resolved. + +## Backpressure telemetry + +SDK traces/logs: bounded queue512, batch128, export timeout2s; metrics экспортируется каждые 5s. +Collector: memory limiter128MiB/spike32MiB, batch256/max512, +exporter queue256, два consumers для каждого backend, retry максимум 20s, cgroup192MiB. +Длительный отказ приводит к потерям telemetry. Бизнес-запросы не ожидают записи telemetry в backend. +Очереди в памяти Collector теряются при аварийном завершении. + +## Проверки, отказы и исследование + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +./scripts/compose.sh start api +python3 scripts/observability_faults.py +python3 scripts/observability_backend.py +python3 scripts/observability_experiment.py +python3 scripts/summarize_observability.py +python3 scripts/verify_observability.py +``` + +Сценарии запускайте последовательно; после failover восстановите две реплики. +Fault suite останавливает worker, создаёт backlog, показывает firing/resolved, +проверяет retry при недоступном S3, блокирует PostgreSQL и проверяет API при выключенном Tempo. +Исторические результаты lab5 не используются как доказательства lab6. +Команды и фактические результаты: evidence/lab6-validation.txt, evidence/lab6-smoke.txt, +evidence/lab6-faults.json, evidence/lab6-research.json. Исследование: [RESEARCH.md](RESEARCH.md). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий README | Проверка / результат | Доказательство | +| --- | --- | --- | +| Минимум lab5, две API-реплики и отдельный worker | Публичный smoke PASS; PostgreSQL/Redis/MinIO и очередь работают | evidence/lab6-smoke.txt; compose.yaml | +| Сохранность после restart | Данные, оригинал вложения и прежняя сессия сохранены | evidence/lab6-persistence.txt; evidence/lab6-container-events.txt | +| Отказ API-реплики | 117 успешных записей, 4 неудачные попытки, восстановление за 3,382s; одна реплика остановлена и затем восстановлена | evidence/lab6-failover.txt; evidence/lab6-failover-containers.json | +| Collector, три хранилища, Grafana и persistent history | Пять scrape targets UP; dashboard provisioned, 15 панелей, 18 запросов выполняются | evidence/lab6-provisioning.json; infra/ | +| OTLP API/worker: counters, histograms, logs и traces | Обязательные сигналы найдены по реальным именам; сохранены server/producer/consumer spans и связанные логи | evidence/lab6-provisioning.json; evidence/trace-*.json; evidence/logs-*.json | +| Низкая кардинальность и безопасная telemetry | Проверены labels реальных series; unit tests не допускают SQL/аргументы/тело/Authorization в spans | evidence/lab6-provisioning.json; evidence/lab6-validation.txt; internal/observability/telemetry_test.go | +| W3C API → broker → worker → S3/DB, включая retry | В одной трассе upload видны 4 producer/consumer попытки и завершение | evidence/lab6-incidents-summary.json; INCIDENTS.md | +| p50/p95/p99, queue lag, pool, saturation и CPU/RAM | Histogram queries объединяют buckets реплик; gauges и process/container ресурсы измерены | infra/grafana/dashboards/messenger.json; evidence/lab6-provisioning.json; evidence/lab6-research.json | +| SLI/SLO и два alerts | Определены выше; оба alerts firing, затем resolved | infra/prometheus/alerts.yaml; evidence/lab6-faults.json | +| Два инцидента и диагностика | Worker/S3 и DB lock локализованы по метрикам, логам и трассам | INCIDENTS.md; RUNBOOK.md; evidence/lab6-incident-metrics.json | +| Ограниченное поведение при отказе backend | 35/35 записей успешны; queue максимум 27/256, затем 0; OOM нет; 3 request traces отсутствуют | evidence/lab6-backend-outage.json | +| Законченный трек 1 на 5 | Off/100%/10%/50%, два повтора, 960 запросов, 48 изображений ready; latency, CPU/RAM, ingest и потери; 50% проверено | RESEARCH.md; evidence/lab6-research.json; evidence/lab6-research.csv | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Полный fault suite завершился PASS. В начале проверки Tempo получил OOM при лимите 256MiB. +Лимит исправлен на 512MiB с GOMEMLIMIT=320MiB; после исправления трассы и логи получены, отказы повторены успешно. +Подробности и начальные неуспешные журналы сохранены в INCIDENTS.md и evidence/. -## Эксперименты и отказы +## Ограничения и источники -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Один host и единственные DB/broker/S3 не обеспечивают HA всего стенда. +Research измеряет выбранный небольшой профиль, а не предельную пропускную способность. +Head sampling может потерять ошибку или редкий медленный запрос: исход неизвестен при принятии решения. +При снижении sampling traces метрики и логи продолжают экспортироваться. +SDK off оставляет wrapper-код и stdout: измеряется дополнительная стоимость OTLP, не всей инструментации. +Waiting учитывает текущие acquisition calls, включая немедленную выдачу соединения. +Go Sys — зарезервированная runtime память, не RSS. Cgroup memory измеряется отдельно. +Ready histogram содержит завершённые jobs; backlog контролируется отдельной метрикой. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Источники: документация OTel Go/Collector, Prometheus histograms, Grafana provisioning, +Loki OTLP ingestion и Tempo configuration; первичные ссылки — в README. +Основа — соседний проект lab5. Реализация и проверки выполнены с помощью O3 Agent Codex. diff --git a/RESEARCH.md b/RESEARCH.md new file mode 100644 index 0000000..8cc5244 --- /dev/null +++ b/RESEARCH.md @@ -0,0 +1,154 @@ +# Цена наблюдаемости: методика трека 1 + +## Гипотеза + +Снижение head sampling уменьшит объём traces и нагрузку telemetry backend. +Метрики и логи останутся полными. Цена — потеря части трасс ошибок и медленных запросов. +Сравниваются SDK off, sampling 100%,10%,50%. Кандидат сбалансированной настройки —50%. +Приоритет учебного стенда: сохранить достаточно трасс для диагностики при меньшем ingest. + +## Повторение + +```sh +make up +make test +python3 scripts/observability_experiment.py --requests 120 --rate 8 --concurrency 4 --repetitions 2 +python3 scripts/summarize_observability.py +``` + +Запускайте последовательно с fault suite. Сценарий пересоздаёт API и worker, +выставляет OTEL_SDK_DISABLED/OTEL_TRACES_SAMPLER_ARG, затем восстанавливает SDK on/sampling 100% в finally. +Collector и backend не пересоздаются между вариантами; их counters сохраняются. +Первый повтор: off→100%→10%→50%; второй — в обратном порядке. + +На каждый вариант создаётся один синтетический пользователь, один чат и одинаковый набор данных. +Из 120 измеряемых запросов 90 читают пользователя,18 создают сообщения,6 загружают PNG,6 ожидают 401. +Размер fixture и текст сообщений одинаковы. Каждая из 6 загрузок должна стать ready. +401 — ожидаемая клиентская ошибка; её не следует путать с 5xx и нарушением SLO API. +Ошибки сверх ожидаемых статусов считаются отдельно; повтор запроса автоматически не выполняется. + +После пересоздания выдерживаются 8 секунд, затем 20 GET для прогрева и ещё6 секунд. +Планировщик задаёт 8 запросов/с и максимум 4 одновременно исполняемых вызова. +При перегрузке клиент может отстать от расписания: это ограниченный генератор, не доказательство capacity. +Latency измеряется от начала фактического вызова; задержка планировщика в percentile не входит. +p50/p95/p99 включают все 120 вызовов, в том числе ожидаемые 401. + +Во время нагрузки docker stats фиксирует CPU/RAM контейнеров примерно каждые 3 секунды. +CPU — сумма средних по контейнерам;100% соответствует одному полностью занятому CPU. +RAM — сумма наблюдавшихся максимумов, которые могли приходиться на разные моменты времени. +Отдельно считаются API+worker и Collector/Prometheus/Loki/Tempo/Grafana. +Измеряется не вся стоимость business stack: PostgreSQL/Redis/S3 не включены в группу application. + +Через 12 секунд после нагрузки снимаются счётчики Collector и backend. +Ingest traces/logs — прирост *_bytes_received_total, metrics — прирост TSDB appended samples. +Это входные bytes/samples, не размер compressed storage и не сетевой трафик всего стенда. +В прирост входят фоновые DB spans, health logs и собственные метрики инфраструктуры. +SDK off отключает экспорт приложения, но backend и их self-monitoring продолжают работать. +Отрицательные дельты counters означали бы restart и делали бы сравнение непригодным. + +ID traces берутся из X-Trace-Id фактических ответов. Каждый ID запрашивается из Tempo. +Входящий traceparent не задаётся: иначе parent-based sampler мог бы унаследовать решение клиента. +Сохранность trace означает наличие хотя бы одного span по ID, а не доказательство полноты каждого span. +Для полного sampling проверяется наличие всех 120 request traces. +Отдельно считаются retained traces ожидаемых 401 и самых медленных 5% вызовов каждого опыта. +Потери из-за решения sampling отделяются от ошибок exporter/refused spans. +Число отсутствующих трасс — измерено; оценка потерь уникальных spans по одним счётчикам невозможна. + +## Среда + +Параметры host/VM и исходный commit записаны в evidence/lab6-environment.json. +Docker VM:2 CPU,2053644288 bytes RAM,arm64. Лимиты компонентов — compose.yaml. +Старые стенды lab4/lab5 удалены до измерений по просьбе пользователя; их volumes сохранены. +Отдельный PostgreSQL вне проекта остаётся работающим и создаёт небольшой фоновый шум. +Графики/таблицы не сравниваются с измерениями lab5: это другой профиль нагрузки. + +## Результаты + +Полный прогон завершился PASS:8 опытов,960 запросов,48 изображений ready,0 неожиданных ошибок. +Исходные данные: [JSON](evidence/lab6-research.json), [CSV](evidence/lab6-research.csv), +[агрегаты](evidence/lab6-research-summary.json), [журнал](evidence/lab6-research-run.txt). +Таблица воспроизводится командой `python3 scripts/summarize_observability.py`. + +| Config | Repeat | RPS | p50 ms | p95 ms | p99 ms | App CPU % | App MiB | Telemetry CPU % | Telemetry MiB | Request traces | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| off | 1 | 8.047 | 11.934 | 29.423 | 42.462 | 8.132 | 70.39 | 4.816 | 616.03 | 0/120 | +| full | 1 | 8.053 | 11.856 | 31.41 | 42.052 | 9.962 | 73.733 | 10.37 | 608.78 | 120/120 | +| ten_percent | 1 | 8.057 | 9.991 | 52.789 | 168.924 | 8.846 | 71.971 | 10.784 | 502.69 | 17/120 | +| balanced | 1 | 8.062 | 10.423 | 30.254 | 52.116 | 8.462 | 71.314 | 10.322 | 651.25 | 72/120 | +| balanced | 2 | 8.048 | 11.346 | 37.555 | 72.706 | 8.014 | 59.703 | 7.096 | 600.74 | 53/120 | +| ten_percent | 2 | 8.061 | 12.281 | 42.42 | 63.216 | 7.942 | 60.066 | 13.176 | 671.03 | 17/120 | +| full | 2 | 8.048 | 12.379 | 31.356 | 45.834 | 9.114 | 74.053 | 7.598 | 676.44 | 120/120 | +| off | 2 | 8.05 | 10.728 | 30.012 | 63.853 | 7.404 | 64.176 | 6.616 | 563.21 | 0/120 | + +Ниже суммы двух повторов каждой настройки; workload каждого варианта —240 измеряемых запросов. +Объём ingest включает фоновые операции за окно наблюдения. + +| Настройка | Tempo bytes | Loki bytes | TSDB samples | Exported spans | Traces запросов | Traces401 | Traces медленных 5% | +| --- | --- | --- | --- | --- | --- | --- | --- | +| off | 0 | 0 | 46185 | 0 | 0/240 | 0/12 | 0/12 | +| full | 697602 | 198451 | 44282 | 4947 | 240/240 | 12/12 | 12/12 | +| ten_percent | 95539 | 196690 | 49391 | 461 | 34/240 | 2/12 | 3/12 | +| balanced | 362056 | 184978 | 44897 | 2402 | 125/240 | 7/12 | 8/12 | + +При 50% trace ingest сократился на 48,10% относительно 100%; exported spans — примерно на 51,45%. +Найдены 125/240 request traces,7/12 traces ожидаемых 401,8/12 traces самых медленных вызовов. +При 10% ingest сократился на 86,30%, но потеряны 10/12 traces401 и 9/12 traces медленных вызовов. +Во втором повторе 10% не сохранилась ни одна из 6 трасс upload. +Это конкретный пример потери редкого пути, несмотря на успешную обработку всех изображений. +Фактические 34/240=14,17% request traces при настройке 10% не обязаны точно совпадать с параметром: +решение случайно, выборка небольшая, а byte/span-weighted доля зависит от типа запроса. + +Во всех 8 опытах прирост exporter failed spans и receiver refused spans равен 0. +При 100% найдены все 240 request traces, включая все 12 ожидаемых 401 и 12 самых медленных вызовов. +Эти наблюдения не доказывают полноту каждого дочернего span. +Accepted и sent counters снимаются асинхронно и могут пересекать разные границы batch/scrape; +их разность не используется как число потерянных spans. +Преднамеренная потеря при sampling показана через запросы по известным trace ID. +Отдельно проверяется потеря при недоступном backend в evidence/lab6-backend-outage.json. + +Средний по двум опытам CPU API+worker:off7,768%,100%9,538%,50%8,238%. +Разница 100% и off составляет 1,77 процентного пункта одного CPU. +Из-за короткого профиля и шума это наблюдение, а не точная оценка причинного overhead. +RAM API+worker в варианте 100% достигала 74,053MiB против 70,390MiB в первом off. +Backend RAM/CPU меняются с GC, compaction и кешами; sampling не даёт доказанного пропорционального снижения RAM. +p99 при 10% колебался 63–169ms, при 100%42–46ms. Ускорение latency снижением sampling не подтверждено. + +Logs ingest сохраняется примерно на одном уровне между on-вариантами:185–198kB за два окна. +Metrics ingest в таблице включает self-monitoring backend, который работает и при SDK off. +Фаза 5-секундного scrape и короткие окна дают существенную погрешность счётчика samples; +по этим значениям нельзя приписать SDK отрицательную или точную дополнительную стоимость метрик. +Метрики приложения при sampling traces продолжают экспортироваться; все обязательные сигналы проверены отдельно. + +## Проверенная сбалансированная настройка + +Для обычного учебного трафика предлагается parentbased_traceidratio=0.5 с полными metrics/logs. +Оба опыта 50% прошли без неожиданных ошибок, все 12 загрузок этого варианта стали ready, +а trace ingest примерно вдвое ниже 100%. Критерий выбора — снижение ingest при сохранении заметной части путей, +а не доказанное ускорение HTTP. Цена: потеря 115/240 request traces и части медленных/ошибочных запросов. +Эта настройка не подходит как гарантия сохранности всех ошибок. + +```sh +OTEL_TRACES_SAMPLER_ARG=0.5 ./scripts/compose.sh up -d --no-deps --force-recreate api worker +``` + +Для защиты и разбора редкого retry восстановите 100%: + +```sh +OTEL_TRACES_SAMPLER_ARG=1.0 ./scripts/compose.sh up -d --no-deps --force-recreate api worker +``` + +Сценарий исследования сам восстановил SDK on/sampling100% после завершения. + +## Ограничения интерпретации + +Два коротких повтора не дают узкий доверительный интервал для CPU и p99. +При 8 RPS результат может определяться сетевой задержкой Docker и клиентом, а не SDK. +Поэтому снижение p99 на одном варианте само по себе не доказывает ускорение приложения. +Изменение ingest и сохранности traces — более прямое следствие sampling. +Пять процентов самых медленных запросов определены относительно этого опыта, не production SLO. + +Head sampling принимает решение до результата операции.50% не гарантирует сохранение 50% ошибок, +а тем более всех редких 5xx. При 10% небольшая выборка ошибок может исчезнуть полностью. +Сохранённые logs/metrics позволяют увидеть факт ошибки, но не восстановить потерянные spans. +Чтобы сохранять почти все ошибки и редкие медленные запросы, нужен проверенный tail sampling +либо отдельная политика захвата. Такой эксперимент в этой работе не заявляется. diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..54cee39 --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,58 @@ +# Диагностика учебного стенда + +Откройте [dashboard](http://localhost:13000/d/messenger/messenger-observability), затем [состояния alerts](http://localhost:19090/alerts). +Историю хранит Prometheus. Логи хранит Loki. Трассы хранит Tempo. +Collector принимает OTLP и пересылает сигналы. Grafana выполняет запросы к хранилищам. + +## HTTP-запросы завершаются с 5xx + +1. Проверьте `APIHighErrorRate`. Числитель — бизнес-запросы с 5xx; знаменатель — все бизнес-запросы. + Health и Swagger исключены из инструментов. Ошибки клиента 4xx входят только в знаменатель. +2. Найдите route и status class на dashboard. Сравните p95/p99 с обычным значением. + Запрос p95: `histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))`. + Значение измеряется в секундах. Buckets объединяют обе API-реплики. +3. В Explore → Loki выполните `{service_name="messenger-api"} | json | status >= 500`. + Выберите запись `http.request`. Сохраните `request_id` и `trace_id`; тело сообщения не требуется. +4. Откройте Tempo по `trace_id`. У HTTP span найдите дочерние `postgresql SELECT/INSERT`. + Длительный ошибочный DB span указывает на ожидание БД. Долгий HTTP span без такого дочернего span требует другого расследования. +5. Проверьте `messenger_db_pool_used`, `messenger_db_pool_max` и метрику ожидания пула. + Высокая занятость — дополнительный признак, сама по себе она не доказывает блокировку PostgreSQL. +6. В лабораторном опыте снимите искусственную блокировку или дождитесь завершения `pg_sleep`. + Убедитесь, что новые записи получают 201, DB spans сокращаются, alert исчезает после очистки окна 1m. + +## Вложения долго остаются queued/processing + +1. Проверьте `QueueLagGrowing`, `max(messenger_queue_depth)` и `max(messenger_queue_oldest_age_seconds)`. + Для общего backlog используйте max: каждая API-реплика наблюдает одну общую таблицу jobs. +2. Сравните worker busy/capacity и свежесть worker-сигналов. Остановленный worker перестаёт экспортировать данные. + Отсутствие свежих данных отличается от рабочего процесса с нулевой занятостью. +3. В Loki найдите `job.published`, затем `job.completed` в той же трассе. + В Tempo проверьте `attachments publish` → `attachments process` → S3 и PostgreSQL. +4. Если consumer отсутствует, запустите `./scripts/compose.sh start worker`. + Если consumer есть, но S3 span длительный или ошибочный, проверьте доступность MinIO. + Повторные producer/consumer spans с той же трассой показывают retry. +5. Убедитесь, что вложения стали ready, очередь опустела, возраст равен нулю, alert больше не firing. + Полученный HTTP 202 подтверждает приём задания, но не окончание обработки. + +## Не появляются трассы, API работает + +1. Проверьте `up{job="collector"}`, `up{job="tempo"}` и `otelcol_exporter_queue_size`. +2. Сравните `otelcol_exporter_sent_spans_total`, `otelcol_exporter_send_failed_spans_total` и `otelcol_receiver_refused_spans_total`. + Failed экспорт учитывает неуспешные попытки; он не равен числу окончательно потерянных уникальных spans. +3. Восстановите Tempo: `./scripts/compose.sh start tempo`. + Убедитесь, что очередь exporter уменьшается и свежий trace находится по ID. +4. После длительного отказа возможны потери: SDK и Collector имеют ограниченные очереди и timeout. + Повторите запрос для диагностики. Не объявляйте отсутствие старой трассы потерей бизнес-данных. + +## Повторение отказов + +Запускайте только на отдельном учебном проекте lab6, последовательно с исследованием. +Сценарий создаёт синтетические данные. Он не обращается к стендам lab4/lab5. + +```sh +python3 scripts/observability_faults.py +``` + +Сценарий восстанавливает worker, MinIO и Tempo в `finally`. +Блокировка PostgreSQL автоматически заканчивается через 45 секунд; соединение закрывается при ошибке. +Наблюдения сохраняются в `evidence/lab6-faults.json`, связанные spans и логи — в соседних JSON. diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..81e2b46 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "fmt" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +const shutdownTimeout = 8 * time.Second + +func main() { + shutdownTelemetry, telemetryErr := observability.Init(context.Background(), "messenger-api") + if telemetryErr != nil { + log.Fatal("telemetry configuration failed") + } + defer shutdownTelemetry() + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + poolConfig, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid PostgreSQL config") + } + poolConfig.ConnConfig.Host = cfg.PostgresHost + poolConfig.ConnConfig.Port = cfg.PostgresPort + poolConfig.ConnConfig.User = cfg.PostgresUser + poolConfig.ConnConfig.Database = cfg.PostgresDatabase + poolConfig.ConnConfig.Password = cfg.PostgresPassword + poolConfig.ConnConfig.ConnectTimeout = cfg.QueryTimeout + poolConfig.ConnConfig.RuntimeParams["timezone"] = "UTC" + poolConfig.ConnConfig.RuntimeParams["statement_timeout"] = fmt.Sprint(cfg.QueryTimeout.Milliseconds()) + poolConfig.ConnConfig.Tracer = observability.DBTracer{} + poolConfig.MaxConns = cfg.PostgresMaxConns + poolConfig.MinConns = 1 + poolConfig.MaxConnLifetime = time.Hour + poolConfig.MaxConnIdleTime = 5 * time.Minute + pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig) + if err != nil { + log.Fatal("cannot create PostgreSQL pool") + } + defer pool.Close() + observability.ObservePool(pool, 0) + redisClient := redis.NewClient(&redis.Options{ + Addr: cfg.RedisAddr, PoolSize: 10, MaxActiveConns: 10, PoolTimeout: cfg.QueryTimeout, + DialTimeout: cfg.QueryTimeout, ReadTimeout: cfg.QueryTimeout, WriteTimeout: cfg.QueryTimeout, + MaxRetries: -1, ContextTimeoutEnabled: true, + }) + defer redisClient.Close() + repo := postgres.New(pool, cfg.QueryTimeout) + sessions := session.NewRedis(redisClient, cfg.QueryTimeout) + useCase := chatusecase.New(repo, sessions, cfg.SessionTTL) + objectStore, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + attachments := attachment.NewAsync(pool, objectStore, options) + background, stopPublisher := context.WithCancel(context.Background()) + defer stopPublisher() + go attachments.RunPublisher(background, redisClient) + handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachments) + port := cfg.Port + e := handler.Router() + e.Server.ReadHeaderTimeout = 5 * time.Second + e.Server.ReadTimeout = 50 * time.Second + e.Server.WriteTimeout = 55 * time.Second + e.Server.IdleTimeout = 60 * time.Second + + serverErrors := make(chan error, 1) + go func() { + log.Printf("HTTP server is listening on :%s", port) + serverErrors <- e.Start(":" + port) + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + select { + case sig := <-stop: + log.Printf("received %s, shutting down", sig) + case err := <-serverErrors: + if !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("HTTP server failed: %v", err) + } + return + } + + handler.Drain() + // Allow two 1-second LB checks to remove this backend before closing the listener. + time.Sleep(3 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := e.Shutdown(ctx); err != nil { + log.Printf("graceful shutdown failed: %v", err) + if closeErr := e.Close(); closeErr != nil { + log.Printf("forced shutdown failed: %v", closeErr) + } + return + } + log.Printf("shutdown complete") +} diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 0000000..2534aa6 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "log" + "os" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + apply := flag.Bool("apply", false, "apply safe repairs; default dry-run") + grace := flag.Duration("grace", 2*time.Minute, "minimum object/metadata age (at least 90s)") + timeout := flag.Duration("timeout", 10*time.Minute, "whole scan deadline") + flag.Parse() + if *grace < attachment.MinimumGrace { + log.Fatal("grace must be at least 90s") + } + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.MaxConns = 2 + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("cannot create DB pool") + } + defer pool.Close() + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + result, err := attachment.New(pool, store).Reconcile(ctx, *apply, *grace) + _ = json.NewEncoder(os.Stdout).Encode(result) + if err != nil { + log.Print(err) + os.Exit(1) + } +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..2c8deaf --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "render" { + if len(os.Args) != 5 { + os.Exit(2) + } + if err := attachment.Render(context.Background(), os.Args[2], os.Args[3], os.Args[4]); err != nil { + if errors.Is(err, attachment.ErrInvalidImage) { + os.Exit(2) + } + os.Exit(1) + } + return + } + failed := flag.Bool("failed", false, "list failed job registry") + replay := flag.String("replay", "", "failed job UUID to replay") + operator := flag.String("operator", "", "operator identity") + reason := flag.String("reason", "", "replay reason") + flag.Parse() + shutdownTelemetry, telemetryErr := observability.Init(context.Background(), "messenger-worker") + if telemetryErr != nil { + log.Fatal("telemetry configuration failed") + } + defer shutdownTelemetry() + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + options, err := attachment.LoadOptions() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["timezone"] = "UTC" + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.ConnConfig.Tracer = observability.DBTracer{} + pc.MaxConns = int32(options.Concurrency + 4) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("DB configuration failed") + } + defer pool.Close() + observability.ObservePool(pool, options.Concurrency) + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("S3 configuration failed") + } + service := attachment.NewAsync(pool, store, options) + if *failed { + rows, e := pool.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts,coalesce(last_error,''),updated_at FROM attachment_jobs WHERE state='failed' ORDER BY updated_at DESC LIMIT 100`) + if e != nil { + log.Fatal("cannot query failed jobs") + } + defer rows.Close() + records := []map[string]interface{}{} + for rows.Next() { + var id, att, code string + var attempt, max int + var updated time.Time + if e = rows.Scan(&id, &att, &attempt, &max, &code, &updated); e != nil { + log.Fatal("invalid failed job") + } + records = append(records, map[string]interface{}{"job_id": id, "attachment_id": att, "attempts": attempt, "max_attempts": max, "error_code": code, "updated_at": updated.UTC()}) + } + if rows.Err() != nil { + log.Fatal("failed jobs query interrupted") + } + json.NewEncoder(os.Stdout).Encode(records) + return + } + if *replay != "" { + operation, stop := context.WithTimeout(ctx, 10*time.Second) + defer stop() + if err = service.Replay(operation, *replay, *operator, *reason); err != nil { + log.Fatal(err) + } + log.Print("Replay committed with audit record") + return + } + r := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr, DialTimeout: 2 * time.Second, ReadTimeout: 2 * time.Second, WriteTimeout: 2 * time.Second, MaxRetries: -1, ContextTimeoutEnabled: true}) + defer r.Close() + log.Printf("Worker started: concurrency=%d prefetch=%d timeout=%s lease=%s pixel_budget=%d", options.Concurrency, options.Prefetch, options.Timeout, options.Lease, options.PixelBudget) + for ctx.Err() == nil { + err = service.RunWorker(ctx, r) + if err != nil { + log.Print("worker broker connection will retry") + } + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5359873 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,250 @@ +name: mtusi2026lab6 +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + redis: + image: redis:7.4.5-alpine3.21 + command: [redis-server, /usr/local/etc/redis/redis.conf] + volumes: + - redis_data:/data + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 384m + minio: + environment: + GOMEMLIMIT: 160MiB + GOGC: '50' + image: mtusi-minio:RELEASE.2025-09-07T16-13-09Z + build: + context: . + dockerfile: Dockerfile.storage + target: minio + secrets: [s3_root_access_key, s3_root_secret_key] + volumes: [s3_data:/data] + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live'] + interval: 2s + timeout: 3s + start_period: 15s + retries: 30 + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 20s + s3_bootstrap: + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + build: + context: . + dockerfile: Dockerfile.storage + target: mc + entrypoint: [/bin/sh, /bootstrap.sh] + configs: + - source: s3_bootstrap + target: /bootstrap.sh + - source: s3_policy + target: /policy.json + secrets: [s3_root_access_key, s3_root_secret_key, s3_access_key, s3_secret_key] + depends_on: + minio: + condition: service_healthy + cpus: 0.5 + mem_limit: 512m + migrations: + image: mtusi-migrations:lab6 + build: + context: . + target: migrations + environment: + GOOSE_DRIVER: postgres + GOOSE_DBSTRING: host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5 + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + api: + build: . + image: mtusi-messenger:lab6 + environment: &api_environment + POSTGRES_HOST: postgres + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_MAX_CONNS: '10' + REDIS_ADDR: redis:6379 + CURSOR_KEY_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + QUERY_TIMEOUT_MS: '2000' + S3_ENDPOINT: minio:9000 + S3_BUCKET: attachments + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + S3_USE_SSL: 'false' + GOMEMLIMIT: 384MiB + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: parentbased_traceidratio + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + GOGC: '50' + secrets: &api_secrets + - postgres_password + - cursor_key + - s3_access_key + - s3_secret_key + deploy: + replicas: 2 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrations: + condition: service_completed_successfully + s3_bootstrap: + condition: service_completed_successfully + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 10s + worker: + image: mtusi-messenger:lab6 + build: . + entrypoint: [/usr/local/bin/worker] + environment: + <<: *api_environment + WORKER_CONCURRENCY: '2' + WORKER_PREFETCH: '4' + WORKER_MAX_ATTEMPTS: '5' + WORKER_TIMEOUT_SECONDS: '25' + WORKER_LEASE_SECONDS: '40' + WORKER_PIXEL_BUDGET: '40000000' + secrets: *api_secrets + healthcheck: {disable: true} + depends_on: + migrations: {condition: service_completed_successfully} + s3_bootstrap: {condition: service_completed_successfully} + redis: {condition: service_healthy} + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 10s + gateway: + image: haproxy:3.2.4-alpine + ports: ['127.0.0.1:${PORT:-18086}:8080'] + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + depends_on: [api] + mem_limit: 64m + collector: + image: otel/opentelemetry-collector-contrib:0.137.0 + command: [--config=/etc/otelcol/config.yaml] + volumes: [./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro] + mem_limit: 192m + depends_on: [tempo, loki, prometheus] + prometheus: + image: prom/prometheus:v3.6.0 + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d, --storage.tsdb.retention.size=1GB, --web.enable-remote-write-receiver] + ports: ['127.0.0.1:19090:9090'] + volumes: + - ./infra/prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + mem_limit: 256m + loki: + image: grafana/loki:3.5.5 + command: [-config.file=/etc/loki/config.yaml] + ports: ['127.0.0.1:13100:3100'] + volumes: + - ./infra/loki.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + mem_limit: 192m + tempo: + image: grafana/tempo:2.8.2 + command: [-config.file=/etc/tempo/config.yaml] + ports: ['127.0.0.1:13200:3200'] + volumes: + - ./infra/tempo.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + environment: + GOMEMLIMIT: 320MiB + GOGC: '50' + mem_limit: 512m + grafana: + image: grafana/grafana:12.2.0 + ports: ['127.0.0.1:13000:3000'] + environment: + GF_PLUGINS_PREINSTALL_DISABLED: 'true' + GF_ANALYTICS_CHECK_FOR_UPDATES: 'false' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_AUTH_ANONYMOUS_ENABLED: 'true' + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + mem_limit: 192m + depends_on: [prometheus, loki, tempo] + tests: + image: mtusi-messenger-tests:lab6 + profiles: [test] + build: + context: . + target: tests + environment: + <<: *api_environment + INTEGRATION_TESTS: '1' + secrets: *api_secrets + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + cpus: 1.0 + mem_limit: 1g +configs: + s3_bootstrap: + file: ./infra/s3-bootstrap.sh + s3_policy: + file: ./infra/s3-policy.json +secrets: + s3_root_access_key: + file: ./secrets/s3_root_access_key + s3_root_secret_key: + file: ./secrets/s3_root_secret_key + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + postgres_password: + file: ./secrets/postgres_password + cursor_key: + file: ./secrets/cursor_key +volumes: + postgres_data: + redis_data: + s3_data: + + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/contracts/embed.go b/contracts/embed.go new file mode 100644 index 0000000..ef4b85a --- /dev/null +++ b/contracts/embed.go @@ -0,0 +1,9 @@ +package contracts + +import _ "embed" + +// OpenAPI is embedded so Swagger UI works from a compiled binary and does not +// depend on the process working directory. +// +//go:embed openapi.json +var OpenAPI []byte diff --git a/dto/create_chat.go b/dto/create_chat.go new file mode 100644 index 0000000..25c42a0 --- /dev/null +++ b/dto/create_chat.go @@ -0,0 +1,13 @@ +package dto + +type CreateChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type CreateChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/create_message.go b/dto/create_message.go new file mode 100644 index 0000000..f89c57c --- /dev/null +++ b/dto/create_message.go @@ -0,0 +1,16 @@ +package dto + +type CreateMessageRequest struct { + ChatID string `json:"-"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type CreateMessageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/create_message_json.go b/dto/create_message_json.go new file mode 100644 index 0000000..b92e99d --- /dev/null +++ b/dto/create_message_json.go @@ -0,0 +1,33 @@ +package dto + +import ( + "bytes" + "encoding/json" + "errors" +) + +// Keep absent attachment_ids equivalent to [], while rejecting JSON null and +// wrong types rather than quietly treating them as an empty message body. +func (r *CreateMessageRequest) UnmarshalJSON(data []byte) error { + type payload CreateMessageRequest + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil || fields == nil { + return errors.New("Ожидается JSON-объект") + } + text, exists := fields["text"] + if !exists || bytes.Equal(bytes.TrimSpace(text), []byte("null")) { + return errors.New("Требуется строка text") + } + if ids, exists := fields["attachment_ids"]; exists && bytes.Equal(bytes.TrimSpace(ids), []byte("null")) { + return errors.New("attachment_ids должен быть массивом") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var decoded payload + if err := decoder.Decode(&decoded); err != nil { + return err + } + decoded.ChatID = r.ChatID + *r = CreateMessageRequest(decoded) + return nil +} diff --git a/dto/create_message_json_test.go b/dto/create_message_json_test.go new file mode 100644 index 0000000..0d97cd0 --- /dev/null +++ b/dto/create_message_json_test.go @@ -0,0 +1,21 @@ +package dto + +import ( + "encoding/json" + "testing" +) + +func TestMessageStrictFields(t *testing.T) { + for _, body := range []string{`null`, `[]`, `{}`, `{"text":null}`, `{"text":"x","attachment_ids":null}`, `{"text":"x","attachment_ids":"x"}`, `{"text":"x","extra":1}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err == nil { + t.Fatalf("accepted %s", body) + } + } + for _, body := range []string{`{"text":"x"}`, `{"text":"x","attachment_ids":[]}`, `{"text":"","attachment_ids":["id"]}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err != nil { + t.Fatalf("rejected %s: %v", body, err) + } + } +} diff --git a/dto/create_user.go b/dto/create_user.go new file mode 100644 index 0000000..af41015 --- /dev/null +++ b/dto/create_user.go @@ -0,0 +1,14 @@ +package dto + +type CreateUserRequest struct { + Password string `json:"password"` + Username string `json:"username"` + DisplayName string `json:"display_name"` +} + +type CreateUserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/current_user.go b/dto/current_user.go new file mode 100644 index 0000000..b82a032 --- /dev/null +++ b/dto/current_user.go @@ -0,0 +1,10 @@ +package dto + +type CurrentUserRequest struct{} + +type CurrentUserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/error.go b/dto/error.go new file mode 100644 index 0000000..c8c17c4 --- /dev/null +++ b/dto/error.go @@ -0,0 +1,11 @@ +package dto + +type ErrorResponse struct { + Error ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} diff --git a/dto/get_chat.go b/dto/get_chat.go new file mode 100644 index 0000000..6ed9e1e --- /dev/null +++ b/dto/get_chat.go @@ -0,0 +1,12 @@ +package dto + +type GetChatRequest struct { + ChatID string `json:"-"` +} + +type GetChatResponse struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/list_chats.go b/dto/list_chats.go new file mode 100644 index 0000000..43f9eb1 --- /dev/null +++ b/dto/list_chats.go @@ -0,0 +1,18 @@ +package dto + +type ListChatsRequest struct { + Limit int `json:"-"` + Cursor string `json:"-"` +} + +type ListChatsResponse struct { + Items []ListChatsItem `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ListChatsItem struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/list_messages.go b/dto/list_messages.go new file mode 100644 index 0000000..35a3842 --- /dev/null +++ b/dto/list_messages.go @@ -0,0 +1,21 @@ +package dto + +type ListMessagesRequest struct { + ChatID string `json:"-"` + Limit int `json:"-"` + Cursor string `json:"-"` +} + +type ListMessagesResponse struct { + Items []ListMessagesItem `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ListMessagesItem struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` +} diff --git a/dto/liveness.go b/dto/liveness.go new file mode 100644 index 0000000..830955e --- /dev/null +++ b/dto/liveness.go @@ -0,0 +1,7 @@ +package dto + +type LivenessRequest struct{} + +type LivenessResponse struct { + Status string `json:"status"` +} diff --git a/dto/readiness.go b/dto/readiness.go new file mode 100644 index 0000000..21573b7 --- /dev/null +++ b/dto/readiness.go @@ -0,0 +1,7 @@ +package dto + +type ReadinessRequest struct{} + +type ReadinessResponse struct { + Status string `json:"status"` +} diff --git a/dto/session.go b/dto/session.go new file mode 100644 index 0000000..bd2c880 --- /dev/null +++ b/dto/session.go @@ -0,0 +1,8 @@ +package dto + +type CreateSessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt Timestamp `json:"expires_at"` + User CreateUserResponse `json:"user"` +} diff --git a/dto/timestamp.go b/dto/timestamp.go new file mode 100644 index 0000000..9dab3e2 --- /dev/null +++ b/dto/timestamp.go @@ -0,0 +1,11 @@ +package dto + +import "time" + +// Timestamp preserves PostgreSQL microseconds and is readable by Python 3.9 clients. +// time.Time's variable-length fractional part is not supported by those clients. +type Timestamp time.Time + +func (t Timestamp) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Time(t).UTC().Format("2006-01-02T15:04:05.000000Z") + `"`), nil +} diff --git a/dto/timestamp_test.go b/dto/timestamp_test.go new file mode 100644 index 0000000..1b48033 --- /dev/null +++ b/dto/timestamp_test.go @@ -0,0 +1,15 @@ +package dto + +import ( + "encoding/json" + "testing" + "time" +) + +func TestTimestampFixedMicroseconds(t *testing.T) { + timestamp := Timestamp(time.Date(2026, 1, 2, 3, 4, 5, 123400000, time.UTC)) + encoded, err := json.Marshal(timestamp) + if err != nil || string(encoded) != `"2026-01-02T03:04:05.123400Z"` { + t.Fatalf("timestamp=%s err=%v", encoded, err) + } +} diff --git a/evidence/README.md b/evidence/README.md index 8df7001..97a1447 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,24 @@ -# Доказательства +# Доказательства lab6 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Все результаты получены на стенде этого репозитория. Данные синтетические. +Времена JSON указаны в UTC; аппаратная конфигурация и исходный commit — lab6-environment.json. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +| Файл | Проверка | +| --- | --- | +| lab6-validation.txt | Offline contract, OpenAPI и Go race tests | +| lab6-final-validation.txt | Повторная проверка контрактов после завершения изменений | +| lab6-smoke.txt | Публичный API, включая асинхронные вложения | +| lab6-persistence.txt, lab6-container-events.txt | Сохранность и scope restart хранилищ | +| lab6-failover.txt, lab6-failover-containers.json | Две реплики, отказ одной, данные/сессия и восстановление | +| lab6-provisioning.json | Пять targets UP, 15 панелей, 18 query, metric label policy | +| lab6-faults.json, lab6-faults-run.txt | Успешный полный fault suite | +| lab6-incident-metrics.json, lab6-incidents-summary.json | Сохранённые наблюдения worker/S3/DB | +| trace-*.json, logs-*.json | Реальные связанные spans и структурные логи из Tempo/Loki | +| lab6-backend-outage.json, lab6-backend-run.txt | Работа API при отказе Tempo, queue/RAM и потери traces | +| lab6-research.json, lab6-research.csv, lab6-research-summary.json | Восемь опытов off/on/sampling, raw summaries, CPU/RAM | +| lab6-research-run.txt | Журнал завершённого исследования | +| lab6-runtime-state.json | Состояние контейнеров после проверок | + +Файлы *-initial.* содержат начальные или прежние прогоны, включая обнаруженные ошибки. +Они не заменяют успешные итоговые журналы. Подробнее: ../INCIDENTS.md и ../RESEARCH.md. +Секреты, токены, тела сообщений, bytes изображений и signed URLs сюда не сохраняются. diff --git a/evidence/lab6-backend-outage.json b/evidence/lab6-backend-outage.json new file mode 100644 index 0000000..531f916 --- /dev/null +++ b/evidence/lab6-backend-outage.json @@ -0,0 +1,220 @@ +{ + "started_at": "2026-11-11T07:08:27.295682+00:00", + "before": { + "failed_spans": 10491.0, + "refused_spans": 0.0, + "queued_batches": 0.0, + "rss_bytes": 81518592.0 + }, + "during": [ + { + "seconds": 0.01, + "failed_spans": 10491.0, + "refused_spans": 0.0, + "queued_batches": 0.0, + "rss_bytes": 81518592.0 + }, + { + "seconds": 5.2, + "failed_spans": 10491.0, + "refused_spans": 0.0, + "queued_batches": 4.0, + "rss_bytes": 81592320.0 + }, + { + "seconds": 10.52, + "failed_spans": 10491.0, + "refused_spans": 0.0, + "queued_batches": 9.0, + "rss_bytes": 81612800.0 + }, + { + "seconds": 15.79, + "failed_spans": 10491.0, + "refused_spans": 0.0, + "queued_batches": 14.0, + "rss_bytes": 81612800.0 + }, + { + "seconds": 20.94, + "failed_spans": 10559.0, + "refused_spans": 0.0, + "queued_batches": 18.0, + "rss_bytes": 81616896.0 + }, + { + "seconds": 26.07, + "failed_spans": 10659.0, + "refused_spans": 0.0, + "queued_batches": 22.0, + "rss_bytes": 81666048.0 + }, + { + "seconds": 31.28, + "failed_spans": 10659.0, + "refused_spans": 0.0, + "queued_batches": 27.0, + "rss_bytes": 81666048.0 + } + ], + "successful_writes": 35, + "after": { + "failed_spans": 10803.0, + "refused_spans": 0.0, + "queued_batches": 0.0, + "rss_bytes": 81674240.0 + }, + "retained_request_traces": 32, + "missing_request_traces": 3, + "exporter_failed_spans_delta": 312.0, + "receiver_refused_spans_delta": 0.0, + "collector": { + "oom": false, + "memory_limit": 201326592 + }, + "status": "PASS", + "finished_at": "2026-11-11T07:09:19.947185+00:00", + "request_trace_results": [ + { + "trace_id": "004adff807639444e55202464ea93fe3", + "found": false + }, + { + "trace_id": "08df5dc321e96dc41068b1be56f04687", + "found": true + }, + { + "trace_id": "1aeaee387a6b2cbb4ad320c3e34d7b3d", + "found": true + }, + { + "trace_id": "1b25416590b8cd69b8063e2fe54d5da6", + "found": true + }, + { + "trace_id": "2ea7a14a909ba468bfbe045fcc9ef556", + "found": true + }, + { + "trace_id": "332601b1543a4a16827ef181d88efb04", + "found": true + }, + { + "trace_id": "3a0f8d48655b7190e4afab69bd9a7357", + "found": true + }, + { + "trace_id": "3ae8ac314578b01a61d4f4889d274467", + "found": true + }, + { + "trace_id": "41b19f9f0aecc4f4805f90e1c4901384", + "found": true + }, + { + "trace_id": "4a2480d683cfbff700d3276502e3f6d0", + "found": true + }, + { + "trace_id": "4b9b780f232470232ff61e6e8ea0031e", + "found": true + }, + { + "trace_id": "51462d539eb2514fa66727227a3e30c0", + "found": true + }, + { + "trace_id": "57b44fd53e4383bfd896fce7c4cb6fa5", + "found": true + }, + { + "trace_id": "7281f6cbd7790df5948f7b03fef14d7a", + "found": true + }, + { + "trace_id": "7cd8d3c5bec5684f4cadb7569cc40d04", + "found": true + }, + { + "trace_id": "7e2ac3261a8a6745667b08c9e6e20314", + "found": true + }, + { + "trace_id": "8074d021c476f9ce1eb64c2141e24ffa", + "found": true + }, + { + "trace_id": "865556f2adf8a006995f6895c4a82513", + "found": true + }, + { + "trace_id": "973fe7d2582609896fb1166313eb219c", + "found": true + }, + { + "trace_id": "9d28d77ca28cf24aa3e6fe1607c408a2", + "found": true + }, + { + "trace_id": "9fc10f8741f5bf971b80c7d58d7bf936", + "found": true + }, + { + "trace_id": "ac5f5b02318a8c2f4a1f4ba129339f56", + "found": true + }, + { + "trace_id": "ac8bc1e2d842b25238518ee0a872ae4d", + "found": true + }, + { + "trace_id": "ad3e00cc2507be3a6af4cce1ff732311", + "found": true + }, + { + "trace_id": "b26abeb379b6568c25f6d0e570978bc6", + "found": false + }, + { + "trace_id": "b92c8b1f65d944e03623ca05cd013dd5", + "found": true + }, + { + "trace_id": "bc795f70921ad071cebb4ce9e053eebe", + "found": true + }, + { + "trace_id": "c60cfe77fa24afb2477a776bcf5abd71", + "found": false + }, + { + "trace_id": "d3359fdbc04ddaa1d3a6275fd4908361", + "found": true + }, + { + "trace_id": "dc13a3387339133eea087b6b1371cc46", + "found": true + }, + { + "trace_id": "df4b8b322e294fe8b8caf1fe3e14bd5b", + "found": true + }, + { + "trace_id": "e5c7704f60b9e8c8d33b91037737eeaa", + "found": true + }, + { + "trace_id": "f0fb0084dd1ed1a177eb297ebf213ef5", + "found": true + }, + { + "trace_id": "f11bb8dc1d60dd494bf0ed79d834338a", + "found": true + }, + { + "trace_id": "f953829e6b5b3bfc853fb866a331a1b4", + "found": true + } + ], + "verification_source": "201 message http.request logs in exact incident time window; query traces again by ID" +} diff --git a/evidence/lab6-backend-run.txt b/evidence/lab6-backend-run.txt new file mode 100644 index 0000000..a0e1a40 --- /dev/null +++ b/evidence/lab6-backend-run.txt @@ -0,0 +1 @@ +{"status": "PASS", "successful_writes": 35, "retained_request_traces": 32, "missing_request_traces": 3, "exporter_failed_spans_delta": 312.0} diff --git a/evidence/lab6-container-events.txt b/evidence/lab6-container-events.txt new file mode 100644 index 0000000..f6c38a6 --- /dev/null +++ b/evidence/lab6-container-events.txt @@ -0,0 +1,60 @@ +1794339137 kill mtusi2026lab6-minio-1 +1794339137 kill mtusi2026lab6-loki-1 +1794339137 kill mtusi2026lab6-redis-1 +1794339137 kill mtusi2026lab6-api-1 +1794339137 kill mtusi2026lab6-prometheus-1 +1794339137 kill mtusi2026lab6-collector-1 +1794339137 kill mtusi2026lab6-grafana-1 +1794339137 kill mtusi2026lab6-api-2 +1794339137 kill mtusi2026lab6-gateway-1 +1794339137 kill mtusi2026lab6-worker-1 +1794339137 kill mtusi2026lab6-postgres-1 +1794339137 kill mtusi2026lab6-tempo-1 +1794339137 stop mtusi2026lab6-minio-1 +1794339137 die mtusi2026lab6-minio-1 +1794339137 stop mtusi2026lab6-collector-1 +1794339137 die mtusi2026lab6-collector-1 +1794339137 stop mtusi2026lab6-redis-1 +1794339137 die mtusi2026lab6-redis-1 +1794339138 stop mtusi2026lab6-grafana-1 +1794339138 die mtusi2026lab6-grafana-1 +1794339138 stop mtusi2026lab6-prometheus-1 +1794339138 die mtusi2026lab6-prometheus-1 +1794339138 stop mtusi2026lab6-worker-1 +1794339138 die mtusi2026lab6-postgres-1 +1794339138 stop mtusi2026lab6-postgres-1 +1794339138 die mtusi2026lab6-worker-1 +1794339138 stop mtusi2026lab6-tempo-1 +1794339138 die mtusi2026lab6-tempo-1 +1794339138 stop mtusi2026lab6-gateway-1 +1794339138 die mtusi2026lab6-gateway-1 +1794339138 start mtusi2026lab6-minio-1 +1794339138 restart mtusi2026lab6-minio-1 +1794339138 start mtusi2026lab6-collector-1 +1794339138 restart mtusi2026lab6-collector-1 +1794339138 stop mtusi2026lab6-loki-1 +1794339138 die mtusi2026lab6-loki-1 +1794339138 start mtusi2026lab6-redis-1 +1794339138 restart mtusi2026lab6-redis-1 +1794339138 start mtusi2026lab6-grafana-1 +1794339138 restart mtusi2026lab6-grafana-1 +1794339138 start mtusi2026lab6-postgres-1 +1794339138 restart mtusi2026lab6-postgres-1 +1794339138 start mtusi2026lab6-worker-1 +1794339138 restart mtusi2026lab6-worker-1 +1794339138 start mtusi2026lab6-prometheus-1 +1794339138 restart mtusi2026lab6-prometheus-1 +1794339138 start mtusi2026lab6-gateway-1 +1794339138 restart mtusi2026lab6-gateway-1 +1794339139 start mtusi2026lab6-tempo-1 +1794339139 restart mtusi2026lab6-tempo-1 +1794339139 start mtusi2026lab6-loki-1 +1794339139 restart mtusi2026lab6-loki-1 +1794339140 stop mtusi2026lab6-api-1 +1794339140 die mtusi2026lab6-api-1 +1794339140 stop mtusi2026lab6-api-2 +1794339140 die mtusi2026lab6-api-2 +1794339140 start mtusi2026lab6-api-1 +1794339140 restart mtusi2026lab6-api-1 +1794339140 start mtusi2026lab6-api-2 +1794339140 restart mtusi2026lab6-api-2 diff --git a/evidence/lab6-environment.json b/evidence/lab6-environment.json new file mode 100644 index 0000000..b51dcd3 --- /dev/null +++ b/evidence/lab6-environment.json @@ -0,0 +1,13 @@ +{ + "at": "2026-11-10T19:28:02.037325+00:00", + "host": "macOS-26.6.2-arm64-arm-64bit", + "architecture": "arm64", + "docker": { + "cpus": 2, + "memory_bytes": 2053644288, + "server_version": "29.5.2" + }, + "base_commit": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "note": "Old mtusi-lab4 and mtusi-lab5 Swarm stacks removed at user request before runtime experiments; persistent volumes retained. Standalone postgres remains running.", + "old_stacks_removed_at": "2026-11-10T19:29:34.767211+00:00" +} diff --git a/evidence/lab6-failover-containers.json b/evidence/lab6-failover-containers.json new file mode 100644 index 0000000..0b51da7 --- /dev/null +++ b/evidence/lab6-failover-containers.json @@ -0,0 +1,40 @@ +{ + "at": "2026-11-11T07:11:03.337621+00:00", + "before": [ + { + "name": "/mtusi2026lab6-api-1", + "alias": "00403456e233", + "status": "running" + }, + { + "name": "/mtusi2026lab6-api-2", + "alias": "60b77a464203", + "status": "running" + } + ], + "after": [ + { + "name": "/mtusi2026lab6-api-1", + "alias": "00403456e233", + "status": "running" + }, + { + "name": "/mtusi2026lab6-api-2", + "alias": "60b77a464203", + "status": "exited" + } + ], + "exit_code": 0, + "restored": [ + { + "name": "/mtusi2026lab6-api-1", + "alias": "00403456e233", + "status": "running" + }, + { + "name": "/mtusi2026lab6-api-2", + "alias": "60b77a464203", + "status": "running" + } + ] +} diff --git a/evidence/lab6-failover-initial.txt b/evidence/lab6-failover-initial.txt new file mode 100644 index 0000000..4e248ef --- /dev/null +++ b/evidence/lab6-failover-initial.txt @@ -0,0 +1,7 @@ +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 5, "first_recovery_seconds": 4.062} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit=0 diff --git a/evidence/lab6-failover.txt b/evidence/lab6-failover.txt new file mode 100644 index 0000000..1531f59 --- /dev/null +++ b/evidence/lab6-failover.txt @@ -0,0 +1,7 @@ +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 4, "first_recovery_seconds": 3.382} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit=0 diff --git a/evidence/lab6-faults-initial.json b/evidence/lab6-faults-initial.json new file mode 100644 index 0000000..362e782 --- /dev/null +++ b/evidence/lab6-faults-initial.json @@ -0,0 +1,1402 @@ +{ + "started_at": "2026-11-10T19:33:11.285373+00:00", + "cases": { + "s3_retry": { + "alerts_during": [], + "jobs": [ + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339282.111, + "6" + ] + } + ] + }, + "worker_stop": { + "queued": 6, + "ready_after_recovery": 6, + "timeline": [ + { + "seconds": 5.09, + "alerts": [], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339196.824, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339196.824, + "0" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339196.831, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339196.831, + "0" + ] + } + ] + }, + { + "seconds": 10.12, + "alerts": [], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339201.848, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339201.848, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339201.848, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339201.855, + "4.240175" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339201.855, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339201.855, + "4.149592" + ] + } + ] + }, + { + "seconds": 15.15, + "alerts": [], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339206.87, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339206.87, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339206.87, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339206.875, + "9.239472" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339206.875, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339206.875, + "9.152506" + ] + } + ] + }, + { + "seconds": 20.17, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "pending", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "1.4229988e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339211.904, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339211.904, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339211.911, + "14.229988" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339211.911, + "14.150514" + ] + } + ] + }, + { + "seconds": 25.21, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "1.923564e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339216.931, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339216.931, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339216.938, + "19.23564" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339216.938, + "19.150344" + ] + } + ] + }, + { + "seconds": 30.23, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "2.4230902e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339221.972, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339221.972, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339221.977, + "24.230902" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339221.977, + "24.150541" + ] + } + ] + }, + { + "seconds": 35.28, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "2.9232348e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339227.008, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339227.008, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339227.012, + "34.232153" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339227.012, + "34.151987" + ] + } + ] + }, + { + "seconds": 40.3, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "3.4232153e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339232.021, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339232.021, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339232.024, + "39.229904" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339232.024, + "39.15179" + ] + } + ] + }, + { + "seconds": 45.32, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "3.9229904e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339237.041, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339237.041, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339237.045, + "44.240613" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339237.045, + "44.165991" + ] + } + ] + }, + { + "seconds": 50.35, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-10T19:33:29.758725496Z", + "value": "4.4240613e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339242.076, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339242.076, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339242.079, + "49.235252" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339242.079, + "49.154033" + ] + } + ] + } + ], + "resolved": [], + "queue_after": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339317.582, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794339317.582, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794339317.582, + "0" + ] + } + ] + }, + "db_lock": { + "requests": [ + { + "status": 503, + "elapsed": 2.0597262500000113, + "trace_id": "38156c884be14bef83b501c78f4599fe" + }, + { + "status": 503, + "elapsed": 2.058679791000003, + "trace_id": "81c7eed2f7ed4ae392808770e5be3ed3" + }, + { + "status": 503, + "elapsed": 2.040349208000009, + "trace_id": "2ef861d086b34419a3f565249ae789c4" + }, + { + "status": 503, + "elapsed": 2.0408058749999896, + "trace_id": "239e0c47f9804038b070ac7bfd843ac2" + }, + { + "status": 503, + "elapsed": 2.038897708999997, + "trace_id": "efc2b9574e3c456a804c528be24f9cc8" + }, + { + "status": 503, + "elapsed": 2.0395332920000158, + "trace_id": "cfadad6fcd954a7f90e67688f7b851aa" + }, + { + "status": 503, + "elapsed": 2.049825917000021, + "trace_id": "50cb0c64ecbb4ab580f8693fb50d869e" + }, + { + "status": 503, + "elapsed": 2.050290082999993, + "trace_id": "253f2565444144c69ea0182460ebee0e" + }, + { + "status": 503, + "elapsed": 2.043883208000011, + "trace_id": "0c1f8799a9d54fcb888086d0798f3e48" + }, + { + "status": 503, + "elapsed": 2.0435042080000017, + "trace_id": "965d066a566a42a4b827d39d25ad663d" + }, + { + "status": 503, + "elapsed": 2.0488313330000096, + "trace_id": "3e277675e98e4704b2d949a978b08bf5" + }, + { + "status": 503, + "elapsed": 2.0497192500000097, + "trace_id": "bd0218f8e6e54569aec5b7a9f24c9dd3" + }, + { + "status": 503, + "elapsed": 2.0297508749999906, + "trace_id": "d4d0b28ae31a415986d8d1ef37f53efc" + }, + { + "status": 503, + "elapsed": 2.0293384999999944, + "trace_id": "cdd3d2f064c041a1bfc1f90c7135208f" + }, + { + "status": 503, + "elapsed": 2.04155908300001, + "trace_id": "6cf441edc1e243f182bfe75c17de30b4" + }, + { + "status": 503, + "elapsed": 2.0410010000000227, + "trace_id": "5080795432314f58864789db28ea9f4e" + }, + { + "status": 503, + "elapsed": 2.0474574169999755, + "trace_id": "bc3a5f5d96974117b68fc1731dc734c0" + }, + { + "status": 503, + "elapsed": 2.0482846670000185, + "trace_id": "b9b553a65b494d159e251f88cb810792" + }, + { + "status": 503, + "elapsed": 2.043250999999998, + "trace_id": "916404b087054478af3f2dc1a165f710" + }, + { + "status": 503, + "elapsed": 2.042649291999993, + "trace_id": "a3fbeb5369ee40539f27965a7feb5949" + }, + { + "status": 503, + "elapsed": 2.0560065410000163, + "trace_id": "e106854cb52a467db1c4f46d753a028b" + }, + { + "status": 503, + "elapsed": 2.056358875000001, + "trace_id": "b45d780b2f4a43bf859840d691797a9b" + }, + { + "status": 503, + "elapsed": 2.0582759159999853, + "trace_id": "ba732569525740e580ff31c0cee437ea" + }, + { + "status": 503, + "elapsed": 2.0591560409999943, + "trace_id": "f7fb10f14b1b446fb0118a17b7dbeff6" + }, + { + "status": 503, + "elapsed": 2.030463792000006, + "trace_id": "988cb0d77df34808abdcc5671e379aee" + }, + { + "status": 503, + "elapsed": 2.029722291999974, + "trace_id": "3288b3df3c694fe78203bb3e63d719f3" + }, + { + "status": 503, + "elapsed": 2.033429417000008, + "trace_id": "81b565e8c72648709e32f6f8ff7cb6d5" + }, + { + "status": 503, + "elapsed": 2.0331184590000078, + "trace_id": "90b5f1d772f0478e8e9ba76a4502f630" + } + ], + "alerts_during": [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-10T19:35:34.758725496Z", + "value": "1e+00" + } + ], + "alerts_after": [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-10T19:35:34.758725496Z", + "value": "7.999999999999999e-01" + } + ] + }, + "tempo_stop": { + "successful_writes": 30, + "exporter_failed": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794339424.622, + "0" + ] + } + ], + "queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794339424.633, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794339424.633, + "13" + ] + } + ], + "collector_memory": [ + { + "metric": { + "__name__": "otelcol_process_memory_rss_bytes", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/service" + }, + "value": [ + 1794339424.642, + "150073344" + ] + } + ], + "after": { + "queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794339445.203, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794339445.203, + "0" + ] + } + ], + "collector_memory": [ + { + "metric": { + "__name__": "otelcol_process_memory_rss_bytes", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/service" + }, + "value": [ + 1794339445.212, + "147681280" + ] + } + ] + } + } + }, + "finished_at": "2026-11-10T19:38:08.127691+00:00" +} diff --git a/evidence/lab6-faults-initial.txt b/evidence/lab6-faults-initial.txt new file mode 100644 index 0000000..1bb09fa --- /dev/null +++ b/evidence/lab6-faults-initial.txt @@ -0,0 +1,13 @@ +Worker outage +DB lock +Telemetry backend outage +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 140, in + main() + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 124, in main + report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])] + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 124, in + report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])] + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 38, in collect_trace + raise RuntimeError('trace/log correlation not found in backend') +RuntimeError: trace/log correlation not found in backend diff --git a/evidence/lab6-faults-recovery-initial.json b/evidence/lab6-faults-recovery-initial.json new file mode 100644 index 0000000..8ba50a7 --- /dev/null +++ b/evidence/lab6-faults-recovery-initial.json @@ -0,0 +1,46 @@ +{ + "started_at": "2026-11-11T06:52:37.598456+00:00", + "cases": { + "s3_retry": { + "alerts_during": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-11T06:52:54.758725496Z", + "value": "7.8294858e+01" + } + ], + "jobs": [ + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "storage_unavailable", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380041.365, + "24" + ] + } + ] + } + }, + "finished_at": "2026-11-11T06:54:02.143528+00:00" +} diff --git a/evidence/lab6-faults-recovery-initial.txt b/evidence/lab6-faults-recovery-initial.txt new file mode 100644 index 0000000..b067ced --- /dev/null +++ b/evidence/lab6-faults-recovery-initial.txt @@ -0,0 +1,13 @@ +Worker outage +Traceback (most recent call last): + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 148, in + main() + File "/Users/dkhakimovig/mtusi2026lab6/scripts/observability_faults.py", line 78, in main + c.wait_ready(actor, att) + File "/Users/dkhakimovig/mtusi2026lab6/tests/client.py", line 196, in wait_ready + att = self.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json'] + File "/Users/dkhakimovig/mtusi2026lab6/tests/client.py", line 132, in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) + File "/Users/dkhakimovig/mtusi2026lab6/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: GET /api/v1/attachments/e377e191-b942-47fa-b691-eefb232588d7: expected [200], received 503 diff --git a/evidence/lab6-faults-run.txt b/evidence/lab6-faults-run.txt new file mode 100644 index 0000000..087486c --- /dev/null +++ b/evidence/lab6-faults-run.txt @@ -0,0 +1,4 @@ +Worker outage +DB lock +Telemetry backend outage +{"status": "PASS", "cases": ["s3_retry", "worker_stop", "db_lock", "tempo_stop"]} diff --git a/evidence/lab6-faults.json b/evidence/lab6-faults.json new file mode 100644 index 0000000..a49dd3e --- /dev/null +++ b/evidence/lab6-faults.json @@ -0,0 +1,1221 @@ +{ + "started_at": "2026-11-11T06:55:34.902572+00:00", + "cases": { + "s3_retry": { + "alerts_during": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-11T06:55:49.758725496Z", + "value": "3.6181197e+01" + } + ], + "jobs": [ + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "storage_unavailable", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380176.223, + "12" + ] + } + ] + }, + "worker_stop": { + "queued": 6, + "ready_after_recovery": 6, + "timeline": [ + { + "seconds": 5.1, + "alerts": [], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380140.468, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380140.468, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380140.468, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380140.477, + "1.178956" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380140.477, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380140.477, + "1.101367" + ] + } + ] + }, + { + "seconds": 10.13, + "alerts": [], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380145.489, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380145.489, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380145.489, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380145.492, + "6.185773" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380145.492, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380145.492, + "6.099005" + ] + } + ] + }, + { + "seconds": 15.15, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "pending", + "activeAt": "2026-11-11T06:55:49.758725496Z", + "value": "1.1185192e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380150.516, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380150.516, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380150.516, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380150.526, + "11.185192" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380150.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380150.526, + "11.100526" + ] + } + ] + }, + { + "seconds": 20.18, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-11T06:55:49.758725496Z", + "value": "1.617813e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380155.544, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380155.544, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380155.557, + "16.17813" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380155.557, + "16.099826" + ] + } + ] + }, + { + "seconds": 25.21, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-11T06:55:49.758725496Z", + "value": "2.1184781e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380160.578, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380160.578, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380160.583, + "21.184781" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380160.583, + "21.102623" + ] + } + ] + }, + { + "seconds": 30.24, + "alerts": [ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "description": "Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "Oldest unfinished attachment exceeds 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-11T06:55:49.758725496Z", + "value": "2.6179077e+01" + } + ], + "queue": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380165.617, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380165.617, + "6" + ] + } + ], + "age": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380165.625, + "26.179077" + ] + }, + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380165.625, + "26.123765" + ] + } + ] + } + ], + "resolved": [], + "queue_after": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380217.941, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794380217.941, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794380217.941, + "0" + ] + } + ] + }, + "db_lock": { + "requests": [ + { + "status": 503, + "elapsed": 2.0210961659999924, + "trace_id": "dbc9b2dc91404a1fa7fae889b013bb34" + }, + { + "status": 503, + "elapsed": 2.0206497080000076, + "trace_id": "31d530346dc546ef8ff0eb6539ebeff6" + }, + { + "status": 503, + "elapsed": 2.015805291999996, + "trace_id": "d4d7e70855784dfea4904be9474befea" + }, + { + "status": 503, + "elapsed": 2.0153950830000014, + "trace_id": "8022705af1cd4c97bb24b6cd8e5bd70b" + }, + { + "status": 503, + "elapsed": 2.040692749999991, + "trace_id": "7a1c9f26fece4cd5a09034505a73303a" + }, + { + "status": 503, + "elapsed": 2.0454220419999984, + "trace_id": "fdd58b7ba3d3467db66db4fce528e390" + }, + { + "status": 503, + "elapsed": 2.0508444580000003, + "trace_id": "dc4844bee468427abb3e2f1a95bab0e8" + }, + { + "status": 503, + "elapsed": 2.049359416999991, + "trace_id": "0ae6f8f2f42749e5ab894f5de7f228fb" + }, + { + "status": 503, + "elapsed": 2.030444750000001, + "trace_id": "10d3a0e710bb47a4bee706221e15a7c0" + }, + { + "status": 503, + "elapsed": 2.0307514579999975, + "trace_id": "5d56b7f368544b7e9c1642bb69c65979" + }, + { + "status": 503, + "elapsed": 2.025713749999994, + "trace_id": "9bd7c9cfe0fd4cdfbfd588e44bce53c2" + }, + { + "status": 503, + "elapsed": 2.0260718749999995, + "trace_id": "351c9d518b4f4ca9aa3da92c82ebaf5e" + }, + { + "status": 503, + "elapsed": 2.0334448750000007, + "trace_id": "6905defc81fc40b6904a7b3faeab9c2e" + }, + { + "status": 503, + "elapsed": 2.0337752920000014, + "trace_id": "5749a9133ed04719b51ef83aa68c89d3" + }, + { + "status": 503, + "elapsed": 2.026468582999996, + "trace_id": "76de6842f9ea40cbbb5ebab3a829b3e7" + }, + { + "status": 503, + "elapsed": 2.0277077909999974, + "trace_id": "0edcc1a1d36444229b1c17cb0c2f3b79" + }, + { + "status": 503, + "elapsed": 2.0174080419999996, + "trace_id": "4f490ee13a624a1d9625f47568866874" + }, + { + "status": 503, + "elapsed": 2.0177502079999954, + "trace_id": "06ce7036ebcd46c388b4ff79151699c0" + }, + { + "status": 503, + "elapsed": 2.0130290420000136, + "trace_id": "c5fc8a458a0a430daff9365dd7acc886" + }, + { + "status": 503, + "elapsed": 2.0130619169999875, + "trace_id": "0b868c3da1b54d5a9fb14a54ef4125b0" + }, + { + "status": 503, + "elapsed": 2.0151341669999994, + "trace_id": "34b18cb657294771ab7f9facef174299" + }, + { + "status": 503, + "elapsed": 2.0152103749999952, + "trace_id": "170e1f5d658b48e595ae0e649557a21a" + }, + { + "status": 503, + "elapsed": 2.016058916999995, + "trace_id": "b2cf1e5ad18e4eda86ad3ddc4baacef8" + }, + { + "status": 503, + "elapsed": 2.016475416000006, + "trace_id": "a1019e7602ae41e68101596d8346e65e" + }, + { + "status": 503, + "elapsed": 2.033585500000001, + "trace_id": "0c24d50f4e8c4397b96347d9e7b90e32" + }, + { + "status": 503, + "elapsed": 2.0337022500000046, + "trace_id": "a461f6f11b9e45018ce55b45fc23a7ec" + }, + { + "status": 503, + "elapsed": 2.0368789169999957, + "trace_id": "8193736c05ce44639f802981c55a7b40" + }, + { + "status": 503, + "elapsed": 2.0360208749999913, + "trace_id": "721b519950c44e55ad5605809822efb0" + } + ], + "alerts_during": [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "1e+00" + } + ], + "recovery_alerts": [ + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "1e+00" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "9.655172413793104e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "9.333333333333332e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "9.032258064516129e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "8.461538461538461e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "7.82608695652174e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "6.666666666666666e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "5.333333333333333e-01" + } + ], + [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "description": "Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md.", + "runbook_url": "http://localhost:13000/d/messenger/messenger-observability", + "summary": "More than 5 percent API server failures over 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-11T06:57:09.758725496Z", + "value": "2.727272727272727e-01" + } + ], + [] + ], + "alerts_after": [] + }, + "tempo_stop": { + "successful_writes": 30, + "exporter_failed": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794380340.586, + "10491" + ] + } + ], + "queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794380340.593, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794380340.593, + "21" + ] + } + ], + "collector_memory": [ + { + "metric": { + "__name__": "otelcol_process_memory_rss_bytes", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/service" + }, + "value": [ + 1794380340.597, + "82190336" + ] + } + ], + "after": { + "queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794380361.113, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794380361.113, + "0" + ] + } + ], + "collector_memory": [ + { + "metric": { + "__name__": "otelcol_process_memory_rss_bytes", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/service" + }, + "value": [ + 1794380361.122, + "82239488" + ] + } + ] + } + } + }, + "correlation": [ + { + "trace_id": "3c112fa1787d40898e457423046b2e97", + "trace_file": "evidence/trace-3c112fa1787d40898e457423046b2e97.json", + "log_streams": 9, + "logs_file": "evidence/logs-3c112fa1787d40898e457423046b2e97.json" + }, + { + "trace_id": "dbc9b2dc91404a1fa7fae889b013bb34", + "trace_file": "evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json", + "log_streams": 1, + "logs_file": "evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json" + } + ], + "status": "PASS", + "finished_at": "2026-11-11T06:59:21.552928+00:00" +} diff --git a/evidence/lab6-final-validation.txt b/evidence/lab6-final-validation.txt new file mode 100644 index 0000000..62f97b9 --- /dev/null +++ b/evidence/lab6-final-validation.txt @@ -0,0 +1,9 @@ +$ make check validate PYTHON=.venv/bin/python +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/dkhakimovig/mtusi2026lab6/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( + +exit=0 diff --git a/evidence/lab6-incident-metrics.json b/evidence/lab6-incident-metrics.json new file mode 100644 index 0000000..602cbf3 --- /dev/null +++ b/evidence/lab6-incident-metrics.json @@ -0,0 +1,2518 @@ +{ + "start": "2026-11-11T06:55:34.902572+00:00", + "end": "2026-11-11T06:59:21.552928+00:00", + "queries": { + "worker_capacity": [ + { + "metric": { + "__name__": "messenger_worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "2" + ], + [ + 1794380139.903, + "2" + ], + [ + 1794380144.903, + "2" + ], + [ + 1794380149.903, + "2" + ], + [ + 1794380174.903, + "2" + ], + [ + 1794380179.903, + "2" + ], + [ + 1794380184.903, + "2" + ], + [ + 1794380189.903, + "2" + ], + [ + 1794380194.903, + "2" + ], + [ + 1794380199.903, + "2" + ], + [ + 1794380204.903, + "2" + ], + [ + 1794380209.903, + "2" + ], + [ + 1794380214.903, + "2" + ], + [ + 1794380219.903, + "2" + ], + [ + 1794380224.903, + "2" + ], + [ + 1794380229.903, + "2" + ], + [ + 1794380234.903, + "2" + ], + [ + 1794380239.903, + "2" + ], + [ + 1794380244.903, + "2" + ], + [ + 1794380249.903, + "2" + ], + [ + 1794380254.903, + "2" + ], + [ + 1794380259.903, + "2" + ], + [ + 1794380264.903, + "2" + ], + [ + 1794380269.903, + "2" + ], + [ + 1794380274.903, + "2" + ], + [ + 1794380279.903, + "2" + ], + [ + 1794380284.903, + "2" + ], + [ + 1794380289.903, + "2" + ], + [ + 1794380294.903, + "2" + ], + [ + 1794380299.903, + "2" + ], + [ + 1794380304.903, + "2" + ], + [ + 1794380309.903, + "2" + ], + [ + 1794380314.903, + "2" + ], + [ + 1794380319.903, + "2" + ], + [ + 1794380324.903, + "2" + ], + [ + 1794380329.903, + "2" + ], + [ + 1794380334.903, + "2" + ], + [ + 1794380339.903, + "2" + ], + [ + 1794380344.903, + "2" + ], + [ + 1794380349.903, + "2" + ], + [ + 1794380354.903, + "2" + ], + [ + 1794380359.903, + "2" + ] + ] + } + ], + "worker_busy": [ + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + } + ], + "db_used": [ + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "1" + ], + [ + 1794380154.903, + "0" + ], + [ + 1794380159.903, + "1" + ], + [ + 1794380164.903, + "0" + ], + [ + 1794380169.903, + "1" + ], + [ + 1794380174.903, + "1" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "1" + ], + [ + 1794380229.903, + "2" + ], + [ + 1794380234.903, + "2" + ], + [ + 1794380239.903, + "1" + ], + [ + 1794380244.903, + "2" + ], + [ + 1794380249.903, + "1" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "1" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "1" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380154.903, + "0" + ], + [ + 1794380159.903, + "0" + ], + [ + 1794380164.903, + "1" + ], + [ + 1794380169.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "1" + ], + [ + 1794380229.903, + "1" + ], + [ + 1794380234.903, + "1" + ], + [ + 1794380239.903, + "1" + ], + [ + 1794380244.903, + "1" + ], + [ + 1794380249.903, + "1" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + } + ], + "db_waiting": [ + { + "metric": { + "__name__": "messenger_db_pool_waiting", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380154.903, + "0" + ], + [ + 1794380159.903, + "0" + ], + [ + 1794380164.903, + "0" + ], + [ + 1794380169.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_waiting", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_waiting", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380154.903, + "0" + ], + [ + 1794380159.903, + "0" + ], + [ + 1794380164.903, + "0" + ], + [ + 1794380169.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + } + ], + "db_max": [ + { + "metric": { + "__name__": "messenger_db_pool_max", + "deployment_environment_name": "lab", + "exported_instance": "c7b5ac648af7", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "10" + ], + [ + 1794380139.903, + "10" + ], + [ + 1794380144.903, + "10" + ], + [ + 1794380149.903, + "10" + ], + [ + 1794380154.903, + "10" + ], + [ + 1794380159.903, + "10" + ], + [ + 1794380164.903, + "10" + ], + [ + 1794380169.903, + "10" + ], + [ + 1794380174.903, + "10" + ], + [ + 1794380179.903, + "10" + ], + [ + 1794380184.903, + "10" + ], + [ + 1794380189.903, + "10" + ], + [ + 1794380194.903, + "10" + ], + [ + 1794380199.903, + "10" + ], + [ + 1794380204.903, + "10" + ], + [ + 1794380209.903, + "10" + ], + [ + 1794380214.903, + "10" + ], + [ + 1794380219.903, + "10" + ], + [ + 1794380224.903, + "10" + ], + [ + 1794380229.903, + "10" + ], + [ + 1794380234.903, + "10" + ], + [ + 1794380239.903, + "10" + ], + [ + 1794380244.903, + "10" + ], + [ + 1794380249.903, + "10" + ], + [ + 1794380254.903, + "10" + ], + [ + 1794380259.903, + "10" + ], + [ + 1794380264.903, + "10" + ], + [ + 1794380269.903, + "10" + ], + [ + 1794380274.903, + "10" + ], + [ + 1794380279.903, + "10" + ], + [ + 1794380284.903, + "10" + ], + [ + 1794380289.903, + "10" + ], + [ + 1794380294.903, + "10" + ], + [ + 1794380299.903, + "10" + ], + [ + 1794380304.903, + "10" + ], + [ + 1794380309.903, + "10" + ], + [ + 1794380314.903, + "10" + ], + [ + 1794380319.903, + "10" + ], + [ + 1794380324.903, + "10" + ], + [ + 1794380329.903, + "10" + ], + [ + 1794380334.903, + "10" + ], + [ + 1794380339.903, + "10" + ], + [ + 1794380344.903, + "10" + ], + [ + 1794380349.903, + "10" + ], + [ + 1794380354.903, + "10" + ], + [ + 1794380359.903, + "10" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_max", + "deployment_environment_name": "lab", + "exported_instance": "e2edd33d8b58", + "exported_job": "messenger-worker", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "6" + ], + [ + 1794380139.903, + "6" + ], + [ + 1794380144.903, + "6" + ], + [ + 1794380149.903, + "6" + ], + [ + 1794380174.903, + "6" + ], + [ + 1794380179.903, + "6" + ], + [ + 1794380184.903, + "6" + ], + [ + 1794380189.903, + "6" + ], + [ + 1794380194.903, + "6" + ], + [ + 1794380199.903, + "6" + ], + [ + 1794380204.903, + "6" + ], + [ + 1794380209.903, + "6" + ], + [ + 1794380214.903, + "6" + ], + [ + 1794380219.903, + "6" + ], + [ + 1794380224.903, + "6" + ], + [ + 1794380229.903, + "6" + ], + [ + 1794380234.903, + "6" + ], + [ + 1794380239.903, + "6" + ], + [ + 1794380244.903, + "6" + ], + [ + 1794380249.903, + "6" + ], + [ + 1794380254.903, + "6" + ], + [ + 1794380259.903, + "6" + ], + [ + 1794380264.903, + "6" + ], + [ + 1794380269.903, + "6" + ], + [ + 1794380274.903, + "6" + ], + [ + 1794380279.903, + "6" + ], + [ + 1794380284.903, + "6" + ], + [ + 1794380289.903, + "6" + ], + [ + 1794380294.903, + "6" + ], + [ + 1794380299.903, + "6" + ], + [ + 1794380304.903, + "6" + ], + [ + 1794380309.903, + "6" + ], + [ + 1794380314.903, + "6" + ], + [ + 1794380319.903, + "6" + ], + [ + 1794380324.903, + "6" + ], + [ + 1794380329.903, + "6" + ], + [ + 1794380334.903, + "6" + ], + [ + 1794380339.903, + "6" + ], + [ + 1794380344.903, + "6" + ], + [ + 1794380349.903, + "6" + ], + [ + 1794380354.903, + "6" + ], + [ + 1794380359.903, + "6" + ] + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_max", + "deployment_environment_name": "lab", + "exported_instance": "e6be289d2413", + "exported_job": "messenger-api", + "instance": "collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "values": [ + [ + 1794380134.903, + "10" + ], + [ + 1794380139.903, + "10" + ], + [ + 1794380144.903, + "10" + ], + [ + 1794380149.903, + "10" + ], + [ + 1794380154.903, + "10" + ], + [ + 1794380159.903, + "10" + ], + [ + 1794380164.903, + "10" + ], + [ + 1794380169.903, + "10" + ], + [ + 1794380174.903, + "10" + ], + [ + 1794380179.903, + "10" + ], + [ + 1794380184.903, + "10" + ], + [ + 1794380189.903, + "10" + ], + [ + 1794380194.903, + "10" + ], + [ + 1794380199.903, + "10" + ], + [ + 1794380204.903, + "10" + ], + [ + 1794380209.903, + "10" + ], + [ + 1794380214.903, + "10" + ], + [ + 1794380219.903, + "10" + ], + [ + 1794380224.903, + "10" + ], + [ + 1794380229.903, + "10" + ], + [ + 1794380234.903, + "10" + ], + [ + 1794380239.903, + "10" + ], + [ + 1794380244.903, + "10" + ], + [ + 1794380249.903, + "10" + ], + [ + 1794380254.903, + "10" + ], + [ + 1794380259.903, + "10" + ], + [ + 1794380264.903, + "10" + ], + [ + 1794380269.903, + "10" + ], + [ + 1794380274.903, + "10" + ], + [ + 1794380279.903, + "10" + ], + [ + 1794380284.903, + "10" + ], + [ + 1794380289.903, + "10" + ], + [ + 1794380294.903, + "10" + ], + [ + 1794380299.903, + "10" + ], + [ + 1794380304.903, + "10" + ], + [ + 1794380309.903, + "10" + ], + [ + 1794380314.903, + "10" + ], + [ + 1794380319.903, + "10" + ], + [ + 1794380324.903, + "10" + ], + [ + 1794380329.903, + "10" + ], + [ + 1794380334.903, + "10" + ], + [ + 1794380339.903, + "10" + ], + [ + 1794380344.903, + "10" + ], + [ + 1794380349.903, + "10" + ], + [ + 1794380354.903, + "10" + ], + [ + 1794380359.903, + "10" + ] + ] + } + ], + "api_5xx": [ + { + "metric": {}, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "0" + ], + [ + 1794380144.903, + "0" + ], + [ + 1794380149.903, + "0" + ], + [ + 1794380154.903, + "0" + ], + [ + 1794380159.903, + "0" + ], + [ + 1794380164.903, + "0" + ], + [ + 1794380169.903, + "0" + ], + [ + 1794380174.903, + "0" + ], + [ + 1794380179.903, + "0" + ], + [ + 1794380184.903, + "0" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0.10931948619841486" + ], + [ + 1794380234.903, + "0.18179834927098862" + ], + [ + 1794380239.903, + "0.2908985127813534" + ], + [ + 1794380244.903, + "0.36364297532682416" + ], + [ + 1794380249.903, + "0.45453719023290484" + ], + [ + 1794380254.903, + "0.5090446323061539" + ], + [ + 1794380259.903, + "0.5091001654575538" + ], + [ + 1794380264.903, + "0.5091094221608058" + ], + [ + 1794380269.903, + "0.5100646689133801" + ], + [ + 1794380274.903, + "0.5099810578464229" + ], + [ + 1794380279.903, + "0.5099996357145459" + ], + [ + 1794380284.903, + "0.4007358968287218" + ], + [ + 1794380289.903, + "0.3279046890370533" + ], + [ + 1794380294.903, + "0.21857923497267756" + ], + [ + 1794380299.903, + "0.14570887367040652" + ], + [ + 1794380304.903, + "0.054644808743169404" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + } + ], + "queue_age": [ + { + "metric": {}, + "values": [ + [ + 1794380134.903, + "0" + ], + [ + 1794380139.903, + "1.178956" + ], + [ + 1794380144.903, + "6.185773" + ], + [ + 1794380149.903, + "11.185192" + ], + [ + 1794380154.903, + "16.17813" + ], + [ + 1794380159.903, + "21.184781" + ], + [ + 1794380164.903, + "26.179077" + ], + [ + 1794380169.903, + "31.182827" + ], + [ + 1794380174.903, + "36.181197" + ], + [ + 1794380179.903, + "41.061529" + ], + [ + 1794380184.903, + "46.063726" + ], + [ + 1794380189.903, + "0" + ], + [ + 1794380194.903, + "0" + ], + [ + 1794380199.903, + "0" + ], + [ + 1794380204.903, + "0" + ], + [ + 1794380209.903, + "0" + ], + [ + 1794380214.903, + "0" + ], + [ + 1794380219.903, + "0" + ], + [ + 1794380224.903, + "0" + ], + [ + 1794380229.903, + "0" + ], + [ + 1794380234.903, + "0" + ], + [ + 1794380239.903, + "0" + ], + [ + 1794380244.903, + "0" + ], + [ + 1794380249.903, + "0" + ], + [ + 1794380254.903, + "0" + ], + [ + 1794380259.903, + "0" + ], + [ + 1794380264.903, + "0" + ], + [ + 1794380269.903, + "0" + ], + [ + 1794380274.903, + "0" + ], + [ + 1794380279.903, + "0" + ], + [ + 1794380284.903, + "0" + ], + [ + 1794380289.903, + "0" + ], + [ + 1794380294.903, + "0" + ], + [ + 1794380299.903, + "0" + ], + [ + 1794380304.903, + "0" + ], + [ + 1794380309.903, + "0" + ], + [ + 1794380314.903, + "0" + ], + [ + 1794380319.903, + "0" + ], + [ + 1794380324.903, + "0" + ], + [ + 1794380329.903, + "0" + ], + [ + 1794380334.903, + "0" + ], + [ + 1794380339.903, + "0" + ], + [ + 1794380344.903, + "0" + ], + [ + 1794380349.903, + "0" + ], + [ + 1794380354.903, + "0" + ], + [ + 1794380359.903, + "0" + ] + ] + } + ] + } +} diff --git a/evidence/lab6-incidents-summary.json b/evidence/lab6-incidents-summary.json new file mode 100644 index 0000000..33bfea7 --- /dev/null +++ b/evidence/lab6-incidents-summary.json @@ -0,0 +1,40 @@ +{ + "queue_first_firing_seconds": 20.18, + "queue_max_observed_age_seconds": 26.179077, + "ready_jobs": 6, + "db_http_status_counts": { + "503": 28 + }, + "db_max_latency_seconds": 2.0508444580000003, + "db_recovery_poll_count": 10, + "trace_summary": [ + { + "trace_id": "3c112fa1787d40898e457423046b2e97", + "span_names": { + "POST /api/v1/chats/{chat_id}/attachments": 1, + "postgresql SELECT": 13, + "postgresql BEGIN": 9, + "s3 PUT": 10, + "postgresql INSERT": 3, + "postgresql COMMIT": 9, + "attachments publish": 4, + "attachments process": 4, + "postgresql UPDATE": 19, + "s3 download and image render": 4, + "s3 GET": 4 + }, + "longest_span_seconds": 0.169544764, + "log_streams": 9 + }, + { + "trace_id": "dbc9b2dc91404a1fa7fae889b013bb34", + "span_names": { + "postgresql SELECT": 5, + "postgresql BEGIN": 1, + "POST /api/v1/chats/{chat_id}/messages": 1 + }, + "longest_span_seconds": 2.008227803, + "log_streams": 1 + } + ] +} diff --git a/evidence/lab6-persistence.txt b/evidence/lab6-persistence.txt new file mode 100644 index 0000000..3bde9b5 --- /dev/null +++ b/evidence/lab6-persistence.txt @@ -0,0 +1,6 @@ +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18086" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit=0 diff --git a/evidence/lab6-provisioning-run.txt b/evidence/lab6-provisioning-run.txt new file mode 100644 index 0000000..ae1b192 --- /dev/null +++ b/evidence/lab6-provisioning-run.txt @@ -0,0 +1,4 @@ +$ python3 scripts/verify_observability.py +{"status": "PASS", "panels": 15, "queries": 18, "label_policy": "PASS"} + +exit=0 diff --git a/evidence/lab6-provisioning.json b/evidence/lab6-provisioning.json new file mode 100644 index 0000000..c540b92 --- /dev/null +++ b/evidence/lab6-provisioning.json @@ -0,0 +1,120 @@ +{ + "at": "2026-11-11T07:14:35.828099+00:00", + "status": "PASS", + "targets": { + "collector": "up", + "loki": "up", + "messenger": "up", + "prometheus": "up", + "tempo": "up" + }, + "dashboard_uid": "messenger", + "panel_count": 15, + "queries": [ + { + "panel": "API RPS (business routes)", + "query": "sum(rate(messenger_http_requests_total[1m])) by (route)", + "series": 7 + }, + { + "panel": "API 5xx / all business requests", + "query": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total[1m])),0.001)", + "series": 1 + }, + { + "panel": "API p50 / p95 / p99 (all replicas, histogram)", + "query": "histogram_quantile(0.5,sum(rate(messenger_http_duration_seconds_bucket[1m])) by(le))", + "series": 1 + }, + { + "panel": "API p50 / p95 / p99 (all replicas, histogram)", + "query": "histogram_quantile(0.95,sum(rate(messenger_http_duration_seconds_bucket[1m])) by(le))", + "series": 1 + }, + { + "panel": "API p50 / p95 / p99 (all replicas, histogram)", + "query": "histogram_quantile(0.99,sum(rate(messenger_http_duration_seconds_bucket[1m])) by(le))", + "series": 1 + }, + { + "panel": "Worker job outcomes per second", + "query": "sum(rate(messenger_jobs_total[1m])) by(outcome)", + "series": 1 + }, + { + "panel": "Worker job p95 duration", + "query": "histogram_quantile(0.95,sum(rate(messenger_job_duration_seconds_bucket[1m])) by(le,outcome))", + "series": 1 + }, + { + "panel": "Time from durable acceptance to ready p95", + "query": "histogram_quantile(0.95,sum(rate(messenger_attachment_ready_seconds_bucket[1m])) by(le))", + "series": 1 + }, + { + "panel": "Authoritative queue depth (max across observers)", + "query": "max(messenger_queue_depth)", + "series": 1 + }, + { + "panel": "Oldest unfinished job age", + "query": "max(messenger_queue_oldest_age_seconds)", + "series": 1 + }, + { + "panel": "DB pool acquired / acquiring / max", + "query": "messenger_db_pool_used", + "series": 3 + }, + { + "panel": "DB pool acquired / acquiring / max", + "query": "messenger_db_pool_waiting", + "series": 3 + }, + { + "panel": "DB pool acquired / acquiring / max", + "query": "messenger_db_pool_max", + "series": 3 + }, + { + "panel": "Worker saturation busy / capacity", + "query": "sum(messenger_worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(messenger_worker_capacity{service_name=\"messenger-worker\"}),1)", + "series": 1 + }, + { + "panel": "Go runtime reserved memory (MemStats.Sys)", + "query": "messenger_process_memory_bytes", + "series": 3 + }, + { + "panel": "Process CPU seconds / second", + "query": "rate(messenger_process_cpu_seconds_total[1m])", + "series": 3 + }, + { + "panel": "SDK failed telemetry export operations", + "query": "messenger_telemetry_export_errors", + "series": 3 + }, + { + "panel": "Exporter failed spans (attempts, not unique loss)", + "query": "sum(otelcol_exporter_send_failed_spans_total) by (exporter)", + "series": 1 + } + ], + "metric_series_checked": 2554, + "required_metrics": [ + "messenger_attachment_ready_seconds_bucket", + "messenger_db_pool_used", + "messenger_db_pool_waiting", + "messenger_http_duration_seconds_bucket", + "messenger_http_requests_total", + "messenger_job_duration_seconds_bucket", + "messenger_jobs_total", + "messenger_queue_depth", + "messenger_queue_oldest_age_seconds", + "messenger_worker_busy", + "messenger_worker_capacity" + ], + "label_policy": "PASS" +} diff --git a/evidence/lab6-research-run.txt b/evidence/lab6-research-run.txt new file mode 100644 index 0000000..0147e96 --- /dev/null +++ b/evidence/lab6-research-run.txt @@ -0,0 +1,16 @@ +Running off repetition 1 +{"configuration": "off", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 11.934, "p95": 29.423, "p99": 42.462}, "trace_retention": {"read": {"sent": 90, "found": 0}, "write": {"sent": 18, "found": 0}, "upload": {"sent": 6, "found": 0}, "client_error": {"sent": 6, "found": 0}}} +Running full repetition 1 +{"configuration": "full", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 11.856, "p95": 31.41, "p99": 42.052}, "trace_retention": {"read": {"sent": 90, "found": 90}, "write": {"sent": 18, "found": 18}, "upload": {"sent": 6, "found": 6}, "client_error": {"sent": 6, "found": 6}}} +Running ten_percent repetition 1 +{"configuration": "ten_percent", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 9.991, "p95": 52.789, "p99": 168.924}, "trace_retention": {"read": {"sent": 90, "found": 13}, "write": {"sent": 18, "found": 2}, "upload": {"sent": 6, "found": 1}, "client_error": {"sent": 6, "found": 1}}} +Running balanced repetition 1 +{"configuration": "balanced", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 10.423, "p95": 30.254, "p99": 52.116}, "trace_retention": {"read": {"sent": 90, "found": 54}, "write": {"sent": 18, "found": 10}, "upload": {"sent": 6, "found": 4}, "client_error": {"sent": 6, "found": 4}}} +Running balanced repetition 2 +{"configuration": "balanced", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 11.346, "p95": 37.555, "p99": 72.706}, "trace_retention": {"read": {"sent": 90, "found": 41}, "write": {"sent": 18, "found": 7}, "upload": {"sent": 6, "found": 2}, "client_error": {"sent": 6, "found": 3}}} +Running ten_percent repetition 2 +{"configuration": "ten_percent", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 12.281, "p95": 42.42, "p99": 63.216}, "trace_retention": {"read": {"sent": 90, "found": 12}, "write": {"sent": 18, "found": 4}, "upload": {"sent": 6, "found": 0}, "client_error": {"sent": 6, "found": 1}}} +Running full repetition 2 +{"configuration": "full", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 12.379, "p95": 31.356, "p99": 45.834}, "trace_retention": {"read": {"sent": 90, "found": 90}, "write": {"sent": 18, "found": 18}, "upload": {"sent": 6, "found": 6}, "client_error": {"sent": 6, "found": 6}}} +Running off repetition 2 +{"configuration": "off", "unexpected_errors": 0, "all_request_latency_ms": {"p50": 10.728, "p95": 30.012, "p99": 63.853}, "trace_retention": {"read": {"sent": 90, "found": 0}, "write": {"sent": 18, "found": 0}, "upload": {"sent": 6, "found": 0}, "client_error": {"sent": 6, "found": 0}}} diff --git a/evidence/lab6-research-summary.json b/evidence/lab6-research-summary.json new file mode 100644 index 0000000..f186180 --- /dev/null +++ b/evidence/lab6-research-summary.json @@ -0,0 +1,70 @@ +{ + "off": { + "counter_delta_totals": { + "exported_spans": 0, + "failed_spans": 0, + "accepted_spans": 0, + "refused_spans": 0, + "tempo_ingested_bytes": 0, + "loki_ingested_bytes": 0, + "prometheus_samples": 46185.0 + }, + "traces_found": 0, + "requests": 240, + "slow_found": 0, + "slow_sent": 12, + "client_error_found": 0, + "client_error_sent": 12 + }, + "full": { + "counter_delta_totals": { + "exported_spans": 4947.0, + "failed_spans": 0, + "accepted_spans": 4898.0, + "refused_spans": 0, + "tempo_ingested_bytes": 697602.0, + "loki_ingested_bytes": 198451.0, + "prometheus_samples": 44282.0 + }, + "traces_found": 240, + "requests": 240, + "slow_found": 12, + "slow_sent": 12, + "client_error_found": 12, + "client_error_sent": 12 + }, + "ten_percent": { + "counter_delta_totals": { + "exported_spans": 461.0, + "failed_spans": 0, + "accepted_spans": 453.0, + "refused_spans": 0, + "tempo_ingested_bytes": 95539.0, + "loki_ingested_bytes": 196690.0, + "prometheus_samples": 49391.0 + }, + "traces_found": 34, + "requests": 240, + "slow_found": 3, + "slow_sent": 12, + "client_error_found": 2, + "client_error_sent": 12 + }, + "balanced": { + "counter_delta_totals": { + "exported_spans": 2402.0, + "failed_spans": 0, + "accepted_spans": 2402.0, + "refused_spans": 0, + "tempo_ingested_bytes": 362056.0, + "loki_ingested_bytes": 184978.0, + "prometheus_samples": 44897.0 + }, + "traces_found": 125, + "requests": 240, + "slow_found": 8, + "slow_sent": 12, + "client_error_found": 7, + "client_error_sent": 12 + } +} diff --git a/evidence/lab6-research.csv b/evidence/lab6-research.csv new file mode 100644 index 0000000..e3aaa67 --- /dev/null +++ b/evidence/lab6-research.csv @@ -0,0 +1,9 @@ +configuration,repetition,rps,unexpected_errors,p50,p95,p99,app_cpu_percent_sum_mean,app_ram_mib_sum_peak,telemetry_cpu_percent_sum_mean,telemetry_ram_mib_sum_peak,sent_requests,retained_request_traces,missing_request_traces,exported_spans,failed_spans,accepted_spans,refused_spans,tempo_ingested_bytes,loki_ingested_bytes,prometheus_samples +off,1,8.047,0,11.934,29.423,42.462,8.132,70.39,4.816,616.03,120,0,120,0.0,0.0,0.0,0.0,0.0,0.0,24720.0 +full,1,8.053,0,11.856,31.41,42.052,9.962,73.733,10.37,608.78,120,120,0,2663.0,0.0,2615.0,0.0,343731.0,94264.0,21738.0 +ten_percent,1,8.057,0,9.991,52.789,168.924,8.846,71.971,10.784,502.69,120,17,103,246.0,0.0,241.0,0.0,52631.0,103980.0,26869.0 +balanced,1,8.062,0,10.423,30.254,52.116,8.462,71.314,10.322,651.25,120,72,48,1214.0,0.0,1214.0,0.0,195601.0,93278.0,22393.0 +balanced,2,8.048,0,11.346,37.555,72.706,8.014,59.703,7.096,600.74,120,53,67,1188.0,0.0,1188.0,0.0,166455.0,91700.0,22504.0 +ten_percent,2,8.061,0,12.281,42.42,63.216,7.942,60.066,13.176,671.03,120,17,103,215.0,0.0,212.0,0.0,42908.0,92710.0,22522.0 +full,2,8.048,0,12.379,31.356,45.834,9.114,74.053,7.598,676.44,120,120,0,2284.0,0.0,2283.0,0.0,353871.0,104187.0,22544.0 +off,2,8.05,0,10.728,30.012,63.853,7.404,64.176,6.616,563.21,120,0,120,0.0,0.0,0.0,0.0,0.0,0.0,21465.0 diff --git a/evidence/lab6-research.json b/evidence/lab6-research.json new file mode 100644 index 0000000..ab3f1fa --- /dev/null +++ b/evidence/lab6-research.json @@ -0,0 +1,3456 @@ +{ + "started_at": "2026-11-11T06:59:53.569425+00:00", + "method": "Fixed-rate schedule, fixed request count/mix; 20 warmups; SDK and backend flush; sampled IDs queried from Tempo.", + "trials": [ + { + "configuration": "off", + "repetition": 1, + "disabled": true, + "sampling_ratio": 1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.913, + "successful_rps": 8.047, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 11.934, + "p95": 29.423, + "p99": 42.462 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 70821.0, + "failed_spans": 10491.0, + "accepted_spans": 142705.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 860569.0, + "loki_ingested_bytes": 2748101.0, + "prometheus_samples": 1793839.0 + }, + "counter_after": { + "exported_spans": 70821.0, + "failed_spans": 10491.0, + "accepted_spans": 142705.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 860569.0, + "loki_ingested_bytes": 2748101.0, + "prometheus_samples": 1818559.0 + }, + "counter_delta": { + "exported_spans": 0.0, + "failed_spans": 0.0, + "accepted_spans": 0.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 0.0, + "loki_ingested_bytes": 0.0, + "prometheus_samples": 24720.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 0 + }, + "write": { + "sent": 18, + "found": 0 + }, + "upload": { + "sent": 6, + "found": 0 + }, + "client_error": { + "sent": 6, + "found": 0 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 0 + }, + "resources": [ + { + "at": 1794380416.330681, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.86%", + "MemUsage": "31.2MiB / 512MiB" + }, + { + "at": 1794380416.330693, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "0.91%", + "MemUsage": "5.23MiB / 512MiB" + }, + { + "at": 1794380416.330699, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.64%", + "MemUsage": "26.22MiB / 512MiB" + }, + { + "at": 1794380416.330704, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.90%", + "MemUsage": "178.8MiB / 512MiB" + }, + { + "at": 1794380416.33071, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.12%", + "MemUsage": "10.16MiB / 64MiB" + }, + { + "at": 1794380416.330715, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.48%", + "MemUsage": "157.6MiB / 192MiB" + }, + { + "at": 1794380416.33072, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.65%", + "MemUsage": "36.29MiB / 512MiB" + }, + { + "at": 1794380416.3307252, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.86%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380416.33073, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "3.68%", + "MemUsage": "253.2MiB / 512MiB" + }, + { + "at": 1794380416.3307352, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.11%", + "MemUsage": "74.16MiB / 192MiB" + }, + { + "at": 1794380416.33074, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "0.88%", + "MemUsage": "120.9MiB / 192MiB" + }, + { + "at": 1794380416.330745, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "3.34%", + "MemUsage": "84.57MiB / 256MiB" + }, + { + "at": 1794380419.602389, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.18%", + "MemUsage": "31.66MiB / 512MiB" + }, + { + "at": 1794380419.602397, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "8.03%", + "MemUsage": "5.559MiB / 512MiB" + }, + { + "at": 1794380419.602401, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.35%", + "MemUsage": "28.15MiB / 512MiB" + }, + { + "at": 1794380419.602405, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.51%", + "MemUsage": "178.8MiB / 512MiB" + }, + { + "at": 1794380419.602408, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.31%", + "MemUsage": "10.17MiB / 64MiB" + }, + { + "at": 1794380419.602412, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.55%", + "MemUsage": "157.6MiB / 192MiB" + }, + { + "at": 1794380419.602416, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.97%", + "MemUsage": "36.36MiB / 512MiB" + }, + { + "at": 1794380419.602419, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.69%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380419.602423, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "3.64%", + "MemUsage": "256.9MiB / 512MiB" + }, + { + "at": 1794380419.602426, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.14%", + "MemUsage": "74.16MiB / 192MiB" + }, + { + "at": 1794380419.602429, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.76%", + "MemUsage": "120.9MiB / 192MiB" + }, + { + "at": 1794380419.602433, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.12%", + "MemUsage": "84.57MiB / 256MiB" + }, + { + "at": 1794380422.796191, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.43%", + "MemUsage": "32.27MiB / 512MiB" + }, + { + "at": 1794380422.796198, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "5.97%", + "MemUsage": "5.531MiB / 512MiB" + }, + { + "at": 1794380422.796202, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.71%", + "MemUsage": "29.36MiB / 512MiB" + }, + { + "at": 1794380422.796206, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.42%", + "MemUsage": "178.8MiB / 512MiB" + }, + { + "at": 1794380422.7962098, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.88%", + "MemUsage": "10.17MiB / 64MiB" + }, + { + "at": 1794380422.796213, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.29%", + "MemUsage": "157.6MiB / 192MiB" + }, + { + "at": 1794380422.7962172, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.11%", + "MemUsage": "36.42MiB / 512MiB" + }, + { + "at": 1794380422.796221, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.28%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380422.796224, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "4.01%", + "MemUsage": "257MiB / 512MiB" + }, + { + "at": 1794380422.7962282, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.58%", + "MemUsage": "74.16MiB / 192MiB" + }, + { + "at": 1794380422.7962308, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "0.91%", + "MemUsage": "120.9MiB / 192MiB" + }, + { + "at": 1794380422.796234, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.90%", + "MemUsage": "84.57MiB / 256MiB" + }, + { + "at": 1794380425.981306, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.85%", + "MemUsage": "32.71MiB / 512MiB" + }, + { + "at": 1794380425.98132, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.05%", + "MemUsage": "5.59MiB / 512MiB" + }, + { + "at": 1794380425.981326, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.13%", + "MemUsage": "30.46MiB / 512MiB" + }, + { + "at": 1794380425.981333, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.68%", + "MemUsage": "178.8MiB / 512MiB" + }, + { + "at": 1794380425.981339, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.15%", + "MemUsage": "10.18MiB / 64MiB" + }, + { + "at": 1794380425.9813461, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.57%", + "MemUsage": "157.6MiB / 192MiB" + }, + { + "at": 1794380425.981353, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.14%", + "MemUsage": "36.46MiB / 512MiB" + }, + { + "at": 1794380425.981358, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.87%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380425.981364, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "0.32%", + "MemUsage": "257MiB / 512MiB" + }, + { + "at": 1794380425.9813702, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.13%", + "MemUsage": "74.16MiB / 192MiB" + }, + { + "at": 1794380425.981375, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.59%", + "MemUsage": "120.9MiB / 192MiB" + }, + { + "at": 1794380425.98138, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "3.24%", + "MemUsage": "84.57MiB / 256MiB" + }, + { + "at": 1794380429.196782, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.29%", + "MemUsage": "33.23MiB / 512MiB" + }, + { + "at": 1794380429.196796, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.44%", + "MemUsage": "5.449MiB / 512MiB" + }, + { + "at": 1794380429.1968021, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.82%", + "MemUsage": "31.57MiB / 512MiB" + }, + { + "at": 1794380429.196809, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.52%", + "MemUsage": "178.8MiB / 512MiB" + }, + { + "at": 1794380429.196815, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.16%", + "MemUsage": "10.18MiB / 64MiB" + }, + { + "at": 1794380429.1968231, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.41%", + "MemUsage": "157.6MiB / 192MiB" + }, + { + "at": 1794380429.196828, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.69%", + "MemUsage": "36.58MiB / 512MiB" + }, + { + "at": 1794380429.196835, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.69%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380429.19684, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "5.20%", + "MemUsage": "257.1MiB / 512MiB" + }, + { + "at": 1794380429.196846, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.17%", + "MemUsage": "74.16MiB / 192MiB" + }, + { + "at": 1794380429.196852, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.32%", + "MemUsage": "120.9MiB / 192MiB" + }, + { + "at": 1794380429.1968582, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.56%", + "MemUsage": "84.57MiB / 256MiB" + } + ] + }, + { + "configuration": "full", + "repetition": 1, + "disabled": false, + "sampling_ratio": 1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.901, + "successful_rps": 8.053, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 11.856, + "p95": 31.41, + "p99": 42.052 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 71379.0, + "failed_spans": 10491.0, + "accepted_spans": 143374.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 996728.0, + "loki_ingested_bytes": 2769132.0, + "prometheus_samples": 1835232.0 + }, + "counter_after": { + "exported_spans": 74042.0, + "failed_spans": 10491.0, + "accepted_spans": 145989.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1340459.0, + "loki_ingested_bytes": 2863396.0, + "prometheus_samples": 1856970.0 + }, + "counter_delta": { + "exported_spans": 2663.0, + "failed_spans": 0.0, + "accepted_spans": 2615.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 343731.0, + "loki_ingested_bytes": 94264.0, + "prometheus_samples": 21738.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 90 + }, + "write": { + "sent": 18, + "found": 18 + }, + "upload": { + "sent": 6, + "found": 6 + }, + "client_error": { + "sent": 6, + "found": 6 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 6 + }, + "resources": [ + { + "at": 1794380464.271816, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.40%", + "MemUsage": "6.32MiB / 512MiB" + }, + { + "at": 1794380464.2718232, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.14%", + "MemUsage": "28MiB / 512MiB" + }, + { + "at": 1794380464.271828, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.68%", + "MemUsage": "27.73MiB / 512MiB" + }, + { + "at": 1794380464.271832, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.98%", + "MemUsage": "66.32MiB / 512MiB" + }, + { + "at": 1794380464.2718349, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.61%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380464.271839, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.54%", + "MemUsage": "157.8MiB / 192MiB" + }, + { + "at": 1794380464.271842, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "5.01%", + "MemUsage": "36.43MiB / 512MiB" + }, + { + "at": 1794380464.2718458, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.26%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380464.271849, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "5.48%", + "MemUsage": "279.1MiB / 512MiB" + }, + { + "at": 1794380464.271853, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.41%", + "MemUsage": "74.18MiB / 192MiB" + }, + { + "at": 1794380464.271856, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.71%", + "MemUsage": "134.9MiB / 192MiB" + }, + { + "at": 1794380464.271859, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.12%", + "MemUsage": "84.6MiB / 256MiB" + }, + { + "at": 1794380467.4436228, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.24%", + "MemUsage": "6.574MiB / 512MiB" + }, + { + "at": 1794380467.44363, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.86%", + "MemUsage": "28.89MiB / 512MiB" + }, + { + "at": 1794380467.443635, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.10%", + "MemUsage": "29.1MiB / 512MiB" + }, + { + "at": 1794380467.443638, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "18.25%", + "MemUsage": "156.5MiB / 512MiB" + }, + { + "at": 1794380467.4436429, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.86%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380467.443646, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.33%", + "MemUsage": "157.8MiB / 192MiB" + }, + { + "at": 1794380467.44365, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "6.72%", + "MemUsage": "39.48MiB / 512MiB" + }, + { + "at": 1794380467.4436538, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.25%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380467.4436579, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "4.47%", + "MemUsage": "279.2MiB / 512MiB" + }, + { + "at": 1794380467.443661, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "8.31%", + "MemUsage": "74.18MiB / 192MiB" + }, + { + "at": 1794380467.4436648, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.35%", + "MemUsage": "134.9MiB / 192MiB" + }, + { + "at": 1794380467.443668, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.55%", + "MemUsage": "84.6MiB / 256MiB" + }, + { + "at": 1794380470.646837, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.04%", + "MemUsage": "6.551MiB / 512MiB" + }, + { + "at": 1794380470.6468449, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.40%", + "MemUsage": "29.92MiB / 512MiB" + }, + { + "at": 1794380470.646849, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.88%", + "MemUsage": "31.77MiB / 512MiB" + }, + { + "at": 1794380470.646854, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "2.44%", + "MemUsage": "156.6MiB / 512MiB" + }, + { + "at": 1794380470.646858, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.84%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380470.646862, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.27%", + "MemUsage": "157.8MiB / 192MiB" + }, + { + "at": 1794380470.6468658, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.79%", + "MemUsage": "39.52MiB / 512MiB" + }, + { + "at": 1794380470.64687, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.71%", + "MemUsage": "5.742MiB / 384MiB" + }, + { + "at": 1794380470.6468742, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "1.24%", + "MemUsage": "279.3MiB / 512MiB" + }, + { + "at": 1794380470.646878, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.61%", + "MemUsage": "74.18MiB / 192MiB" + }, + { + "at": 1794380470.646882, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.01%", + "MemUsage": "134.9MiB / 192MiB" + }, + { + "at": 1794380470.6468859, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.71%", + "MemUsage": "84.6MiB / 256MiB" + }, + { + "at": 1794380473.827551, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.41%", + "MemUsage": "6.672MiB / 512MiB" + }, + { + "at": 1794380473.827565, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.81%", + "MemUsage": "30.95MiB / 512MiB" + }, + { + "at": 1794380473.8275719, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.29%", + "MemUsage": "33.43MiB / 512MiB" + }, + { + "at": 1794380473.8275788, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.78%", + "MemUsage": "156.6MiB / 512MiB" + }, + { + "at": 1794380473.827586, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.06%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380473.8275928, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.54%", + "MemUsage": "157.8MiB / 192MiB" + }, + { + "at": 1794380473.827601, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "5.03%", + "MemUsage": "39.82MiB / 512MiB" + }, + { + "at": 1794380473.827609, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "3.38%", + "MemUsage": "5.742MiB / 384MiB" + }, + { + "at": 1794380473.827615, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.10%", + "MemUsage": "279.3MiB / 512MiB" + }, + { + "at": 1794380473.827622, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.08%", + "MemUsage": "74.18MiB / 192MiB" + }, + { + "at": 1794380473.827629, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.46%", + "MemUsage": "134.9MiB / 192MiB" + }, + { + "at": 1794380473.827636, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.32%", + "MemUsage": "84.6MiB / 256MiB" + }, + { + "at": 1794380477.06196, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.74%", + "MemUsage": "6.793MiB / 512MiB" + }, + { + "at": 1794380477.061972, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "4.01%", + "MemUsage": "32.01MiB / 512MiB" + }, + { + "at": 1794380477.061976, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.81%", + "MemUsage": "34.93MiB / 512MiB" + }, + { + "at": 1794380477.0619788, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.83%", + "MemUsage": "157.3MiB / 512MiB" + }, + { + "at": 1794380477.0619829, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.31%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380477.0619872, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.53%", + "MemUsage": "157.8MiB / 192MiB" + }, + { + "at": 1794380477.061991, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.14%", + "MemUsage": "39.97MiB / 512MiB" + }, + { + "at": 1794380477.061995, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.45%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380477.0619981, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "7.53%", + "MemUsage": "279.3MiB / 512MiB" + }, + { + "at": 1794380477.062001, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.06%", + "MemUsage": "74.18MiB / 192MiB" + }, + { + "at": 1794380477.062005, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.32%", + "MemUsage": "134.9MiB / 192MiB" + }, + { + "at": 1794380477.0620081, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.34%", + "MemUsage": "84.6MiB / 256MiB" + } + ] + }, + { + "configuration": "ten_percent", + "repetition": 1, + "disabled": false, + "sampling_ratio": 0.1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.894, + "successful_rps": 8.057, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 9.991, + "p95": 52.789, + "p99": 168.924 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 74625.0, + "failed_spans": 10491.0, + "accepted_spans": 146514.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1452907.0, + "loki_ingested_bytes": 2877796.0, + "prometheus_samples": 1879402.0 + }, + "counter_after": { + "exported_spans": 74871.0, + "failed_spans": 10491.0, + "accepted_spans": 146755.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1505538.0, + "loki_ingested_bytes": 2981776.0, + "prometheus_samples": 1906271.0 + }, + "counter_delta": { + "exported_spans": 246.0, + "failed_spans": 0.0, + "accepted_spans": 241.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 52631.0, + "loki_ingested_bytes": 103980.0, + "prometheus_samples": 26869.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 13 + }, + "write": { + "sent": 18, + "found": 2 + }, + "upload": { + "sent": 6, + "found": 1 + }, + "client_error": { + "sent": 6, + "found": 1 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 2 + }, + "resources": [ + { + "at": 1794380515.465934, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.15%", + "MemUsage": "27.01MiB / 512MiB" + }, + { + "at": 1794380515.465947, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.50%", + "MemUsage": "27.58MiB / 512MiB" + }, + { + "at": 1794380515.465953, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "5.98%", + "MemUsage": "6.379MiB / 512MiB" + }, + { + "at": 1794380515.4659579, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "9.13%", + "MemUsage": "65.5MiB / 512MiB" + }, + { + "at": 1794380515.465964, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.89%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380515.4659698, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.37%", + "MemUsage": "158.5MiB / 192MiB" + }, + { + "at": 1794380515.465976, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.83%", + "MemUsage": "38.38MiB / 512MiB" + }, + { + "at": 1794380515.465981, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.46%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380515.465986, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "3.51%", + "MemUsage": "279.8MiB / 512MiB" + }, + { + "at": 1794380515.465991, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.55%", + "MemUsage": "73.34MiB / 192MiB" + }, + { + "at": 1794380515.465996, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.77%", + "MemUsage": "119MiB / 192MiB" + }, + { + "at": 1794380515.4659998, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "1.45%", + "MemUsage": "84.82MiB / 256MiB" + }, + { + "at": 1794380518.671969, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.35%", + "MemUsage": "27.74MiB / 512MiB" + }, + { + "at": 1794380518.6719759, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.69%", + "MemUsage": "29.7MiB / 512MiB" + }, + { + "at": 1794380518.6719801, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "8.41%", + "MemUsage": "7.09MiB / 512MiB" + }, + { + "at": 1794380518.671983, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.40%", + "MemUsage": "65.55MiB / 512MiB" + }, + { + "at": 1794380518.671987, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.08%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380518.671991, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.83%", + "MemUsage": "158.5MiB / 192MiB" + }, + { + "at": 1794380518.6719942, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.58%", + "MemUsage": "38.45MiB / 512MiB" + }, + { + "at": 1794380518.671998, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.02%", + "MemUsage": "5.742MiB / 384MiB" + }, + { + "at": 1794380518.6720011, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.52%", + "MemUsage": "279.9MiB / 512MiB" + }, + { + "at": 1794380518.672004, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "3.33%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380518.672008, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "4.60%", + "MemUsage": "119MiB / 192MiB" + }, + { + "at": 1794380518.672011, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.89%", + "MemUsage": "84.83MiB / 256MiB" + }, + { + "at": 1794380522.148596, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.43%", + "MemUsage": "28.74MiB / 512MiB" + }, + { + "at": 1794380522.148603, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.99%", + "MemUsage": "31.42MiB / 512MiB" + }, + { + "at": 1794380522.148607, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.50%", + "MemUsage": "7.23MiB / 512MiB" + }, + { + "at": 1794380522.148611, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.27%", + "MemUsage": "66.3MiB / 512MiB" + }, + { + "at": 1794380522.148615, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.27%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380522.148619, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.91%", + "MemUsage": "158.5MiB / 192MiB" + }, + { + "at": 1794380522.148623, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "6.33%", + "MemUsage": "38.56MiB / 512MiB" + }, + { + "at": 1794380522.1486259, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.91%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380522.14863, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "5.76%", + "MemUsage": "279.9MiB / 512MiB" + }, + { + "at": 1794380522.148633, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "4.01%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380522.1486368, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.81%", + "MemUsage": "119MiB / 192MiB" + }, + { + "at": 1794380522.14864, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.93%", + "MemUsage": "84.83MiB / 256MiB" + }, + { + "at": 1794380525.5592651, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.09%", + "MemUsage": "29.54MiB / 512MiB" + }, + { + "at": 1794380525.559272, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.68%", + "MemUsage": "33.57MiB / 512MiB" + }, + { + "at": 1794380525.559277, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "2.39%", + "MemUsage": "7.367MiB / 512MiB" + }, + { + "at": 1794380525.5592809, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "2.64%", + "MemUsage": "66.31MiB / 512MiB" + }, + { + "at": 1794380525.559285, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.11%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380525.559289, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.50%", + "MemUsage": "158.5MiB / 192MiB" + }, + { + "at": 1794380525.559293, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.15%", + "MemUsage": "38.98MiB / 512MiB" + }, + { + "at": 1794380525.559297, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.86%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380525.5593002, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.33%", + "MemUsage": "279.9MiB / 512MiB" + }, + { + "at": 1794380525.559304, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.12%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380525.559308, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.07%", + "MemUsage": "119.7MiB / 192MiB" + }, + { + "at": 1794380525.5593112, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "1.75%", + "MemUsage": "84.83MiB / 256MiB" + }, + { + "at": 1794380528.9313412, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.94%", + "MemUsage": "30.29MiB / 512MiB" + }, + { + "at": 1794380528.931348, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.62%", + "MemUsage": "34.31MiB / 512MiB" + }, + { + "at": 1794380528.9313521, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.51%", + "MemUsage": "7.371MiB / 512MiB" + }, + { + "at": 1794380528.931356, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.64%", + "MemUsage": "66.31MiB / 512MiB" + }, + { + "at": 1794380528.93136, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.48%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380528.931364, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.71%", + "MemUsage": "158.5MiB / 192MiB" + }, + { + "at": 1794380528.9313679, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.35%", + "MemUsage": "39.2MiB / 512MiB" + }, + { + "at": 1794380528.931371, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.47%", + "MemUsage": "5.742MiB / 384MiB" + }, + { + "at": 1794380528.931375, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "0.87%", + "MemUsage": "280MiB / 512MiB" + }, + { + "at": 1794380528.9313788, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.49%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380528.931382, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "4.19%", + "MemUsage": "119.7MiB / 192MiB" + }, + { + "at": 1794380528.931386, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.56%", + "MemUsage": "84.83MiB / 256MiB" + } + ] + }, + { + "configuration": "balanced", + "repetition": 1, + "disabled": false, + "sampling_ratio": 0.5, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.884, + "successful_rps": 8.062, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 10.423, + "p95": 30.254, + "p99": 52.116 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 75451.0, + "failed_spans": 10491.0, + "accepted_spans": 147335.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1586248.0, + "loki_ingested_bytes": 3008394.0, + "prometheus_samples": 1924241.0 + }, + "counter_after": { + "exported_spans": 76665.0, + "failed_spans": 10491.0, + "accepted_spans": 148549.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1781849.0, + "loki_ingested_bytes": 3101672.0, + "prometheus_samples": 1946634.0 + }, + "counter_delta": { + "exported_spans": 1214.0, + "failed_spans": 0.0, + "accepted_spans": 1214.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 195601.0, + "loki_ingested_bytes": 93278.0, + "prometheus_samples": 22393.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 54 + }, + "write": { + "sent": 18, + "found": 10 + }, + "upload": { + "sent": 6, + "found": 4 + }, + "client_error": { + "sent": 6, + "found": 4 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 4 + }, + "resources": [ + { + "at": 1794380564.844873, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.29%", + "MemUsage": "6.441MiB / 512MiB" + }, + { + "at": 1794380564.8448808, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.52%", + "MemUsage": "27.99MiB / 512MiB" + }, + { + "at": 1794380564.844885, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.87%", + "MemUsage": "26.95MiB / 512MiB" + }, + { + "at": 1794380564.8448892, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.04%", + "MemUsage": "208.6MiB / 512MiB" + }, + { + "at": 1794380564.844893, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.36%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380564.844897, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.55%", + "MemUsage": "158.6MiB / 192MiB" + }, + { + "at": 1794380564.8449001, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "5.59%", + "MemUsage": "36.83MiB / 512MiB" + }, + { + "at": 1794380564.844904, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "3.19%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380564.844907, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.75%", + "MemUsage": "280.3MiB / 512MiB" + }, + { + "at": 1794380564.844911, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.56%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380564.844914, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "3.24%", + "MemUsage": "119.7MiB / 192MiB" + }, + { + "at": 1794380564.844918, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.78%", + "MemUsage": "86.1MiB / 256MiB" + }, + { + "at": 1794380568.019284, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "8.17%", + "MemUsage": "6.566MiB / 512MiB" + }, + { + "at": 1794380568.019295, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.61%", + "MemUsage": "29.29MiB / 512MiB" + }, + { + "at": 1794380568.0193, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.34%", + "MemUsage": "27.83MiB / 512MiB" + }, + { + "at": 1794380568.0193028, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.26%", + "MemUsage": "208.7MiB / 512MiB" + }, + { + "at": 1794380568.019307, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.86%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380568.0193112, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.54%", + "MemUsage": "158.6MiB / 192MiB" + }, + { + "at": 1794380568.0193138, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.31%", + "MemUsage": "36.94MiB / 512MiB" + }, + { + "at": 1794380568.019318, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.34%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380568.0193212, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "7.18%", + "MemUsage": "280.5MiB / 512MiB" + }, + { + "at": 1794380568.019325, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.83%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380568.019328, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "4.77%", + "MemUsage": "121.2MiB / 192MiB" + }, + { + "at": 1794380568.0193322, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "6.98%", + "MemUsage": "79.73MiB / 256MiB" + }, + { + "at": 1794380571.252476, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "2.86%", + "MemUsage": "6.457MiB / 512MiB" + }, + { + "at": 1794380571.2524831, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.13%", + "MemUsage": "31.81MiB / 512MiB" + }, + { + "at": 1794380571.252487, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.53%", + "MemUsage": "29.19MiB / 512MiB" + }, + { + "at": 1794380571.252491, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "3.67%", + "MemUsage": "208.7MiB / 512MiB" + }, + { + "at": 1794380571.252495, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.66%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380571.2524989, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "2.01%", + "MemUsage": "158.6MiB / 192MiB" + }, + { + "at": 1794380571.252502, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "6.32%", + "MemUsage": "39MiB / 512MiB" + }, + { + "at": 1794380571.252506, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.00%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380571.25251, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.31%", + "MemUsage": "280.5MiB / 512MiB" + }, + { + "at": 1794380571.252513, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.10%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380571.2525172, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.56%", + "MemUsage": "122.2MiB / 192MiB" + }, + { + "at": 1794380571.2525198, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "2.81%", + "MemUsage": "79.73MiB / 256MiB" + }, + { + "at": 1794380574.496011, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "2.28%", + "MemUsage": "6.914MiB / 512MiB" + }, + { + "at": 1794380574.496019, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.36%", + "MemUsage": "33.5MiB / 512MiB" + }, + { + "at": 1794380574.496023, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "4.05%", + "MemUsage": "29.75MiB / 512MiB" + }, + { + "at": 1794380574.496027, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.40%", + "MemUsage": "208.7MiB / 512MiB" + }, + { + "at": 1794380574.496031, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.95%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380574.4960341, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.32%", + "MemUsage": "158.6MiB / 192MiB" + }, + { + "at": 1794380574.496038, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.96%", + "MemUsage": "39.43MiB / 512MiB" + }, + { + "at": 1794380574.496042, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.64%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380574.496045, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "0.75%", + "MemUsage": "280.6MiB / 512MiB" + }, + { + "at": 1794380574.4960482, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.41%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380574.496052, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.04%", + "MemUsage": "123.9MiB / 192MiB" + }, + { + "at": 1794380574.4960551, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.06%", + "MemUsage": "79.74MiB / 256MiB" + }, + { + "at": 1794380577.744085, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.52%", + "MemUsage": "6.66MiB / 512MiB" + }, + { + "at": 1794380577.744092, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.74%", + "MemUsage": "32.77MiB / 512MiB" + }, + { + "at": 1794380577.744097, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.04%", + "MemUsage": "30.9MiB / 512MiB" + }, + { + "at": 1794380577.7441, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.78%", + "MemUsage": "209.3MiB / 512MiB" + }, + { + "at": 1794380577.744104, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.62%", + "MemUsage": "10.3MiB / 64MiB" + }, + { + "at": 1794380577.744108, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.30%", + "MemUsage": "158.6MiB / 192MiB" + }, + { + "at": 1794380577.744112, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.67%", + "MemUsage": "39.66MiB / 512MiB" + }, + { + "at": 1794380577.7441149, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.22%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380577.744119, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "3.18%", + "MemUsage": "280.8MiB / 512MiB" + }, + { + "at": 1794380577.744122, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.79%", + "MemUsage": "73.35MiB / 192MiB" + }, + { + "at": 1794380577.7441258, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "7.10%", + "MemUsage": "104.2MiB / 192MiB" + }, + { + "at": 1794380577.744129, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.71%", + "MemUsage": "79.78MiB / 256MiB" + } + ] + }, + { + "configuration": "balanced", + "repetition": 2, + "disabled": false, + "sampling_ratio": 0.5, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.91, + "successful_rps": 8.048, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 11.346, + "p95": 37.555, + "p99": 72.706 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 77307.0, + "failed_spans": 10491.0, + "accepted_spans": 149191.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 1909599.0, + "loki_ingested_bytes": 3129831.0, + "prometheus_samples": 1969126.0 + }, + "counter_after": { + "exported_spans": 78495.0, + "failed_spans": 10491.0, + "accepted_spans": 150379.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2076054.0, + "loki_ingested_bytes": 3221531.0, + "prometheus_samples": 1991630.0 + }, + "counter_delta": { + "exported_spans": 1188.0, + "failed_spans": 0.0, + "accepted_spans": 1188.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 166455.0, + "loki_ingested_bytes": 91700.0, + "prometheus_samples": 22504.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 41 + }, + "write": { + "sent": 18, + "found": 7 + }, + "upload": { + "sent": 6, + "found": 2 + }, + "client_error": { + "sent": 6, + "found": 3 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 4 + }, + "resources": [ + { + "at": 1794380614.191704, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.92%", + "MemUsage": "6.285MiB / 512MiB" + }, + { + "at": 1794380614.1917162, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.58%", + "MemUsage": "7.398MiB / 512MiB" + }, + { + "at": 1794380614.191726, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.42%", + "MemUsage": "44.97MiB / 512MiB" + }, + { + "at": 1794380614.191731, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.28%", + "MemUsage": "182.9MiB / 512MiB" + }, + { + "at": 1794380614.1917372, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.26%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380614.191742, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.50%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380614.1917472, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.93%", + "MemUsage": "38.86MiB / 512MiB" + }, + { + "at": 1794380614.191752, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.63%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380614.191757, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "1.11%", + "MemUsage": "281MiB / 512MiB" + }, + { + "at": 1794380614.191762, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.99%", + "MemUsage": "73.36MiB / 192MiB" + }, + { + "at": 1794380614.191767, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.75%", + "MemUsage": "104.5MiB / 192MiB" + }, + { + "at": 1794380614.1917708, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.07%", + "MemUsage": "80.75MiB / 256MiB" + }, + { + "at": 1794380617.4892359, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "6.02%", + "MemUsage": "6.629MiB / 512MiB" + }, + { + "at": 1794380617.4892428, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.62%", + "MemUsage": "7.609MiB / 512MiB" + }, + { + "at": 1794380617.489248, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.09%", + "MemUsage": "45.03MiB / 512MiB" + }, + { + "at": 1794380617.489252, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.44%", + "MemUsage": "182.9MiB / 512MiB" + }, + { + "at": 1794380617.489256, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.96%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380617.48926, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.40%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380617.489263, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.30%", + "MemUsage": "39.08MiB / 512MiB" + }, + { + "at": 1794380617.4892669, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.28%", + "MemUsage": "5.746MiB / 384MiB" + }, + { + "at": 1794380617.489271, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "12.66%", + "MemUsage": "281.2MiB / 512MiB" + }, + { + "at": 1794380617.489275, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.20%", + "MemUsage": "73.36MiB / 192MiB" + }, + { + "at": 1794380617.4892778, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "0.85%", + "MemUsage": "104.5MiB / 192MiB" + }, + { + "at": 1794380617.4892821, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.56%", + "MemUsage": "80.76MiB / 256MiB" + }, + { + "at": 1794380620.818201, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.57%", + "MemUsage": "6.391MiB / 512MiB" + }, + { + "at": 1794380620.8182101, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.73%", + "MemUsage": "7.637MiB / 512MiB" + }, + { + "at": 1794380620.818214, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.23%", + "MemUsage": "45.12MiB / 512MiB" + }, + { + "at": 1794380620.818218, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "2.37%", + "MemUsage": "182.9MiB / 512MiB" + }, + { + "at": 1794380620.818222, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.08%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380620.818225, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "2.66%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380620.818229, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.62%", + "MemUsage": "39.09MiB / 512MiB" + }, + { + "at": 1794380620.818233, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "3.60%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380620.8182359, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "4.76%", + "MemUsage": "282.7MiB / 512MiB" + }, + { + "at": 1794380620.81824, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.80%", + "MemUsage": "73.36MiB / 192MiB" + }, + { + "at": 1794380620.818243, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.41%", + "MemUsage": "104.5MiB / 192MiB" + }, + { + "at": 1794380620.818246, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "2.90%", + "MemUsage": "80.76MiB / 256MiB" + }, + { + "at": 1794380624.05145, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.10%", + "MemUsage": "6.812MiB / 512MiB" + }, + { + "at": 1794380624.051457, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.29%", + "MemUsage": "7.383MiB / 512MiB" + }, + { + "at": 1794380624.0514612, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.07%", + "MemUsage": "45.14MiB / 512MiB" + }, + { + "at": 1794380624.051465, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.72%", + "MemUsage": "182.9MiB / 512MiB" + }, + { + "at": 1794380624.051469, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.42%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380624.0514739, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.49%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380624.051477, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.62%", + "MemUsage": "39.42MiB / 512MiB" + }, + { + "at": 1794380624.051481, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.85%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380624.0514848, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "1.10%", + "MemUsage": "281.3MiB / 512MiB" + }, + { + "at": 1794380624.051489, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.07%", + "MemUsage": "73.48MiB / 192MiB" + }, + { + "at": 1794380624.0514932, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.76%", + "MemUsage": "104.5MiB / 192MiB" + }, + { + "at": 1794380624.051497, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.05%", + "MemUsage": "80.76MiB / 256MiB" + }, + { + "at": 1794380627.3197892, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.81%", + "MemUsage": "6.906MiB / 512MiB" + }, + { + "at": 1794380627.3198028, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.12%", + "MemUsage": "7.324MiB / 512MiB" + }, + { + "at": 1794380627.319807, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.50%", + "MemUsage": "45.16MiB / 512MiB" + }, + { + "at": 1794380627.319811, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.59%", + "MemUsage": "183.1MiB / 512MiB" + }, + { + "at": 1794380627.319815, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.82%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380627.319819, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.64%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380627.319824, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "5.22%", + "MemUsage": "39.73MiB / 512MiB" + }, + { + "at": 1794380627.319828, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "4.11%", + "MemUsage": "5.738MiB / 384MiB" + }, + { + "at": 1794380627.3198311, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "10.79%", + "MemUsage": "281.3MiB / 512MiB" + }, + { + "at": 1794380627.319835, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.65%", + "MemUsage": "73.48MiB / 192MiB" + }, + { + "at": 1794380627.319838, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.66%", + "MemUsage": "104.5MiB / 192MiB" + }, + { + "at": 1794380627.319841, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.67%", + "MemUsage": "80.76MiB / 256MiB" + } + ] + }, + { + "configuration": "ten_percent", + "repetition": 2, + "disabled": false, + "sampling_ratio": 0.1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.886, + "successful_rps": 8.061, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 12.281, + "p95": 42.42, + "p99": 63.216 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 78801.0, + "failed_spans": 10491.0, + "accepted_spans": 150690.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2140822.0, + "loki_ingested_bytes": 3248033.0, + "prometheus_samples": 2014191.0 + }, + "counter_after": { + "exported_spans": 79016.0, + "failed_spans": 10491.0, + "accepted_spans": 150902.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2183730.0, + "loki_ingested_bytes": 3340743.0, + "prometheus_samples": 2036713.0 + }, + "counter_delta": { + "exported_spans": 215.0, + "failed_spans": 0.0, + "accepted_spans": 212.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 42908.0, + "loki_ingested_bytes": 92710.0, + "prometheus_samples": 22522.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 12 + }, + "write": { + "sent": 18, + "found": 4 + }, + "upload": { + "sent": 6, + "found": 0 + }, + "client_error": { + "sent": 6, + "found": 1 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 1 + }, + "resources": [ + { + "at": 1794380664.504447, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.23%", + "MemUsage": "6.398MiB / 512MiB" + }, + { + "at": 1794380664.504456, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.28%", + "MemUsage": "7.145MiB / 512MiB" + }, + { + "at": 1794380664.504461, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.77%", + "MemUsage": "45.11MiB / 512MiB" + }, + { + "at": 1794380664.5044658, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.93%", + "MemUsage": "69.79MiB / 512MiB" + }, + { + "at": 1794380664.504471, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.40%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380664.5044749, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.49%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380664.504479, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.23%", + "MemUsage": "36.45MiB / 512MiB" + }, + { + "at": 1794380664.504483, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.69%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380664.5044868, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "6.42%", + "MemUsage": "282.6MiB / 512MiB" + }, + { + "at": 1794380664.504491, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.46%", + "MemUsage": "73.05MiB / 192MiB" + }, + { + "at": 1794380664.5044942, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.10%", + "MemUsage": "104.7MiB / 192MiB" + }, + { + "at": 1794380664.504498, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.05%", + "MemUsage": "80.78MiB / 256MiB" + }, + { + "at": 1794380667.7384489, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.24%", + "MemUsage": "6.629MiB / 512MiB" + }, + { + "at": 1794380667.7384589, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.73%", + "MemUsage": "7.262MiB / 512MiB" + }, + { + "at": 1794380667.738465, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.33%", + "MemUsage": "45.16MiB / 512MiB" + }, + { + "at": 1794380667.7384691, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.73%", + "MemUsage": "70.23MiB / 512MiB" + }, + { + "at": 1794380667.738473, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "2.04%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380667.738477, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.64%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380667.7384808, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.68%", + "MemUsage": "36.52MiB / 512MiB" + }, + { + "at": 1794380667.738485, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.65%", + "MemUsage": "5.734MiB / 384MiB" + }, + { + "at": 1794380667.7384892, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "5.29%", + "MemUsage": "282.7MiB / 512MiB" + }, + { + "at": 1794380667.738493, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "3.08%", + "MemUsage": "73.05MiB / 192MiB" + }, + { + "at": 1794380667.738497, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.07%", + "MemUsage": "104.7MiB / 192MiB" + }, + { + "at": 1794380667.7385018, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "1.29%", + "MemUsage": "80.79MiB / 256MiB" + }, + { + "at": 1794380670.980447, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "0.94%", + "MemUsage": "6.629MiB / 512MiB" + }, + { + "at": 1794380670.980454, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.58%", + "MemUsage": "7.348MiB / 512MiB" + }, + { + "at": 1794380670.980458, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.32%", + "MemUsage": "45.23MiB / 512MiB" + }, + { + "at": 1794380670.980462, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "30.96%", + "MemUsage": "199.8MiB / 512MiB" + }, + { + "at": 1794380670.980466, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.78%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380670.98047, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "6.30%", + "MemUsage": "158.9MiB / 192MiB" + }, + { + "at": 1794380670.980474, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.97%", + "MemUsage": "37.14MiB / 512MiB" + }, + { + "at": 1794380670.9804769, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.14%", + "MemUsage": "5.742MiB / 384MiB" + }, + { + "at": 1794380670.980481, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "4.70%", + "MemUsage": "282.7MiB / 512MiB" + }, + { + "at": 1794380670.980484, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.57%", + "MemUsage": "73.05MiB / 192MiB" + }, + { + "at": 1794380670.9804878, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.55%", + "MemUsage": "104.7MiB / 192MiB" + }, + { + "at": 1794380670.980492, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "1.18%", + "MemUsage": "80.78MiB / 256MiB" + }, + { + "at": 1794380674.3469691, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.41%", + "MemUsage": "6.918MiB / 512MiB" + }, + { + "at": 1794380674.346976, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.37%", + "MemUsage": "7.422MiB / 512MiB" + }, + { + "at": 1794380674.346981, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.76%", + "MemUsage": "45.25MiB / 512MiB" + }, + { + "at": 1794380674.3469849, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.41%", + "MemUsage": "251.1MiB / 512MiB" + }, + { + "at": 1794380674.3469892, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.60%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380674.346994, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.65%", + "MemUsage": "159.4MiB / 192MiB" + }, + { + "at": 1794380674.346998, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.91%", + "MemUsage": "38.19MiB / 512MiB" + }, + { + "at": 1794380674.347001, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "3.28%", + "MemUsage": "5.82MiB / 384MiB" + }, + { + "at": 1794380674.347005, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "1.40%", + "MemUsage": "281.1MiB / 512MiB" + }, + { + "at": 1794380674.347008, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.63%", + "MemUsage": "73.74MiB / 192MiB" + }, + { + "at": 1794380674.347012, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.63%", + "MemUsage": "104.9MiB / 192MiB" + }, + { + "at": 1794380674.347015, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.11%", + "MemUsage": "80.75MiB / 256MiB" + }, + { + "at": 1794380677.620321, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.09%", + "MemUsage": "7.016MiB / 512MiB" + }, + { + "at": 1794380677.620333, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "1.78%", + "MemUsage": "7.77MiB / 512MiB" + }, + { + "at": 1794380677.620338, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.88%", + "MemUsage": "45.28MiB / 512MiB" + }, + { + "at": 1794380677.620342, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.40%", + "MemUsage": "252.2MiB / 512MiB" + }, + { + "at": 1794380677.6203468, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.78%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380677.620352, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.49%", + "MemUsage": "159.4MiB / 192MiB" + }, + { + "at": 1794380677.6203551, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.27%", + "MemUsage": "38.35MiB / 512MiB" + }, + { + "at": 1794380677.62036, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.40%", + "MemUsage": "5.82MiB / 384MiB" + }, + { + "at": 1794380677.620364, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "4.42%", + "MemUsage": "281.1MiB / 512MiB" + }, + { + "at": 1794380677.620367, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.11%", + "MemUsage": "73.74MiB / 192MiB" + }, + { + "at": 1794380677.6203709, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.37%", + "MemUsage": "104.9MiB / 192MiB" + }, + { + "at": 1794380677.6203752, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.68%", + "MemUsage": "80.76MiB / 256MiB" + } + ] + }, + { + "configuration": "full", + "repetition": 2, + "disabled": false, + "sampling_ratio": 1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.91, + "successful_rps": 8.048, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 12.379, + "p95": 31.356, + "p99": 45.834 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 79996.0, + "failed_spans": 10491.0, + "accepted_spans": 151943.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2321594.0, + "loki_ingested_bytes": 3356097.0, + "prometheus_samples": 2059308.0 + }, + "counter_after": { + "exported_spans": 82280.0, + "failed_spans": 10491.0, + "accepted_spans": 154226.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2675465.0, + "loki_ingested_bytes": 3460284.0, + "prometheus_samples": 2081852.0 + }, + "counter_delta": { + "exported_spans": 2284.0, + "failed_spans": 0.0, + "accepted_spans": 2283.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 353871.0, + "loki_ingested_bytes": 104187.0, + "prometheus_samples": 22544.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 90 + }, + "write": { + "sent": 18, + "found": 18 + }, + "upload": { + "sent": 6, + "found": 6 + }, + "client_error": { + "sent": 6, + "found": 6 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 6 + }, + "resources": [ + { + "at": 1794380714.788736, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.73%", + "MemUsage": "27.2MiB / 512MiB" + }, + { + "at": 1794380714.788745, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.66%", + "MemUsage": "28.13MiB / 512MiB" + }, + { + "at": 1794380714.7887518, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.31%", + "MemUsage": "7.215MiB / 512MiB" + }, + { + "at": 1794380714.788757, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.98%", + "MemUsage": "305.8MiB / 512MiB" + }, + { + "at": 1794380714.7887628, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.19%", + "MemUsage": "10.31MiB / 64MiB" + }, + { + "at": 1794380714.788768, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.34%", + "MemUsage": "105.5MiB / 192MiB" + }, + { + "at": 1794380714.788773, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "6.03%", + "MemUsage": "33.9MiB / 512MiB" + }, + { + "at": 1794380714.788779, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.75%", + "MemUsage": "5.055MiB / 384MiB" + }, + { + "at": 1794380714.788784, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "6.67%", + "MemUsage": "282.3MiB / 512MiB" + }, + { + "at": 1794380714.788789, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.31%", + "MemUsage": "75.46MiB / 192MiB" + }, + { + "at": 1794380714.788794, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.89%", + "MemUsage": "105.8MiB / 192MiB" + }, + { + "at": 1794380714.788799, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.37%", + "MemUsage": "83.04MiB / 256MiB" + }, + { + "at": 1794380718.03025, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.16%", + "MemUsage": "28.31MiB / 512MiB" + }, + { + "at": 1794380718.030262, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.97%", + "MemUsage": "30.61MiB / 512MiB" + }, + { + "at": 1794380718.030268, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.06%", + "MemUsage": "7.633MiB / 512MiB" + }, + { + "at": 1794380718.030273, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.52%", + "MemUsage": "305.8MiB / 512MiB" + }, + { + "at": 1794380718.030278, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.04%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380718.030284, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.52%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380718.03029, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.33%", + "MemUsage": "34.06MiB / 512MiB" + }, + { + "at": 1794380718.0302951, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.42%", + "MemUsage": "5.055MiB / 384MiB" + }, + { + "at": 1794380718.0303, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.79%", + "MemUsage": "282.5MiB / 512MiB" + }, + { + "at": 1794380718.030305, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "2.02%", + "MemUsage": "75.46MiB / 192MiB" + }, + { + "at": 1794380718.03031, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "3.02%", + "MemUsage": "105.8MiB / 192MiB" + }, + { + "at": 1794380718.030316, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.45%", + "MemUsage": "83.07MiB / 256MiB" + }, + { + "at": 1794380721.214758, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.81%", + "MemUsage": "29.39MiB / 512MiB" + }, + { + "at": 1794380721.214765, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.80%", + "MemUsage": "32.41MiB / 512MiB" + }, + { + "at": 1794380721.214769, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.34%", + "MemUsage": "7.281MiB / 512MiB" + }, + { + "at": 1794380721.214773, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "3.01%", + "MemUsage": "305.9MiB / 512MiB" + }, + { + "at": 1794380721.214777, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "0.87%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380721.214781, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.96%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380721.214784, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.88%", + "MemUsage": "34.1MiB / 512MiB" + }, + { + "at": 1794380721.214787, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.83%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380721.2147908, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "15.86%", + "MemUsage": "282.6MiB / 512MiB" + }, + { + "at": 1794380721.214795, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.64%", + "MemUsage": "76.02MiB / 192MiB" + }, + { + "at": 1794380721.214798, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.60%", + "MemUsage": "95.17MiB / 192MiB" + }, + { + "at": 1794380721.2148008, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "2.31%", + "MemUsage": "83.04MiB / 256MiB" + }, + { + "at": 1794380724.391743, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.91%", + "MemUsage": "30.35MiB / 512MiB" + }, + { + "at": 1794380724.391752, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.99%", + "MemUsage": "34.03MiB / 512MiB" + }, + { + "at": 1794380724.3917592, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "2.94%", + "MemUsage": "7.578MiB / 512MiB" + }, + { + "at": 1794380724.391764, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "2.06%", + "MemUsage": "305.9MiB / 512MiB" + }, + { + "at": 1794380724.3917701, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.23%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380724.391776, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.63%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380724.391781, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "2.90%", + "MemUsage": "36.59MiB / 512MiB" + }, + { + "at": 1794380724.391787, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.75%", + "MemUsage": "5.055MiB / 384MiB" + }, + { + "at": 1794380724.391792, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "5.31%", + "MemUsage": "282.7MiB / 512MiB" + }, + { + "at": 1794380724.391798, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.45%", + "MemUsage": "76.07MiB / 192MiB" + }, + { + "at": 1794380724.3918028, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "3.37%", + "MemUsage": "95.17MiB / 192MiB" + }, + { + "at": 1794380724.391808, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.06%", + "MemUsage": "83.03MiB / 256MiB" + }, + { + "at": 1794380727.594573, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "4.43%", + "MemUsage": "31.32MiB / 512MiB" + }, + { + "at": 1794380727.59458, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.31%", + "MemUsage": "35.1MiB / 512MiB" + }, + { + "at": 1794380727.5945852, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.15%", + "MemUsage": "7.574MiB / 512MiB" + }, + { + "at": 1794380727.5945878, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.92%", + "MemUsage": "305.9MiB / 512MiB" + }, + { + "at": 1794380727.594592, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.22%", + "MemUsage": "10.32MiB / 64MiB" + }, + { + "at": 1794380727.5945961, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.44%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380727.5946, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.88%", + "MemUsage": "36.86MiB / 512MiB" + }, + { + "at": 1794380727.594603, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.94%", + "MemUsage": "5.055MiB / 384MiB" + }, + { + "at": 1794380727.594607, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "6.66%", + "MemUsage": "283MiB / 512MiB" + }, + { + "at": 1794380727.59461, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "1.69%", + "MemUsage": "65.42MiB / 192MiB" + }, + { + "at": 1794380727.594614, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.89%", + "MemUsage": "95.17MiB / 192MiB" + }, + { + "at": 1794380727.5946171, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.54%", + "MemUsage": "83MiB / 256MiB" + } + ] + }, + { + "configuration": "off", + "repetition": 2, + "disabled": true, + "sampling_ratio": 1, + "requests": 120, + "target_rps": 8, + "concurrency_limit": 4, + "elapsed_seconds": 14.906, + "successful_rps": 8.05, + "unexpected_errors": 0, + "error_types": [], + "all_request_latency_ms": { + "p50": 10.728, + "p95": 30.012, + "p99": 63.853 + }, + "ready_uploads": 6, + "counter_before": { + "exported_spans": 82850.0, + "failed_spans": 10491.0, + "accepted_spans": 154734.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2781663.0, + "loki_ingested_bytes": 3467038.0, + "prometheus_samples": 2104387.0 + }, + "counter_after": { + "exported_spans": 82850.0, + "failed_spans": 10491.0, + "accepted_spans": 154734.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 2781663.0, + "loki_ingested_bytes": 3467038.0, + "prometheus_samples": 2125852.0 + }, + "counter_delta": { + "exported_spans": 0.0, + "failed_spans": 0.0, + "accepted_spans": 0.0, + "refused_spans": 0.0, + "tempo_ingested_bytes": 0.0, + "loki_ingested_bytes": 0.0, + "prometheus_samples": 21465.0 + }, + "trace_retention": { + "read": { + "sent": 90, + "found": 0 + }, + "write": { + "sent": 18, + "found": 0 + }, + "upload": { + "sent": 6, + "found": 0 + }, + "client_error": { + "sent": 6, + "found": 0 + } + }, + "slow_request_trace_retention": { + "definition": "slowest 5 percent of this trial by measured client latency", + "sent": 6, + "found": 0 + }, + "resources": [ + { + "at": 1794380763.264864, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.28%", + "MemUsage": "5.348MiB / 512MiB" + }, + { + "at": 1794380763.264875, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "3.70%", + "MemUsage": "25.38MiB / 512MiB" + }, + { + "at": 1794380763.264879, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "3.01%", + "MemUsage": "25.77MiB / 512MiB" + }, + { + "at": 1794380763.264884, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.91%", + "MemUsage": "214MiB / 512MiB" + }, + { + "at": 1794380763.264889, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.66%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380763.264894, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.65%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380763.264899, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.50%", + "MemUsage": "37.45MiB / 512MiB" + }, + { + "at": 1794380763.2649028, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.97%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380763.2649071, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "1.47%", + "MemUsage": "283.4MiB / 512MiB" + }, + { + "at": 1794380763.264912, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.22%", + "MemUsage": "65.41MiB / 192MiB" + }, + { + "at": 1794380763.264917, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "6.56%", + "MemUsage": "95.17MiB / 192MiB" + }, + { + "at": 1794380763.2649212, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "2.73%", + "MemUsage": "83.02MiB / 256MiB" + }, + { + "at": 1794380766.5247118, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "7.62%", + "MemUsage": "5.707MiB / 512MiB" + }, + { + "at": 1794380766.524723, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.06%", + "MemUsage": "25.96MiB / 512MiB" + }, + { + "at": 1794380766.524728, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.61%", + "MemUsage": "27.59MiB / 512MiB" + }, + { + "at": 1794380766.5247328, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.57%", + "MemUsage": "214MiB / 512MiB" + }, + { + "at": 1794380766.5247371, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.24%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380766.524742, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.39%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380766.524747, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "6.87%", + "MemUsage": "37.98MiB / 512MiB" + }, + { + "at": 1794380766.5247512, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.50%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380766.524755, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "7.45%", + "MemUsage": "283.4MiB / 512MiB" + }, + { + "at": 1794380766.52476, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.11%", + "MemUsage": "65.41MiB / 192MiB" + }, + { + "at": 1794380766.5247638, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "1.32%", + "MemUsage": "95.17MiB / 192MiB" + }, + { + "at": 1794380766.524768, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.00%", + "MemUsage": "83.02MiB / 256MiB" + }, + { + "at": 1794380769.715033, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.39%", + "MemUsage": "5.773MiB / 512MiB" + }, + { + "at": 1794380769.715041, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.13%", + "MemUsage": "26.4MiB / 512MiB" + }, + { + "at": 1794380769.7150452, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "2.61%", + "MemUsage": "28.71MiB / 512MiB" + }, + { + "at": 1794380769.7150478, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.83%", + "MemUsage": "214MiB / 512MiB" + }, + { + "at": 1794380769.7150521, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.38%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380769.7150562, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.58%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380769.71506, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.23%", + "MemUsage": "38.03MiB / 512MiB" + }, + { + "at": 1794380769.715063, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "1.58%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380769.715067, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "9.67%", + "MemUsage": "283.5MiB / 512MiB" + }, + { + "at": 1794380769.71507, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.08%", + "MemUsage": "65.41MiB / 192MiB" + }, + { + "at": 1794380769.715073, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "3.53%", + "MemUsage": "95.18MiB / 192MiB" + }, + { + "at": 1794380769.7150772, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.36%", + "MemUsage": "83.02MiB / 256MiB" + }, + { + "at": 1794380773.000909, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.23%", + "MemUsage": "5.758MiB / 512MiB" + }, + { + "at": 1794380773.000919, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.08%", + "MemUsage": "26.97MiB / 512MiB" + }, + { + "at": 1794380773.0009248, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.74%", + "MemUsage": "29.85MiB / 512MiB" + }, + { + "at": 1794380773.00093, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "0.56%", + "MemUsage": "214MiB / 512MiB" + }, + { + "at": 1794380773.000937, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.19%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380773.0009432, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.50%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380773.000948, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "4.93%", + "MemUsage": "38.11MiB / 512MiB" + }, + { + "at": 1794380773.0009532, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.67%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380773.000958, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.07%", + "MemUsage": "283.5MiB / 512MiB" + }, + { + "at": 1794380773.000963, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.22%", + "MemUsage": "65.41MiB / 192MiB" + }, + { + "at": 1794380773.000968, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "4.03%", + "MemUsage": "95.18MiB / 192MiB" + }, + { + "at": 1794380773.000973, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "0.84%", + "MemUsage": "83.02MiB / 256MiB" + }, + { + "at": 1794380776.255959, + "Name": "mtusi2026lab6-worker-1", + "CPUPerc": "1.37%", + "MemUsage": "5.816MiB / 512MiB" + }, + { + "at": 1794380776.2559738, + "Name": "mtusi2026lab6-api-1", + "CPUPerc": "2.29%", + "MemUsage": "27.38MiB / 512MiB" + }, + { + "at": 1794380776.255981, + "Name": "mtusi2026lab6-api-2", + "CPUPerc": "1.90%", + "MemUsage": "30.98MiB / 512MiB" + }, + { + "at": 1794380776.255987, + "Name": "mtusi2026lab6-tempo-1", + "CPUPerc": "1.27%", + "MemUsage": "214MiB / 512MiB" + }, + { + "at": 1794380776.2559931, + "Name": "mtusi2026lab6-gateway-1", + "CPUPerc": "1.50%", + "MemUsage": "10.33MiB / 64MiB" + }, + { + "at": 1794380776.256001, + "Name": "mtusi2026lab6-grafana-1", + "CPUPerc": "0.63%", + "MemUsage": "105.6MiB / 192MiB" + }, + { + "at": 1794380776.256006, + "Name": "mtusi2026lab6-postgres-1", + "CPUPerc": "3.23%", + "MemUsage": "38.35MiB / 512MiB" + }, + { + "at": 1794380776.256012, + "Name": "mtusi2026lab6-redis-1", + "CPUPerc": "2.59%", + "MemUsage": "5.059MiB / 384MiB" + }, + { + "at": 1794380776.256018, + "Name": "mtusi2026lab6-minio-1", + "CPUPerc": "2.27%", + "MemUsage": "283.5MiB / 512MiB" + }, + { + "at": 1794380776.256023, + "Name": "mtusi2026lab6-collector-1", + "CPUPerc": "0.15%", + "MemUsage": "65.41MiB / 192MiB" + }, + { + "at": 1794380776.256029, + "Name": "mtusi2026lab6-loki-1", + "CPUPerc": "2.44%", + "MemUsage": "95.18MiB / 192MiB" + }, + { + "at": 1794380776.256035, + "Name": "mtusi2026lab6-prometheus-1", + "CPUPerc": "3.60%", + "MemUsage": "83.02MiB / 256MiB" + } + ] + } + ], + "status": "PASS", + "finished_at": "2026-11-11T07:06:42.525464+00:00" +} diff --git a/evidence/lab6-runtime-state.json b/evidence/lab6-runtime-state.json new file mode 100644 index 0000000..30ca92b --- /dev/null +++ b/evidence/lab6-runtime-state.json @@ -0,0 +1,145 @@ +{ + "at": "2026-11-11T07:17:02.023361+00:00", + "containers": [ + { + "name": "/mtusi2026lab6-api-1", + "image": "mtusi-messenger:lab6", + "status": "running", + "started_at": "2026-11-11T07:06:34.329528303Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000 + }, + { + "name": "/mtusi2026lab6-api-2", + "image": "mtusi-messenger:lab6", + "status": "running", + "started_at": "2026-11-11T07:11:20.392887139Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000 + }, + { + "name": "/mtusi2026lab6-collector-1", + "image": "otel/opentelemetry-collector-contrib:0.137.0", + "status": "running", + "started_at": "2026-11-10T19:32:17.968197475Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 201326592, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-gateway-1", + "image": "haproxy:3.2.4-alpine", + "status": "running", + "started_at": "2026-11-10T19:32:18.286616663Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 67108864, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-grafana-1", + "image": "grafana/grafana:12.2.0", + "status": "running", + "started_at": "2026-11-10T19:32:18.090061627Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 201326592, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-loki-1", + "image": "grafana/loki:3.5.5", + "status": "running", + "started_at": "2026-11-10T19:32:18.51793344Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 201326592, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-migrations-1", + "image": "mtusi-migrations:lab6", + "status": "exited", + "started_at": "2026-11-10T19:38:06.556403206Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000 + }, + { + "name": "/mtusi2026lab6-minio-1", + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "status": "running", + "started_at": "2026-11-11T06:56:16.504084898Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000 + }, + { + "name": "/mtusi2026lab6-postgres-1", + "image": "postgres:17.6-alpine3.22", + "status": "running", + "started_at": "2026-11-10T19:32:18.144322815Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000 + }, + { + "name": "/mtusi2026lab6-prometheus-1", + "image": "prom/prometheus:v3.6.0", + "status": "running", + "started_at": "2026-11-10T19:32:18.132436Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 268435456, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-redis-1", + "image": "redis:7.4.5-alpine3.21", + "status": "running", + "started_at": "2026-11-10T19:32:17.96910494Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 402653184, + "nano_cpus": 500000000 + }, + { + "name": "/mtusi2026lab6-s3_bootstrap-1", + "image": "mtusi-mc:RELEASE.2025-08-13T08-35-41Z", + "status": "exited", + "started_at": "2026-11-10T19:38:06.557042047Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 500000000 + }, + { + "name": "/mtusi2026lab6-tempo-1", + "image": "grafana/tempo:2.8.2", + "status": "running", + "started_at": "2026-11-11T07:09:04.705778982Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 0 + }, + { + "name": "/mtusi2026lab6-worker-1", + "image": "mtusi-messenger:lab6", + "status": "running", + "started_at": "2026-11-11T07:06:34.208498895Z", + "oom_killed": false, + "exit_code": 0, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000 + } + ] +} diff --git a/evidence/lab6-smoke.txt b/evidence/lab6-smoke.txt new file mode 100644 index 0000000..8072ea2 --- /dev/null +++ b/evidence/lab6-smoke.txt @@ -0,0 +1,5 @@ +$ make test +python3 tests/smoke.py --base-url "http://localhost:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. + +exit=0 diff --git a/evidence/lab6-validation.txt b/evidence/lab6-validation.txt new file mode 100644 index 0000000..7df7119 --- /dev/null +++ b/evidence/lab6-validation.txt @@ -0,0 +1,12 @@ +$ make check validate PYTHON=.venv/bin/python +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/dkhakimovig/mtusi2026lab6/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( + +exit=0 + +$ GOMODCACHE=/private/tmp/lab6-gomodcache GOCACHE=/private/tmp/lab6-gocache go test -race ./... +PASS: all tested packages; exit=0. Run completed before runtime fault injection. diff --git a/evidence/logs-3c112fa1787d40898e457423046b2e97.json b/evidence/logs-3c112fa1787d40898e457423046b2e97.json new file mode 100644 index 0000000..cdc49ff --- /dev/null +++ b/evidence/logs-3c112fa1787d40898e457423046b2e97.json @@ -0,0 +1,349 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380182289689386", + "scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6", + "severity_number": "9", + "span_id": "48b1a00146d15161", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380182289687670", + "{\"attempt\":4,\"event\":\"job.completed\",\"job_type\":\"attachment.process.v1\",\"outcome\":\"ready\",\"service.name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"48b1a00146d15161\",\"timestamp\":\"2026-11-11T06:56:22.289660585Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380182158663493", + "scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "3899f41081f472da", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380182158661106", + "{\"attempt\":4,\"event\":\"job.published\",\"job_type\":\"attachment.process.v1\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"3899f41081f472da\",\"timestamp\":\"2026-11-11T06:56:22.158638627Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380173660299027", + "scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6", + "severity_number": "9", + "span_id": "4fdca42f0ece4b9b", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380173660289945", + "{\"attempt\":3,\"event\":\"job.completed\",\"job_type\":\"attachment.process.v1\",\"outcome\":\"storage_unavailable\",\"service.name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"4fdca42f0ece4b9b\",\"timestamp\":\"2026-11-11T06:56:13.660190955Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380173552991633", + "scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "315d1330b194b738", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380173552988008", + "{\"attempt\":3,\"event\":\"job.published\",\"job_type\":\"attachment.process.v1\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"315d1330b194b738\",\"timestamp\":\"2026-11-11T06:56:13.552942013Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380169154868452", + "scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6", + "severity_number": "9", + "span_id": "625b78e11c90b490", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380169154854245", + "{\"attempt\":2,\"event\":\"job.completed\",\"job_type\":\"attachment.process.v1\",\"outcome\":\"storage_unavailable\",\"service.name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"625b78e11c90b490\",\"timestamp\":\"2026-11-11T06:56:09.154780337Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380169082259575", + "scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "07283b45d91f7178", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380169082247785", + "{\"attempt\":2,\"event\":\"job.published\",\"job_type\":\"attachment.process.v1\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"07283b45d91f7178\",\"timestamp\":\"2026-11-11T06:56:09.082187541Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380166805494596", + "scope_name": "messenger", + "service_instance_id": "e2edd33d8b58", + "service_name": "messenger-worker", + "service_version": "lab6", + "severity_number": "9", + "span_id": "59ca1b39745a3bd0", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380166805492471", + "{\"attempt\":1,\"event\":\"job.completed\",\"job_type\":\"attachment.process.v1\",\"outcome\":\"storage_unavailable\",\"service.name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"59ca1b39745a3bd0\",\"timestamp\":\"2026-11-11T06:56:06.80541598Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380135473315188", + "scope_name": "messenger", + "service_instance_id": "e6be289d2413", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "fe33cab1b1482745", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380135473314104", + "{\"attempt\":1,\"event\":\"job.published\",\"job_type\":\"attachment.process.v1\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"fe33cab1b1482745\",\"timestamp\":\"2026-11-11T06:55:35.473299437Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380135368996615", + "scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "26a9ac28885c10eb", + "trace_id": "3c112fa1787d40898e457423046b2e97" + }, + "values": [ + [ + "1794380135368994073", + "{\"duration_seconds\":0.01217951,\"event\":\"http.request\",\"method\":\"POST\",\"request_id\":\"abd64a3a-9e50-4f65-8eba-5359554edcb6\",\"route\":\"/api/v1/chats/{chat_id}/attachments\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"26a9ac28885c10eb\",\"status\":202,\"timestamp\":\"2026-11-11T06:55:35.368902734Z\",\"trace_id\":\"3c112fa1787d40898e457423046b2e97\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 27486554, + "linesProcessedPerSecond": 56638, + "totalBytesProcessed": 1721836, + "totalLinesProcessed": 3548, + "execTime": 0.062643, + "queueTime": 0.000637, + "subqueries": 0, + "totalEntriesReturned": 9, + "splits": 2, + "shards": 2, + "totalPostFilterLines": 9, + "totalStructuredMetadataBytesProcessed": 511258 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 2, + "totalChunksMatched": 3, + "totalBatches": 3, + "totalLinesSent": 9, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 334028, + "headChunkLines": 867, + "decompressedBytes": 1387808, + "decompressedLines": 2681, + "compressedBytes": 243276, + "totalDuplicates": 0, + "postFilterLines": 9, + "headChunkStructuredMetadataBytes": 55488, + "decompressedStructuredMetadataBytes": 455770 + }, + "chunkRefsFetchTime": 497540, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 1, + "entriesStored": 1, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 2, + "downloadTime": 2583823, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json b/evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json new file mode 100644 index 0000000..c5133d9 --- /dev/null +++ b/evidence/logs-dbc9b2dc91404a1fa7fae889b013bb34.json @@ -0,0 +1,181 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1794380221967106280", + "scope_name": "messenger", + "service_instance_id": "c7b5ac648af7", + "service_name": "messenger-api", + "service_version": "lab6", + "severity_number": "9", + "span_id": "0180e630b41605b3", + "trace_id": "dbc9b2dc91404a1fa7fae889b013bb34" + }, + "values": [ + [ + "1794380221967096071", + "{\"duration_seconds\":2.008002544,\"event\":\"http.request\",\"method\":\"POST\",\"request_id\":\"403a5ef3-252f-496f-958c-272b032071e8\",\"route\":\"/api/v1/chats/{chat_id}/messages\",\"service.name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"0180e630b41605b3\",\"status\":503,\"timestamp\":\"2026-11-11T06:57:01.966887521Z\",\"trace_id\":\"dbc9b2dc91404a1fa7fae889b013bb34\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 86774690, + "linesProcessedPerSecond": 178807, + "totalBytesProcessed": 1721836, + "totalLinesProcessed": 3548, + "execTime": 0.019843, + "queueTime": 0.000248, + "subqueries": 0, + "totalEntriesReturned": 1, + "splits": 2, + "shards": 2, + "totalPostFilterLines": 1, + "totalStructuredMetadataBytesProcessed": 511258 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 2, + "totalChunksMatched": 3, + "totalBatches": 3, + "totalLinesSent": 1, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 334028, + "headChunkLines": 867, + "decompressedBytes": 1387808, + "decompressedLines": 2681, + "compressedBytes": 243276, + "totalDuplicates": 0, + "postFilterLines": 1, + "headChunkStructuredMetadataBytes": 55488, + "decompressedStructuredMetadataBytes": 455770 + }, + "chunkRefsFetchTime": 231355, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 1, + "entriesRequested": 1, + "entriesStored": 0, + "bytesReceived": 261, + "bytesSent": 0, + "requests": 1, + "downloadTime": 44288, + "queryLengthServed": 116000000000 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/trace-3c112fa1787d40898e457423046b2e97.json b/evidence/trace-3c112fa1787d40898e457423046b2e97.json new file mode 100644 index 0000000..1d19a0d --- /dev/null +++ b/evidence/trace-3c112fa1787d40898e457423046b2e97.json @@ -0,0 +1,2109 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "c7b5ac648af7" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "JqmsKIhcEOs=", + "parentSpanId": "EjRWeJASNFY=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794380135356717557", + "endTimeUnixNano": "1794380135368997573", + "attributes": [ + { + "key": "method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "status_class", + "value": { + "stringValue": "2xx" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "qf5TvWPAyKA=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135357744326", + "endTimeUnixNano": "1794380135358027634", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "PFA7tPa+WhU=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135358044385", + "endTimeUnixNano": "1794380135358363904", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "SWAz1RfYp04=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135361760188", + "endTimeUnixNano": "1794380135361842234", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "439N24T6anc=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135361853568", + "endTimeUnixNano": "1794380135362034162", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Z27IDKzWUSc=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135362047288", + "endTimeUnixNano": "1794380135362310720", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "JMZp8mUKHHo=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135362320846", + "endTimeUnixNano": "1794380135362450145", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "bEYnP0kS/is=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135362452812", + "endTimeUnixNano": "1794380135366897157", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "s1jLJLYm92M=", + "parentSpanId": "bEYnP0kS/is=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135362682034", + "endTimeUnixNano": "1794380135366872323", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "UtmK1IQVkxA=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135366910617", + "endTimeUnixNano": "1794380135367811628", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Aiien611Fb8=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135367821171", + "endTimeUnixNano": "1794380135368245863", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "WzOMUDblYXY=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135368255280", + "endTimeUnixNano": "1794380135368501545", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "yEeIXrQyDkQ=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380135368505461", + "endTimeUnixNano": "1794380135368736933", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "MV0TMLGUtzg=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794380173550942472", + "endTimeUnixNano": "1794380173552994299", + "status": { + "code": "STATUS_CODE_OK" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "e6be289d2413" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "/jPKsbFIJ0U=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794380135472289627", + "endTimeUnixNano": "1794380135473316396", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Byg7RdkfcXg=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794380169080086225", + "endTimeUnixNano": "1794380169082262908", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "OJn0EIH0cto=", + "parentSpanId": "JqmsKIhcEOs=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794380182156679447", + "endTimeUnixNano": "1794380182158664958", + "status": { + "code": "STATUS_CODE_OK" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "e2edd33d8b58" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "WcobOXRaO9A=", + "parentSpanId": "/jPKsbFIJ0U=", + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794380166635951124", + "endTimeUnixNano": "1794380166805495888", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:jobs:v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "storage_unavailable" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "AiA6ue66BqA=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166642339092", + "endTimeUnixNano": "1794380166642495450", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "z6zf13ie0lU=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166642500699", + "endTimeUnixNano": "1794380166644235260", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "2R9k8WFTpWM=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166644240884", + "endTimeUnixNano": "1794380166645641773", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "SMffP56d2+s=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166645666479", + "endTimeUnixNano": "1794380166647248973", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "GjyMl28MaZ8=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166647257680", + "endTimeUnixNano": "1794380166647823577", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "epFt087zLrs=", + "parentSpanId": "WcobOXRaO9A=", + "name": "s3 download and image render", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794380166647834992", + "endTimeUnixNano": "1794380166798363668", + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "X3CXk0ESNMI=", + "parentSpanId": "epFt087zLrs=", + "name": "s3 GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166648590034", + "endTimeUnixNano": "1794380166652020534", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "uRCNNSwPwM4=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166798431743", + "endTimeUnixNano": "1794380166798673383", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "eB5o6j0OaTs=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166798686882", + "endTimeUnixNano": "1794380166799260111", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "s1kOjEvgvEg=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166799317396", + "endTimeUnixNano": "1794380166800022236", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "yaC55DgyisM=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166800034693", + "endTimeUnixNano": "1794380166800677498", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "SRejNlR1b6g=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166800688288", + "endTimeUnixNano": "1794380166801302138", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "NGAkbj4cbc0=", + "parentSpanId": "WcobOXRaO9A=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380166801307845", + "endTimeUnixNano": "1794380166801980647", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Ylt44RyQtJA=", + "parentSpanId": "Byg7RdkfcXg=", + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794380169092293540", + "endTimeUnixNano": "1794380169154871535", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:jobs:v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "storage_unavailable" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "SHAGqDY9vxc=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169092378948", + "endTimeUnixNano": "1794380169092604507", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "ukovGkIFess=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169092615673", + "endTimeUnixNano": "1794380169093302058", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "AahyOUYpaeY=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169093315473", + "endTimeUnixNano": "1794380169096007393", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "8ByeK+4Y4IU=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169096016766", + "endTimeUnixNano": "1794380169097313252", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "b+HLVHmXgYI=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169097319918", + "endTimeUnixNano": "1794380169098897082", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "q1Kgu0mG8as=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "s3 download and image render", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794380169098942536", + "endTimeUnixNano": "1794380169146670374", + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "njushqyvARQ=", + "parentSpanId": "q1Kgu0mG8as=", + "name": "s3 GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169099516141", + "endTimeUnixNano": "1794380169106160387", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "aTyGMXilnYo=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169146726992", + "endTimeUnixNano": "1794380169147085995", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "ZiOEkkLKJbk=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169147099536", + "endTimeUnixNano": "1794380169147742175", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "ndXFt6BeazY=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169147812126", + "endTimeUnixNano": "1794380169148377357", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "HnLSzZ2FizM=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169148388147", + "endTimeUnixNano": "1794380169148851555", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "vdhfK2BIC3s=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169148861347", + "endTimeUnixNano": "1794380169149378875", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "z1wNQuW0w3U=", + "parentSpanId": "Ylt44RyQtJA=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380169149387374", + "endTimeUnixNano": "1794380169150469675", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "T9ykLw7OS5s=", + "parentSpanId": "MV0TMLGUtzg=", + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794380173597573592", + "endTimeUnixNano": "1794380173660302485", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:jobs:v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "3" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "storage_unavailable" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "4kXqJvRjdiI=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173597958302", + "endTimeUnixNano": "1794380173598055250", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Ej1AWsBffdo=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173598058458", + "endTimeUnixNano": "1794380173598278060", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "oZ0P5oH7yT0=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173598281476", + "endTimeUnixNano": "1794380173598505744", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "STZBCRuDmv0=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173598508411", + "endTimeUnixNano": "1794380173598805922", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "AFu2WO1TY4s=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173598809504", + "endTimeUnixNano": "1794380173599101848", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "4CQayLRmlps=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "s3 download and image render", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794380173599109681", + "endTimeUnixNano": "1794380173653304384", + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Pnjp3iYq6xM=", + "parentSpanId": "4CQayLRmlps=", + "name": "s3 GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173599774028", + "endTimeUnixNano": "1794380173605565089", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "V0E7QniIeCQ=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173653392708", + "endTimeUnixNano": "1794380173653728881", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "NBRIVsoBVXg=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173653747088", + "endTimeUnixNano": "1794380173654538255", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "BDt9zTeMAX4=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173654552420", + "endTimeUnixNano": "1794380173655092613", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "3RxftGyUjOo=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173655104362", + "endTimeUnixNano": "1794380173655795998", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "KcAN1+8fYRY=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173655806456", + "endTimeUnixNano": "1794380173656168210", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "4wBWzXSI+n4=", + "parentSpanId": "T9ykLw7OS5s=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380173656176792", + "endTimeUnixNano": "1794380173657147899", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "SLGgAUbRUWE=", + "parentSpanId": "OJn0EIH0cto=", + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794380182223690307", + "endTimeUnixNano": "1794380182289690348", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:jobs:v1" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "4" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "ready" + } + } + ], + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "bGI0nlsP46Q=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182223698805", + "endTimeUnixNano": "1794380182223837741", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "UVlIzIX1O5M=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182223842388", + "endTimeUnixNano": "1794380182224101047", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "OYlXx4BXuU4=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182224105903", + "endTimeUnixNano": "1794380182224815532", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "3xpgOSlcVaw=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182224819508", + "endTimeUnixNano": "1794380182225668660", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "lPNBNa5zpqg=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182225673348", + "endTimeUnixNano": "1794380182226124568", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "3UpPtTlAJNE=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "s3 download and image render", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794380182226135117", + "endTimeUnixNano": "1794380182285434839", + "status": {} + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "+oxZuuRzWJ4=", + "parentSpanId": "3UpPtTlAJNE=", + "name": "s3 GET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182226342162", + "endTimeUnixNano": "1794380182228523582", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "gBU1EOqnGnQ=", + "parentSpanId": "3UpPtTlAJNE=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182274276444", + "endTimeUnixNano": "1794380182277741910", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "yIwQM65Gx9I=", + "parentSpanId": "gBU1EOqnGnQ=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182274361171", + "endTimeUnixNano": "1794380182277732994", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "ITe3RcQZcK4=", + "parentSpanId": "3UpPtTlAJNE=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182277760245", + "endTimeUnixNano": "1794380182281164385", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "XMv4xkkncos=", + "parentSpanId": "ITe3RcQZcK4=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182277807297", + "endTimeUnixNano": "1794380182281154129", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "9RzyW1+/JHQ=", + "parentSpanId": "3UpPtTlAJNE=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182281197120", + "endTimeUnixNano": "1794380182283263465", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "k7DgIvUgTKk=", + "parentSpanId": "9RzyW1+/JHQ=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182281317471", + "endTimeUnixNano": "1794380182283258944", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Gpx0S+MRNmI=", + "parentSpanId": "3UpPtTlAJNE=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182283279372", + "endTimeUnixNano": "1794380182285281586", + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "Dz7siALuzVw=", + "parentSpanId": "Gpx0S+MRNmI=", + "name": "s3 PUT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182283342708", + "endTimeUnixNano": "1794380182285278111", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "PUT" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "FEI+gRP2zdk=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182285452672", + "endTimeUnixNano": "1794380182285587171", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "7y0R6dLAddA=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182285593492", + "endTimeUnixNano": "1794380182286316056", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "vcuQ6mkoJDk=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182286326940", + "endTimeUnixNano": "1794380182286511756", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "WfYaxU+hLAY=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182286515649", + "endTimeUnixNano": "1794380182287047242", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "PBEvoXh9QImORXQjBGsulw==", + "spanId": "+pcmKv1G89E=", + "parentSpanId": "SLGgAUbRUWE=", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380182287052601", + "endTimeUnixNano": "1794380182287378405", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + } + ] + } + ] + } + ] +} diff --git a/evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json b/evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json new file mode 100644 index 0000000..4c67fe4 --- /dev/null +++ b/evidence/trace-dbc9b2dc91404a1fa7fae889b013bb34.json @@ -0,0 +1,239 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "c7b5ac648af7" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "EcXZvs2zSiA=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219960223589", + "endTimeUnixNano": "1794380219960724417", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "gkE63CJbPbQ=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219960746208", + "endTimeUnixNano": "1794380219961817030", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "ep+Ye6udB30=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219961950029", + "endTimeUnixNano": "1794380219962743021", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "TxWSichX9ZA=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219962766813", + "endTimeUnixNano": "1794380219962922353", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "m/ApFyKYZ0k=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219962928019", + "endTimeUnixNano": "1794380219963135225", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "5sQcL8g048w=", + "parentSpanId": "AYDmMLQWBbM=", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794380219963139559", + "endTimeUnixNano": "1794380221964078996", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": { + "message": "dependency unavailable", + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "28my3JFASh+n+uiJsBO7NA==", + "spanId": "AYDmMLQWBbM=", + "parentSpanId": "EjRWeJASNFY=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794380219958881186", + "endTimeUnixNano": "1794380221967108989", + "attributes": [ + { + "key": "method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "status_class", + "value": { + "stringValue": "5xx" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "message": "server error", + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + } + ] +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a7408a1 --- /dev/null +++ b/go.mod @@ -0,0 +1,63 @@ +module gitflic.ru/akselbant/mtusi2026lab1-fork + +go 1.26.4 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.7.6 + github.com/labstack/echo/v4 v4.13.4 + github.com/minio/minio-go/v7 v7.0.95 + github.com/redis/go-redis/v9 v9.12.1 + github.com/swaggest/swgui v1.8.9 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + golang.org/x/crypto v0.41.0 + golang.org/x/sync v0.16.0 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/labstack/gommon v0.4.2 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/vearutop/statigz v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..818ff8b --- /dev/null +++ b/go.sum @@ -0,0 +1,143 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/bool64/dev v0.2.45 h1:3nLKhAS/6Oklk3Mt2lHYSN/Cb4tdAD77KLwzeP+6eYE= +github.com/bool64/dev v0.2.45/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= +github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/swaggest/swgui v1.8.9 h1:cxAgIwouPpZPlvX68jY5fpwarzLbkc8/IL6DMj+H460= +github.com/swaggest/swgui v1.8.9/go.mod h1:eTJfgwudbyw9xMwqO26vs82ei2u6//JnUAofx2vGB3M= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vearutop/statigz v1.4.0 h1:RQL0KG3j/uyA/PFpHeZ/L6l2ta920/MxlOAIGEOuwmU= +github.com/vearutop/statigz v1.4.0/go.mod h1:LYTolBLiz9oJISwiVKnOQoIwhO1LWX1A7OECawGS8XE= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/README.md b/infra/README.md index 5a39e86..ca4a250 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,23 @@ -# Инфраструктура студента +# Инфраструктура lab6 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Полная топология находится в [compose.yaml](../compose.yaml). Запуск: `make up` из корня. +Файл compose.example.yaml сохранён как исходный worksheet задания и не используется при запуске. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Конфигурация | Назначение | +| --- | --- | +| haproxy.cfg | Балансировка двух API-реплик, DNS Docker, readiness каждой реплики | +| redis.conf | Сессии, Redis Streams и AOF | +| migrate.sh | Goose migrations с credentials из файла секрета | +| minio-entrypoint.sh | Запуск MinIO с root credentials из файлов | +| s3-bootstrap.sh, s3-policy.json | Приватный bucket и пользователь приложения с ограниченной policy | +| otel/collector.yaml | Приём OTLP, memory limiter, batch, экспорт трёх сигналов | +| prometheus/ | Scrape приложения/Collector/backend и локальные alert rules | +| loki.yaml | OTLP logs, filesystem storage, retention | +| tempo.yaml | OTLP traces, WAL/blocks, retention | +| grafana/provisioning/ | Автоматическая регистрация трёх datasources и dashboard provider | +| grafana/dashboards/messenger.json | Импортируемый dashboard с 15 панелями | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Named volumes сохраняют PostgreSQL, Redis, S3 и историю всех telemetry backend/Grafana. +Bootstrap идемпотентен. Credentials создаёт scripts/bootstrap.sh; они не хранятся в Git. +Сопоставление версий, readiness, volumes, портов и лимитов: [REPORT.md](../REPORT.md). +Диагностика по сохранённым сигналам: [RUNBOOK.md](../RUNBOOK.md). diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..7fcfe1f --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,519 @@ +{ + "uid": "messenger", + "title": "Messenger observability", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "API RPS (business routes)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(messenger_http_requests_total[$__rate_interval])) by (route)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "API 5xx / all business requests", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total[$__rate_interval])),0.001)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "API p50 / p95 / p99 (all replicas, histogram)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.5,sum(rate(messenger_http_duration_seconds_bucket[$__rate_interval])) by(le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95,sum(rate(messenger_http_duration_seconds_bucket[$__rate_interval])) by(le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(messenger_http_duration_seconds_bucket[$__rate_interval])) by(le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 4, + "title": "Worker job outcomes per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(messenger_jobs_total[$__rate_interval])) by(outcome)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "Worker job p95 duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(messenger_job_duration_seconds_bucket[$__rate_interval])) by(le,outcome))", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Time from durable acceptance to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum(rate(messenger_attachment_ready_seconds_bucket[$__rate_interval])) by(le))", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "Authoritative queue depth (max across observers)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(messenger_queue_depth)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Oldest unfinished job age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(messenger_queue_oldest_age_seconds)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "DB pool acquired / acquiring / max", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "messenger_db_pool_used", + "legendFormat": "used {{service_name}} {{service_instance_id}}", + "refId": "A" + }, + { + "expr": "messenger_db_pool_waiting", + "legendFormat": "waiting {{service_name}} {{service_instance_id}}", + "refId": "B" + }, + { + "expr": "messenger_db_pool_max", + "legendFormat": "max {{service_name}} {{service_instance_id}}", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "Worker saturation busy / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(messenger_worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(messenger_worker_capacity{service_name=\"messenger-worker\"}),1)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "Go runtime reserved memory (MemStats.Sys)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "messenger_process_memory_bytes", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "Process CPU seconds / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "rate(messenger_process_cpu_seconds_total[$__rate_interval])", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "SDK failed telemetry export operations", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "messenger_telemetry_export_errors", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "Exporter failed spans (attempts, not unique loss)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(otelcol_exporter_send_failed_spans_total) by (exporter)", + "legendFormat": "{{service_name}} {{outcome}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 99, + "title": "API and worker structured logs; trace ID opens Tempo", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 24, + "h": 10 + }, + "targets": [ + { + "expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", + "refId": "A" + } + ] + } + ], + "annotations": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..cab1e93 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,6 @@ +apiVersion: 1 +providers: + - name: Messenger + type: file + options: {path: /var/lib/grafana/dashboards} + updateIntervalSeconds: 10 diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..d4ca408 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,29 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([a-f0-9]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + serviceMap: {datasourceUid: prometheus} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..2eec42f --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,36 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + timeout connect 1s + timeout client 55s + timeout server 55s + timeout check 1s + retries 0 + option http-server-close + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1 + server-template replica 1-8 api:8080 check resolvers docker init-addr none diff --git a/infra/loki.yaml b/infra/loki.yaml new file mode 100644 index 0000000..194c36d --- /dev/null +++ b/infra/loki.yaml @@ -0,0 +1,27 @@ +auth_enabled: false +server: {http_listen_port: 3100, grpc_listen_port: 9096} +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: {chunks_directory: /loki/chunks, rules_directory: /loki/rules} + replication_factor: 1 + ring: + kvstore: {store: inmemory} +schema_config: + configs: + - from: '2024-04-01' + store: tsdb + object_store: filesystem + schema: v13 + index: {prefix: index_, period: 24h} +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 8 +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: {reporting_enabled: false} diff --git a/infra/migrate.sh b/infra/migrate.sh new file mode 100755 index 0000000..70a11b2 --- /dev/null +++ b/infra/migrate.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Keep the password out of argv and disable automatic dotenv loading. +if [ -n "${POSTGRES_PASSWORD_FILE:-}" ]; then + PGPASSWORD=$(cat "$POSTGRES_PASSWORD_FILE") + export PGPASSWORD +fi +exec goose -env /dev/null -dir /migrations -timeout 30s "$@" diff --git a/infra/minio-entrypoint.sh b/infra/minio-entrypoint.sh new file mode 100755 index 0000000..ba24993 --- /dev/null +++ b/infra/minio-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_access_key)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_secret_key)" +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..fd0b28f 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,23 @@ -# Конфигурация наблюдаемости — реализовать +# OpenTelemetry Collector -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +Конфигурация: [collector.yaml](collector.yaml), образ contrib 0.137.0 закреплён в Compose. +API и worker используют SDK Go и передают три сигнала по OTLP HTTP/protobuf на collector:4318. +GRPC receiver слушает :4317. Оба порта доступны только внутри сети Compose. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +| Pipeline | Processors | Exporter / backend | +| --- | --- | --- | +| metrics | memory_limiter, batch | Prometheus exporter :8889; Prometheus scrape каждые 5s | +| logs | memory_limiter, batch | OTLP HTTP → Loki /otlp | +| traces | memory_limiter, batch | OTLP HTTP → Tempo :4318 | -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Memory limiter:128MiB, spike32MiB, check1s. Batch:256, max512, timeout1s. +Каждый OTLP exporter имеет queue256,2 consumers, timeout2s и retry максимум 20s. +Collector ограничен cgroup192MiB. После исчерпания очереди/таймаутов сигналы могут потеряться. +Очереди находятся в RAM и не переживают аварийный restart. Работа приложения не зависит от успешной записи telemetry. +Метрики самого Collector доступны внутри сети на :8888 и собираются Prometheus. + +Данные хранят отдельные Prometheus/Loki/Tempo; Collector не заменяет хранилище. +Grafana provisioning: ../grafana/provisioning, dashboard: ../grafana/dashboards/messenger.json. +Alert rules: ../prometheus/alerts.yaml. Процедуры расследования: ../../RUNBOOK.md. + +Доказательства и ограничения: ../../REPORT.md, ../../INCIDENTS.md, ../../RESEARCH.md. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..97140c7 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,41 @@ +receivers: + otlp: + protocols: + grpc: {endpoint: '0.0.0.0:4317'} + http: {endpoint: '0.0.0.0:4318'} +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 256 + send_batch_max_size: 512 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 15s + resource_to_telemetry_conversion: {enabled: true} + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + sending_queue: {enabled: true, queue_size: 256, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 5s, max_elapsed_time: 20s} + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: {enabled: true, queue_size: 256, num_consumers: 2} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 5s, max_elapsed_time: 20s} +service: + telemetry: + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: {host: 0.0.0.0, port: 8888} + pipelines: + traces: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlphttp/tempo]} + metrics: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [prometheus]} + logs: {receivers: [otlp], processors: [memory_limiter, batch], exporters: [otlphttp/loki]} diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml new file mode 100644 index 0000000..163ba0d --- /dev/null +++ b/infra/prometheus/alerts.yaml @@ -0,0 +1,19 @@ +groups: + - name: messenger-slo + rules: + - alert: APIHighErrorRate + expr: sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 0.01) > 0.05 + for: 5s + labels: {severity: warning} + annotations: + summary: More than 5 percent API server failures over 1 minute + runbook_url: http://localhost:13000/d/messenger/messenger-observability + description: Excludes health and client 4xx. Inspect PostgreSQL spans and pool; see RUNBOOK.md. + - alert: QueueLagGrowing + expr: max(messenger_queue_oldest_age_seconds) > 10 and max(messenger_queue_depth) > 0 + for: 5s + labels: {severity: warning} + annotations: + summary: Oldest unfinished attachment exceeds 10 seconds + runbook_url: http://localhost:13000/d/messenger/messenger-observability + description: Inspect worker capacity, busy, outcomes and dependency spans; see RUNBOOK.md. diff --git a/infra/prometheus/prometheus.yaml b/infra/prometheus/prometheus.yaml new file mode 100644 index 0000000..134f73f --- /dev/null +++ b/infra/prometheus/prometheus.yaml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: [/etc/prometheus/alerts.yaml] +scrape_configs: + - job_name: messenger + static_configs: [{targets: ['collector:8889']}] + - job_name: collector + static_configs: [{targets: ['collector:8888']}] + - job_name: tempo + static_configs: [{targets: ['tempo:3200']}] + - job_name: loki + static_configs: [{targets: ['loki:3100']}] + - job_name: prometheus + static_configs: [{targets: ['localhost:9090']}] diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..0b8eb4c --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,9 @@ +bind 0.0.0.0 +protected-mode no +port 6379 +dir /data +appendonly yes +appendfsync always +save "" +maxmemory 256mb +maxmemory-policy noeviction diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100755 index 0000000..4251f8f --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +export MC_CONFIG_DIR=/tmp/mc +mc alias set admin http://minio:9000 "$(cat /run/secrets/s3_root_access_key)" "$(cat /run/secrets/s3_root_secret_key)" >/dev/null +mc ready admin >/dev/null +mc mb --ignore-existing admin/attachments >/dev/null +mc anonymous set none admin/attachments >/dev/null +mc admin policy create admin attachments-api /policy.json >/dev/null +mc admin user add admin "$(cat /run/secrets/s3_access_key)" "$(cat /run/secrets/s3_secret_key)" >/dev/null +mc admin policy attach admin attachments-api --user "$(cat /run/secrets/s3_access_key)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API account configured.' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..20797ca --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::attachments"], + "Condition": {"StringLike": {"s3:prefix": ["original/*", "staging/*"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::attachments/original/*", "arn:aws:s3:::attachments/staging/*"] + } + ] +} diff --git a/infra/tempo.yaml b/infra/tempo.yaml new file mode 100644 index 0000000..0bd40aa --- /dev/null +++ b/infra/tempo.yaml @@ -0,0 +1,17 @@ +server: {http_listen_port: 3200} +distributor: + receivers: + otlp: + protocols: + grpc: {endpoint: '0.0.0.0:4317'} + http: {endpoint: '0.0.0.0:4318'} +ingester: + max_block_duration: 1m +compactor: + compaction: {block_retention: 168h} +storage: + trace: + backend: local + wal: {path: /var/tempo/wal} + local: {path: /var/tempo/blocks} +usage_report: {reporting_enabled: false} diff --git a/internal/app/chat/access.go b/internal/app/chat/access.go new file mode 100644 index 0000000..36f71a5 --- /dev/null +++ b/internal/app/chat/access.go @@ -0,0 +1,31 @@ +package chat + +import ( + "errors" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) authorizedChat(c echo.Context, requestedChat domain.Chat) (domain.Chat, bool, error) { + chat, err := h.useCase.GetChat(c.Request().Context(), authenticatedUser(c).ID, requestedChat) + if err != nil { + return domain.Chat{}, false, h.useCaseError(c, err) + } + return chat, true, nil +} + +func (h *Chat) cursorPosition(raw string, present bool, scope, owner string) (*domain.Position, error) { + if !present { + return nil, nil + } + if raw == "" { + return nil, errors.New("empty cursor") + } + createdAt, id, err := h.cursor.decode(raw, scope, owner) + if err != nil { + return nil, err + } + return &domain.Position{CreatedAt: createdAt, ID: id}, nil +} diff --git a/internal/app/chat/attachments.go b/internal/app/chat/attachments.go new file mode 100644 index 0000000..c345e17 --- /dev/null +++ b/internal/app/chat/attachments.go @@ -0,0 +1,75 @@ +package chat + +import ( + "context" + "mime" + "net/http" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/labstack/echo/v4" +) + +func (h *Chat) WithAttachments(service *attachment.Service) *Chat { h.attachments = service; return h } +func (h *Chat) uploadAttachment(c echo.Context) error { + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: c.Param("chat_id")}) + if err != nil || !ok { + return err + } + filename, present, err := oneQueryValue(c, "filename") + if err != nil || !present { + return h.invalid(c, "Требуется один filename") + } + if c.Request().ContentLength > attachment.MaxBytes { + return h.writeError(c, 413, "payload_too_large", "Файл превышает 10 MiB") + } + // Both socket reads and the application operation have a fixed deadline. + deadline := time.Now().Add(attachment.UploadTimeout) + _ = http.NewResponseController(c.Response().Writer).SetReadDeadline(deadline) + defer http.NewResponseController(c.Response().Writer).SetReadDeadline(time.Time{}) + c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, attachment.MaxBytes+1) + a, err := h.attachments.Upload(c.Request().Context(), chat.ID, authenticatedUser(c).ID, filename, c.Request().Header.Get("Content-Type"), c.Request().Body) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusAccepted, a) +} +func (h *Chat) getAttachment(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusOK, a) +} +func (h *Chat) attachmentContent(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + variant, present, err := oneQueryValue(c, "variant") + if err != nil { + return h.invalid(c, "Некорректный variant") + } + if a.Status != "ready" { + return h.writeError(c, 409, "conflict", "Вложение не готово") + } + if !present { + variant = "original" + } + a, err = h.attachments.Variant(a, variant) + if err != nil { + return h.useCaseError(c, err) + } + ctx, cancel := context.WithTimeout(c.Request().Context(), 45*time.Second) + defer cancel() + obj, err := h.attachments.Content(ctx, a) + if err != nil { + return h.useCaseError(c, err) + } + defer obj.Close() + c.Response().Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Response().Header().Set("X-Content-Type-Options", "nosniff") + c.Response().Header().Set("Cache-Control", "private, no-store") + return c.Stream(http.StatusOK, a.ContentType, obj) +} diff --git a/internal/app/chat/create_chat.go b/internal/app/chat/create_chat.go new file mode 100644 index 0000000..7e57b2e --- /dev/null +++ b/internal/app/chat/create_chat.go @@ -0,0 +1,33 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) createChat(c echo.Context) error { + var request dto.CreateChatRequest + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + callerID := authenticatedUser(c).ID + chat, err := h.useCase.CreateChat(c.Request().Context(), callerID, createChatRequestToDomain(request)) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createChatResponseFromDomain(chat)) +} + +func createChatRequestToDomain(request dto.CreateChatRequest) domain.Chat { + return domain.Chat{Title: request.Title, MemberIDs: request.MemberIDs} +} + +func createChatResponseFromDomain(chat domain.Chat) dto.CreateChatResponse { + return dto.CreateChatResponse{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + } +} diff --git a/internal/app/chat/create_message.go b/internal/app/chat/create_message.go new file mode 100644 index 0000000..3c0c27b --- /dev/null +++ b/internal/app/chat/create_message.go @@ -0,0 +1,58 @@ +package chat + +import ( + "net/http" + "regexp" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func (h *Chat) createMessage(c echo.Context) error { + request := dto.CreateMessageRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: request.ChatID}) + if err != nil { + return err + } + if !ok { + return nil + } + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + request.ChatID = chat.ID + + keys := c.Request().Header.Values("Idempotency-Key") + if len(keys) != 1 || !idempotencyKeyPattern.MatchString(keys[0]) { + return h.invalid(c, "Требуется Idempotency-Key: ASCII [A-Za-z0-9._:-]{1,128}") + } + input := createMessageRequestToDomain(request) + input.IdempotencyKey = keys[0] + message, err := h.useCase.CreateMessage(c.Request().Context(), authenticatedUser(c).ID, input) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createMessageResponseFromDomain(message)) +} + +func createMessageRequestToDomain(request dto.CreateMessageRequest) domain.Message { + return domain.Message{ChatID: request.ChatID, Text: request.Text, AttachmentIDs: request.AttachmentIDs} +} + +func createMessageResponseFromDomain(message domain.Message) dto.CreateMessageResponse { + return dto.CreateMessageResponse{ + ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), + } +} + +func nonnilIDs(ids []string) []string { + if ids == nil { + return []string{} + } + return ids +} diff --git a/internal/app/chat/create_user.go b/internal/app/chat/create_user.go new file mode 100644 index 0000000..f607817 --- /dev/null +++ b/internal/app/chat/create_user.go @@ -0,0 +1,32 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) createUser(c echo.Context) error { + var request dto.CreateUserRequest + if err := decodeJSON(c, &request); err != nil { + return h.jsonError(c, err) + } + user, err := h.useCase.CreateUser(c.Request().Context(), createUserRequestToDomain(request), request.Password) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, createUserResponseFromDomain(user)) +} + +func createUserRequestToDomain(request dto.CreateUserRequest) domain.User { + return domain.User{Username: request.Username, DisplayName: request.DisplayName} +} + +func createUserResponseFromDomain(user domain.User) dto.CreateUserResponse { + return dto.CreateUserResponse{ + ID: user.ID, Username: user.Username, DisplayName: user.DisplayName, CreatedAt: dto.Timestamp(user.CreatedAt), + } +} diff --git a/internal/app/chat/current_user.go b/internal/app/chat/current_user.go new file mode 100644 index 0000000..23277de --- /dev/null +++ b/internal/app/chat/current_user.go @@ -0,0 +1,21 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) currentUser(c echo.Context) error { + user := authenticatedUser(c) + return c.JSON(http.StatusOK, currentUserResponseFromDomain(user)) +} + +func currentUserResponseFromDomain(user domain.User) dto.CurrentUserResponse { + return dto.CurrentUserResponse{ + ID: user.ID, Username: user.Username, DisplayName: user.DisplayName, CreatedAt: dto.Timestamp(user.CreatedAt), + } +} diff --git a/internal/app/chat/cursor.go b/internal/app/chat/cursor.go new file mode 100644 index 0000000..83836f1 --- /dev/null +++ b/internal/app/chat/cursor.go @@ -0,0 +1,75 @@ +package chat + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/google/uuid" +) + +type cursorPayload struct { + Scope string `json:"scope"` + Owner string `json:"owner"` + CreatedAt string `json:"created_at"` + ID string `json:"id"` +} + +type cursorCodec struct { + secret [32]byte +} + +func (c cursorCodec) encode(scope, owner string, createdAt time.Time, id string) string { + payload, err := json.Marshal(cursorPayload{ + Scope: scope, + Owner: owner, + CreatedAt: createdAt.UTC().Format(time.RFC3339Nano), + ID: id, + }) + if err != nil { + panic("cannot encode cursor: " + err.Error()) + } + signature := c.sign(payload) + return base64.RawURLEncoding.EncodeToString(payload) + "." + base64.RawURLEncoding.EncodeToString(signature) +} + +func (c cursorCodec) decode(raw, scope, owner string) (time.Time, string, error) { + parts := strings.Split(raw, ".") + if len(parts) != 2 { + return time.Time{}, "", errors.New("malformed cursor") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return time.Time{}, "", errors.New("malformed cursor") + } + signature, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil || !hmac.Equal(signature, c.sign(payload)) { + return time.Time{}, "", errors.New("invalid cursor signature") + } + var value cursorPayload + if err := json.Unmarshal(payload, &value); err != nil { + return time.Time{}, "", errors.New("malformed cursor payload") + } + if value.Scope != scope || value.Owner != owner { + return time.Time{}, "", errors.New("cursor belongs to another resource") + } + parsedID, err := uuid.Parse(value.ID) + if err != nil || parsedID.String() != value.ID { + return time.Time{}, "", errors.New("malformed cursor id") + } + createdAt, err := time.Parse(time.RFC3339Nano, value.CreatedAt) + if err != nil { + return time.Time{}, "", errors.New("malformed cursor timestamp") + } + return createdAt, value.ID, nil +} + +func (c cursorCodec) sign(payload []byte) []byte { + mac := hmac.New(sha256.New, c.secret[:]) + _, _ = mac.Write(payload) + return mac.Sum(nil) +} diff --git a/internal/app/chat/errors.go b/internal/app/chat/errors.go new file mode 100644 index 0000000..7c532f8 --- /dev/null +++ b/internal/app/chat/errors.go @@ -0,0 +1,98 @@ +package chat + +import ( + "errors" + "net/http" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) invalid(c echo.Context, message string) error { + return h.writeError(c, http.StatusBadRequest, "invalid_request", message) +} + +func (h *Chat) jsonError(c echo.Context, err error) error { + if errors.Is(err, errUnsupportedMediaType) { + return h.writeError(c, http.StatusUnsupportedMediaType, "unsupported_media_type", "Content-Type должен быть application/json") + } + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return h.writeError(c, http.StatusRequestEntityTooLarge, "payload_too_large", "Тело запроса слишком большое") + } + return h.invalid(c, err.Error()) +} + +func (h *Chat) useCaseError(c echo.Context, err error) error { + applicationError, ok := domain.AsApplicationError(err) + if !ok { + return err + } + status := http.StatusInternalServerError + switch applicationError.Code { + case "unsupported_media_type": + status = http.StatusUnsupportedMediaType + case "rate_limited": + status = http.StatusTooManyRequests + c.Response().Header().Set("Retry-After", "2") + case "payload_too_large": + status = http.StatusRequestEntityTooLarge + case domain.ErrorUnavailable: + status = http.StatusServiceUnavailable + case domain.ErrorInvalid: + status = http.StatusBadRequest + case domain.ErrorUnauthorized: + status = http.StatusUnauthorized + challenge := `Bearer realm="messenger"` + if c.Path() == "/api/v1/auth/sessions" { + challenge = `Basic realm="messenger", charset="UTF-8"` + } + c.Response().Header().Set("WWW-Authenticate", challenge) + case domain.ErrorNotFound: + status = http.StatusNotFound + case domain.ErrorConflict: + status = http.StatusConflict + } + return h.writeError(c, status, string(applicationError.Code), applicationError.Message) +} + +func (h *Chat) writeError(c echo.Context, status int, code, message string) error { + return c.JSON(status, dto.ErrorResponse{Error: dto.ErrorDetails{ + Code: code, + Message: message, + RequestID: requestID(c), + }}) +} + +func (h *Chat) httpErrorHandler(err error, c echo.Context) { + if c.Response().Committed { + return + } + status := http.StatusInternalServerError + code := "internal_error" + message := "Внутренняя ошибка сервера" + var httpError *echo.HTTPError + if errors.As(err, &httpError) { + status = httpError.Code + switch status { + case http.StatusNotFound: + code, message = "not_found", "Объект не найден" + case http.StatusMethodNotAllowed, http.StatusBadRequest: + code, message = "invalid_request", "Некорректный запрос" + } + } + _ = h.writeError(c, status, code, message) +} + +func requestID(c echo.Context) string { + value, _ := c.Get(requestIDKey).(string) + if value == "" { + value = uuid.NewString() + c.Set(requestIDKey, value) + c.Response().Header().Set("X-Request-Id", value) + } + return value +} diff --git a/internal/app/chat/get_chat.go b/internal/app/chat/get_chat.go new file mode 100644 index 0000000..eaddd9e --- /dev/null +++ b/internal/app/chat/get_chat.go @@ -0,0 +1,32 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) getChat(c echo.Context) error { + request := dto.GetChatRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, getChatRequestToDomain(request)) + if err != nil { + return err + } + if !ok { + return nil + } + return c.JSON(http.StatusOK, getChatResponseFromDomain(chat)) +} + +func getChatRequestToDomain(request dto.GetChatRequest) domain.Chat { + return domain.Chat{ID: request.ChatID} +} + +func getChatResponseFromDomain(chat domain.Chat) dto.GetChatResponse { + return dto.GetChatResponse{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + } +} diff --git a/internal/app/chat/list_chats.go b/internal/app/chat/list_chats.go new file mode 100644 index 0000000..1a02d5f --- /dev/null +++ b/internal/app/chat/list_chats.go @@ -0,0 +1,53 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) listChats(c echo.Context) error { + limit, err := parseLimit(c) + if err != nil { + return h.invalid(c, err.Error()) + } + rawCursor, cursorPresent, err := oneQueryValue(c, "cursor") + if err != nil { + return h.invalid(c, "cursor должен быть указан один раз") + } + request := dto.ListChatsRequest{Limit: limit, Cursor: rawCursor} + userID := authenticatedUser(c).ID + position, err := h.cursorPosition(request.Cursor, cursorPresent, "chats", userID) + if err != nil { + return h.invalid(c, "Некорректный cursor") + } + + page, err := h.useCase.ListChats(c.Request().Context(), userID, listChatsRequestToDomain(request, position)) + if err != nil { + return h.useCaseError(c, err) + } + var next *string + if page.HasMore { + last := page.Items[len(page.Items)-1] + encoded := h.cursor.encode("chats", userID, last.CreatedAt, last.ID) + next = &encoded + } + return c.JSON(http.StatusOK, listChatsResponseFromDomain(page, next)) +} + +func listChatsRequestToDomain(request dto.ListChatsRequest, position *domain.Position) domain.Pagination { + return domain.Pagination{After: position, Limit: request.Limit} +} + +func listChatsResponseFromDomain(page domain.ChatPage, nextCursor *string) dto.ListChatsResponse { + items := make([]dto.ListChatsItem, 0, len(page.Items)) + for _, chat := range page.Items { + items = append(items, dto.ListChatsItem{ + ID: chat.ID, Title: chat.Title, MemberIDs: chat.MemberIDs, CreatedAt: dto.Timestamp(chat.CreatedAt), + }) + } + return dto.ListChatsResponse{Items: items, NextCursor: nextCursor} +} diff --git a/internal/app/chat/list_messages.go b/internal/app/chat/list_messages.go new file mode 100644 index 0000000..1d7f252 --- /dev/null +++ b/internal/app/chat/list_messages.go @@ -0,0 +1,70 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (h *Chat) listMessages(c echo.Context) error { + request := dto.ListMessagesRequest{ChatID: c.Param("chat_id")} + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: request.ChatID}) + if err != nil { + return err + } + if !ok { + return nil + } + limit, err := parseLimit(c) + if err != nil { + return h.invalid(c, err.Error()) + } + rawCursor, cursorPresent, err := oneQueryValue(c, "cursor") + if err != nil { + return h.invalid(c, "cursor должен быть указан один раз") + } + request.ChatID = chat.ID + request.Limit = limit + request.Cursor = rawCursor + position, err := h.cursorPosition(request.Cursor, cursorPresent, "messages", request.ChatID) + if err != nil { + return h.invalid(c, "Некорректный cursor") + } + + page, err := h.useCase.ListMessages( + c.Request().Context(), authenticatedUser(c).ID, + listMessagesChatToDomain(request), listMessagesPaginationToDomain(request, position), + ) + if err != nil { + return h.useCaseError(c, err) + } + var next *string + if page.HasMore { + last := page.Items[len(page.Items)-1] + encoded := h.cursor.encode("messages", request.ChatID, last.CreatedAt, last.ID) + next = &encoded + } + return c.JSON(http.StatusOK, listMessagesResponseFromDomain(page, next)) +} + +func listMessagesChatToDomain(request dto.ListMessagesRequest) domain.Chat { + return domain.Chat{ID: request.ChatID} +} + +func listMessagesPaginationToDomain(request dto.ListMessagesRequest, position *domain.Position) domain.Pagination { + return domain.Pagination{After: position, Limit: request.Limit} +} + +func listMessagesResponseFromDomain(page domain.MessagePage, nextCursor *string) dto.ListMessagesResponse { + items := make([]dto.ListMessagesItem, 0, len(page.Items)) + for _, message := range page.Items { + items = append(items, dto.ListMessagesItem{ + ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), + }) + } + return dto.ListMessagesResponse{Items: items, NextCursor: nextCursor} +} diff --git a/internal/app/chat/liveness.go b/internal/app/chat/liveness.go new file mode 100644 index 0000000..6b6029f --- /dev/null +++ b/internal/app/chat/liveness.go @@ -0,0 +1,13 @@ +package chat + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func (h *Chat) liveness(c echo.Context) error { + return c.JSON(http.StatusOK, dto.LivenessResponse{Status: "ok"}) +} diff --git a/internal/app/chat/middleware.go b/internal/app/chat/middleware.go new file mode 100644 index 0000000..353b517 --- /dev/null +++ b/internal/app/chat/middleware.go @@ -0,0 +1,43 @@ +package chat + +import ( + "net/http" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const ( + requestIDKey = "request_id" + userKey = "current_user" +) + +func (h *Chat) requestIDMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + requestID := uuid.NewString() + c.Set(requestIDKey, requestID) + c.Response().Header().Set("X-Request-Id", requestID) + c.Response().Header().Set("X-Instance-Id", h.instanceID) + if h.draining.Load() && c.Request().URL.Path != "/health/live" { + return h.writeError(c, http.StatusServiceUnavailable, "unavailable", "Реплика завершает работу") + } + return next(c) + } +} + +func (h *Chat) authenticate(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + user, err := h.useCase.Authenticate(c.Request().Context(), bearerToken(c)) + if err != nil { + return h.useCaseError(c, err) + } + c.Set(userKey, user) + return next(c) + } +} + +func authenticatedUser(c echo.Context) domain.User { + return c.Get(userKey).(domain.User) +} diff --git a/internal/app/chat/readiness.go b/internal/app/chat/readiness.go new file mode 100644 index 0000000..950ca17 --- /dev/null +++ b/internal/app/chat/readiness.go @@ -0,0 +1,25 @@ +package chat + +import ( + "context" + "net/http" + "time" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func (h *Chat) readiness(c echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), 2*time.Second) + defer cancel() + if err := h.useCase.Ready(ctx); err != nil { + return h.useCaseError(c, err) + } + if h.attachments != nil { + if err := h.attachments.Ready(ctx); err != nil { + return h.useCaseError(c, err) + } + } + return c.JSON(http.StatusOK, dto.ReadinessResponse{Status: "ok"}) +} diff --git a/internal/app/chat/request.go b/internal/app/chat/request.go new file mode 100644 index 0000000..2cb05d4 --- /dev/null +++ b/internal/app/chat/request.go @@ -0,0 +1,74 @@ +package chat + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strconv" + + "github.com/labstack/echo/v4" +) + +const ( + defaultLimit = 50 + maxLimit = 100 + maxJSONBody = 1 << 20 +) + +var errUnsupportedMediaType = errors.New("unsupported media type") + +func decodeJSON(c echo.Context, target any) error { + mediaType, _, err := mime.ParseMediaType(c.Request().Header.Get(echo.HeaderContentType)) + if err != nil || mediaType != echo.MIMEApplicationJSON { + return errUnsupportedMediaType + } + reader := http.MaxBytesReader(c.Response(), c.Request().Body, maxJSONBody) + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return err + } + return errors.New("Некорректное JSON-тело") + } + var extra any + err = decoder.Decode(&extra) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return err + } + if !errors.Is(err, io.EOF) { + return errors.New("JSON-тело должно содержать ровно один объект") + } + return nil +} + +func parseLimit(c echo.Context) (int, error) { + raw, present, err := oneQueryValue(c, "limit") + if err != nil { + return 0, errors.New("limit должен быть указан один раз") + } + if !present { + return defaultLimit, nil + } + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 || limit > maxLimit { + return 0, fmt.Errorf("limit должен быть целым числом от 1 до %d", maxLimit) + } + return limit, nil +} + +func oneQueryValue(c echo.Context, name string) (string, bool, error) { + values, present := c.QueryParams()[name] + if !present { + return "", false, nil + } + if len(values) != 1 { + return "", true, errors.New("query parameter must occur once") + } + return values[0], true, nil +} diff --git a/internal/app/chat/service.go b/internal/app/chat/service.go new file mode 100644 index 0000000..d7fa368 --- /dev/null +++ b/internal/app/chat/service.go @@ -0,0 +1,76 @@ +package chat + +import ( + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "os" + "sync/atomic" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/swaggest/swgui/v5emb" +) + +// Chat is the HTTP handler and depends only on the application boundary. +type Chat struct { + attachments *attachment.Service + instanceID string + draining atomic.Bool + useCase UseCase + cursor cursorCodec +} + +func NewChat(useCase UseCase, cursorKey [32]byte) *Chat { + instanceID := os.Getenv("INSTANCE_ID") + if instanceID == "" { + instanceID, _ = os.Hostname() + } + if instanceID == "" { + instanceID = uuid.NewString() + } + return &Chat{useCase: useCase, cursor: cursorCodec{secret: cursorKey}, instanceID: instanceID} +} + +func (h *Chat) Drain() { h.draining.Store(true) } + +func (h *Chat) Router() *echo.Echo { + e := echo.New() + e.HideBanner = true + e.HidePort = true + e.HTTPErrorHandler = h.httpErrorHandler + e.Use(h.requestIDMiddleware) + e.Use(observability.Middleware) + e.Use(middleware.Recover()) + + e.GET("/health/live", h.liveness) + e.GET("/health/ready", h.readiness) + swaggerUI := echo.WrapHandler(v5emb.New( + "MTUSI Messenger", + "/swagger/openapi.json", + "/swagger/", + )) + e.GET("/swagger", h.swagger) + e.GET("/swagger/", swaggerUI) + e.GET("/swagger/openapi.json", h.swaggerSpec) + e.GET("/swagger/*", swaggerUI) + e.POST("/api/v1/users", h.createUser) + + e.POST("/api/v1/auth/sessions", h.createSession) + + api := e.Group("/api/v1", h.authenticate) + api.GET("/users/me", h.currentUser) + api.DELETE("/auth/sessions/current", h.deleteSession) + api.POST("/chats", h.createChat) + api.GET("/chats", h.listChats) + api.GET("/chats/:chat_id", h.getChat) + api.POST("/chats/:chat_id/messages", h.createMessage) + api.GET("/chats/:chat_id/messages", h.listMessages) + if h.attachments != nil { + api.POST("/chats/:chat_id/attachments", h.uploadAttachment) + api.GET("/attachments/:attachment_id", h.getAttachment) + api.GET("/attachments/:attachment_id/content", h.attachmentContent) + } + + return e +} diff --git a/internal/app/chat/service_test.go b/internal/app/chat/service_test.go new file mode 100644 index 0000000..bf582c2 --- /dev/null +++ b/internal/app/chat/service_test.go @@ -0,0 +1,353 @@ +package chat_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + chathandler "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +type testAPI struct { + server *httptest.Server + tokens sync.Map +} + +type userResponse struct { + ID string `json:"id"` +} + +type chatResponse struct { + ID string `json:"id"` +} + +type messageResponse struct { + ID string `json:"id"` + Text string `json:"text"` +} + +type messagePageResponse struct { + Items []messageResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type errorResponse struct { + Error struct { + Code string `json:"code"` + } `json:"error"` +} + +func newTestAPI(t *testing.T) *testAPI { + t.Helper() + useCase := chatusecase.New(repository.NewMemory(), &testSessions{values: map[string]domain.Session{}}, time.Hour) + server := httptest.NewServer(chathandler.NewChat(useCase, [32]byte{1}).Router()) + t.Cleanup(server.Close) + return &testAPI{server: server} +} + +func (a *testAPI) request(t *testing.T, client *http.Client, method, path, actorID, body string, wantStatus int, target any) { + t.Helper() + request, err := http.NewRequest(method, a.server.URL+path, bytes.NewBufferString(body)) + if err != nil { + t.Fatalf("create request: %v", err) + } + if body != "" { + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + } + if actorID != "" { + token, _ := a.tokens.Load(actorID) + request.Header.Set("Authorization", "Bearer "+token.(string)) + } + response, err := client.Do(request) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer response.Body.Close() + responseBody, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read response: %v", err) + } + if response.StatusCode != wantStatus { + t.Fatalf("%s %s: status=%d, want=%d, body=%s", method, path, response.StatusCode, wantStatus, responseBody) + } + if response.Header.Get("X-Request-Id") == "" { + t.Fatalf("%s %s: missing X-Request-Id", method, path) + } + if target != nil { + if err := json.Unmarshal(responseBody, target); err != nil { + t.Fatalf("decode response %q: %v", responseBody, err) + } + } +} + +func (a *testAPI) createUser(t *testing.T, username string) userResponse { + t.Helper() + var user userResponse + a.request(t, http.DefaultClient, http.MethodPost, "/api/v1/users", "", + fmt.Sprintf(`{"username":%q,"display_name":%q,"password":"test-password-12345"}`, username, username), http.StatusCreated, &user) + req, err := http.NewRequest(http.MethodPost, a.server.URL+"/api/v1/auth/sessions", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(username+":test-password-12345"))) + response, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var login struct { + Token string `json:"token"` + } + if response.StatusCode != http.StatusCreated { + t.Fatalf("login status: %d", response.StatusCode) + } + if err = json.NewDecoder(response.Body).Decode(&login); err != nil { + t.Fatal(err) + } + a.tokens.Store(user.ID, login.Token) + return user +} + +func (a *testAPI) requestError(t *testing.T, method, path, actorID, body string, wantStatus int, wantCode string) { + t.Helper() + var response errorResponse + a.request(t, http.DefaultClient, method, path, actorID, body, wantStatus, &response) + if response.Error.Code != wantCode { + t.Fatalf("%s %s: error code=%q, want=%q", method, path, response.Error.Code, wantCode) + } +} + +func (a *testAPI) createChat(t *testing.T, ownerID string, memberIDs []string) chatResponse { + t.Helper() + body, err := json.Marshal(map[string]any{"title": "test chat", "member_ids": memberIDs}) + if err != nil { + t.Fatalf("encode chat: %v", err) + } + var chat chatResponse + a.request(t, http.DefaultClient, http.MethodPost, "/api/v1/chats", ownerID, string(body), http.StatusCreated, &chat) + return chat +} + +func (a *testAPI) createMessage(t *testing.T, client *http.Client, actorID, chatID, text string) messageResponse { + t.Helper() + body, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + t.Fatalf("encode message: %v", err) + } + var message messageResponse + a.request(t, client, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", actorID, string(body), http.StatusCreated, &message) + return message +} + +func TestRejectsMalformedJSON(t *testing.T) { + api := newTestAPI(t) + api.requestError(t, http.MethodPost, "/api/v1/users", "", + `{"username":`, http.StatusBadRequest, "invalid_request") +} + +func TestHidesForeignChat(t *testing.T) { + api := newTestAPI(t) + owner := api.createUser(t, "owner_user") + outsider := api.createUser(t, "outsider_user") + chat := api.createChat(t, owner.ID, []string{owner.ID}) + api.requestError(t, http.MethodGet, "/api/v1/chats/"+chat.ID, outsider.ID, + "", http.StatusNotFound, "not_found") +} + +func TestRejectsEmptyMessage(t *testing.T) { + api := newTestAPI(t) + user := api.createUser(t, "empty_message_user") + chat := api.createChat(t, user.ID, []string{user.ID}) + api.requestError(t, http.MethodPost, "/api/v1/chats/"+chat.ID+"/messages", user.ID, + `{"text":" "}`, http.StatusBadRequest, "invalid_request") +} + +func TestRejectsDuplicateUsername(t *testing.T) { + api := newTestAPI(t) + api.createUser(t, "duplicate_user") + api.requestError(t, http.MethodPost, "/api/v1/users", "", + `{"username":"duplicate_user","display_name":"Duplicate","password":"test-password-12345"}`, http.StatusConflict, "conflict") +} + +func TestPaginatesMessagesAcrossMultiplePages(t *testing.T) { + api := newTestAPI(t) + user := api.createUser(t, "pagination_user") + chat := api.createChat(t, user.ID, []string{user.ID}) + for index := 0; index < 7; index++ { + api.createMessage(t, http.DefaultClient, user.ID, chat.ID, fmt.Sprintf("message-%d", index)) + } + + seen := make(map[string]struct{}) + cursor := "" + pageSizes := make([]int, 0, 4) + for { + path := "/api/v1/chats/" + chat.ID + "/messages?limit=2" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + var page messagePageResponse + api.request(t, http.DefaultClient, http.MethodGet, path, user.ID, "", http.StatusOK, &page) + pageSizes = append(pageSizes, len(page.Items)) + for _, message := range page.Items { + if _, duplicate := seen[message.ID]; duplicate { + t.Fatalf("message %s appeared on multiple pages", message.ID) + } + seen[message.ID] = struct{}{} + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if len(seen) != 7 { + t.Fatalf("received %d unique messages, want 7", len(seen)) + } + wantPageSizes := []int{2, 2, 2, 1} + if fmt.Sprint(pageSizes) != fmt.Sprint(wantPageSizes) { + t.Fatalf("page sizes=%v, want=%v", pageSizes, wantPageSizes) + } +} + +func TestConcurrentMessagesAreUnique(t *testing.T) { + api := newTestAPI(t) + + users := make([]userResponse, 10) + memberIDs := make([]string, 10) + for index := range users { + users[index] = api.createUser(t, fmt.Sprintf("parallel_user_%02d", index)) + memberIDs[index] = users[index].ID + } + chat := api.createChat(t, users[0].ID, memberIDs) + + start := make(chan struct{}) + errors := make(chan error, 10) + var waitGroup sync.WaitGroup + + for clientIndex := 0; clientIndex < 10; clientIndex++ { + waitGroup.Add(1) + go func(clientIndex int) { + defer waitGroup.Done() + client := &http.Client{Transport: &http.Transport{}, Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + <-start + for messageIndex := 0; messageIndex < 10; messageIndex++ { + body, err := json.Marshal(map[string]string{ + "text": fmt.Sprintf("client-%02d-message-%02d", clientIndex, messageIndex), + }) + if err != nil { + errors <- err + return + } + request, err := http.NewRequest(http.MethodPost, + api.server.URL+"/api/v1/chats/"+chat.ID+"/messages", bytes.NewReader(body)) + if err != nil { + errors <- err + return + } + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + token, _ := api.tokens.Load(users[clientIndex].ID) + request.Header.Set("Authorization", "Bearer "+token.(string)) + response, err := client.Do(request) + if err != nil { + errors <- err + return + } + _, readErr := io.Copy(io.Discard, response.Body) + closeErr := response.Body.Close() + if readErr != nil { + errors <- readErr + return + } + if closeErr != nil { + errors <- closeErr + return + } + if response.StatusCode != http.StatusCreated { + errors <- fmt.Errorf("client %d message %d: status=%d", clientIndex, messageIndex, response.StatusCode) + return + } + } + }(clientIndex) + } + close(start) + waitGroup.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + + seenIDs := make(map[string]struct{}, 10*10) + seenTexts := make(map[string]struct{}, 10*10) + cursor := "" + for { + path := "/api/v1/chats/" + chat.ID + "/messages?limit=17" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + var page messagePageResponse + api.request(t, http.DefaultClient, http.MethodGet, path, users[0].ID, "", http.StatusOK, &page) + for _, message := range page.Items { + if _, duplicate := seenIDs[message.ID]; duplicate { + t.Fatalf("duplicate message ID %s", message.ID) + } + if _, duplicate := seenTexts[message.Text]; duplicate { + t.Fatalf("duplicate message text %q", message.Text) + } + seenIDs[message.ID] = struct{}{} + seenTexts[message.Text] = struct{}{} + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if len(seenIDs) != 10*10 || len(seenTexts) != 10*10 { + t.Fatalf("received %d unique IDs and %d unique texts, want %d of each", + len(seenIDs), len(seenTexts), 10*10) + } +} + +// Session fake is only used by HTTP unit tests; the server always uses Redis. +type testSessions struct { + mu sync.Mutex + values map[string]domain.Session +} + +func (s *testSessions) Ready(context.Context) error { return nil } +func (s *testSessions) Put(_ context.Context, key string, value domain.Session) error { + s.mu.Lock() + defer s.mu.Unlock() + s.values[key] = value + return nil +} +func (s *testSessions) Get(_ context.Context, key string) (domain.Session, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.values[key] + return value, ok, nil +} +func (s *testSessions) Delete(_ context.Context, key string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.values[key] + delete(s.values, key) + return ok, nil +} diff --git a/internal/app/chat/session.go b/internal/app/chat/session.go new file mode 100644 index 0000000..29ce755 --- /dev/null +++ b/internal/app/chat/session.go @@ -0,0 +1,45 @@ +package chat + +import ( + "io" + "net/http" + "strings" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +func bearerToken(c echo.Context) string { + parts := strings.Fields(c.Request().Header.Get("Authorization")) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + return "" + } + return parts[1] +} + +func (h *Chat) createSession(c echo.Context) error { + username, password, ok := c.Request().BasicAuth() + if !ok { + c.Response().Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + return h.writeError(c, http.StatusUnauthorized, "unauthorized", "Неверные учётные данные") + } + body, err := io.ReadAll(io.LimitReader(c.Request().Body, 1)) + if err != nil || len(body) != 0 { + return h.invalid(c, "Тело запроса входа должно быть пустым") + } + login, err := h.useCase.CreateSession(c.Request().Context(), username, password) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, dto.CreateSessionResponse{ + Token: login.Token, TokenType: "Bearer", ExpiresAt: dto.Timestamp(login.ExpiresAt), User: createUserResponseFromDomain(login.User), + }) +} + +func (h *Chat) deleteSession(c echo.Context) error { + if err := h.useCase.DeleteSession(c.Request().Context(), bearerToken(c)); err != nil { + return h.useCaseError(c, err) + } + return c.NoContent(http.StatusNoContent) +} diff --git a/internal/app/chat/swagger.go b/internal/app/chat/swagger.go new file mode 100644 index 0000000..fc87c65 --- /dev/null +++ b/internal/app/chat/swagger.go @@ -0,0 +1,38 @@ +package chat + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v4" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/contracts" +) + +var swaggerOpenAPI = makeSwaggerOpenAPI() + +func (h *Chat) swagger(c echo.Context) error { + return c.Redirect(http.StatusTemporaryRedirect, "/swagger/") +} + +func (h *Chat) swaggerSpec(c echo.Context) error { + return c.Blob(http.StatusOK, echo.MIMEApplicationJSON, swaggerOpenAPI) +} + +func makeSwaggerOpenAPI() []byte { + var document map[string]any + if err := json.Unmarshal(contracts.OpenAPI, &document); err != nil { + panic("cannot decode embedded OpenAPI contract: " + err.Error()) + } + // A relative URL keeps Swagger requests on the same scheme, host and port + // as the UI. This also works behind a reverse proxy and on a custom PORT. + document["servers"] = []map[string]string{{ + "url": "/", + "description": "Текущий адрес Swagger UI", + }} + encoded, err := json.Marshal(document) + if err != nil { + panic("cannot encode Swagger OpenAPI contract: " + err.Error()) + } + return encoded +} diff --git a/internal/app/chat/usecase.go b/internal/app/chat/usecase.go new file mode 100644 index 0000000..811a6ef --- /dev/null +++ b/internal/app/chat/usecase.go @@ -0,0 +1,21 @@ +package chat + +import ( + "context" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +// UseCase is the application boundary required by the HTTP handler. +type UseCase interface { + Ready(context.Context) error + CreateSession(context.Context, string, string) (domain.Login, error) + DeleteSession(context.Context, string) error + Authenticate(ctx context.Context, token string) (domain.User, error) + CreateUser(ctx context.Context, user domain.User, password string) (domain.User, error) + CreateChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) + GetChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) + ListChats(ctx context.Context, callerID string, pagination domain.Pagination) (domain.ChatPage, error) + CreateMessage(ctx context.Context, callerID string, message domain.Message) (domain.Message, error) + ListMessages(ctx context.Context, callerID string, chat domain.Chat, pagination domain.Pagination) (domain.MessagePage, error) +} diff --git a/internal/attachment/async.go b/internal/attachment/async.go new file mode 100644 index 0000000..b6b3894 --- /dev/null +++ b/internal/attachment/async.go @@ -0,0 +1,158 @@ +package attachment + +import ( + "context" + "encoding/json" + "fmt" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "os" + "strconv" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +type Options struct { + Concurrency, Prefetch, MaxOutstanding, MaxAttempts, PixelBudget int + Timeout, Lease time.Duration +} + +func LoadOptions() (Options, error) { + o := Options{} + for _, v := range []struct { + name string + target *int + def, max int + }{ + {"WORKER_CONCURRENCY", &o.Concurrency, 2, 8}, {"WORKER_PREFETCH", &o.Prefetch, 4, 32}, + {"MAX_OUTSTANDING_JOBS", &o.MaxOutstanding, 200, 10000}, {"WORKER_MAX_ATTEMPTS", &o.MaxAttempts, 5, 20}, + {"WORKER_PIXEL_BUDGET", &o.PixelBudget, 40000000, 80000000}, + } { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var err error + n, err = strconv.Atoi(raw) + if err != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > v.max { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = n + } + for _, v := range []struct { + name string + target *time.Duration + def int + }{{"WORKER_TIMEOUT_SECONDS", &o.Timeout, 25}, {"WORKER_LEASE_SECONDS", &o.Lease, 40}} { + n := v.def + if raw := os.Getenv(v.name); raw != "" { + var e error + n, e = strconv.Atoi(raw) + if e != nil { + return o, fmt.Errorf("invalid %s", v.name) + } + } + if n < 1 || n > 300 { + return o, fmt.Errorf("invalid %s", v.name) + } + *v.target = time.Duration(n) * time.Second + } + if o.Lease < o.Timeout+5*time.Second { + return o, fmt.Errorf("worker lease must exceed timeout by at least 5s") + } + if o.PixelBudget < int(MaxPixels) { + return o, fmt.Errorf("WORKER_PIXEL_BUDGET must allow one 40MP image") + } + return o, nil +} +func NewAsync(pool *pgxpool.Pool, store *Store, o Options) *Service { + s := New(pool, store) + s.Async = true + s.Options = o + return s +} +func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body interface{ Read([]byte) (int, error) }) (Attachment, error) { + if !s.Async { + return s.uploadSync(ctx, chatID, userID, name, contentType, body) + } + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // All API replicas serialize admission, including inserts before their COMMIT. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return Attachment{}, unavailable() + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return Attachment{}, unavailable() + } + if pending >= s.Options.MaxOutstanding { + return Attachment{}, appError("rate_limited", "Очередь заполнена; повторите загрузку позднее") + } + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "queued", Variants: []any{}, PipelineVersion: 1} + stage := "staging/" + a.ID + a.Key = effectKey(a.ID, 1, "original") + // The stage lock protects even a slow PUT from the orphan cleaner. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, stage); err != nil { + return Attachment{}, unavailable() + } + if err = s.put(ctx, stage, original.File, original.Size, original.ContentType); err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height,status,staging_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,'queued',$11) RETURNING created_at,updated_at`, a.ID, chatID, userID, name, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height, stage).Scan(&a.CreatedAt, &a.UpdatedAt) + if err != nil { + return Attachment{}, unavailable() + } + job := uuid.NewString() + traceparent, tracestate := observability.Inject(ctx) + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts,traceparent,tracestate) VALUES($1,$2,'queued',$3,$4,$5)`, job, a.ID, s.Options.MaxAttempts, traceparent, tracestate); err != nil { + return Attachment{}, unavailable() + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_outbox(job_id) VALUES($1)`, job); err != nil { + return Attachment{}, unavailable() + } + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + return a, nil +} +func effectKey(id string, version int, name string) string { + return fmt.Sprintf("original/%s/v%d/%s", id, version, name) +} +func (s *Service) Variant(a Attachment, name string) (Attachment, error) { + if name == "original" { + return a, nil + } + for _, item := range a.Variants { + raw, _ := json.Marshal(item) + var v Variant + if json.Unmarshal(raw, &v) == nil && v.Name == name { + a.Key = effectKey(a.ID, a.PipelineVersion, name) + a.ContentType = v.ContentType + return a, nil + } + } + return Attachment{}, appError(domain.ErrorNotFound, "Вариант не найден") +} diff --git a/internal/attachment/image.go b/internal/attachment/image.go new file mode 100644 index 0000000..3e688cf --- /dev/null +++ b/internal/attachment/image.go @@ -0,0 +1,128 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "os" + "strings" + "unicode" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const MaxBytes int64 = 10 * 1024 * 1024 +const MaxPixels int64 = 40_000_000 + +func appError(code domain.ErrorCode, message string) error { + return domain.NewApplicationError(code, message) +} +func unavailable() error { + return appError(domain.ErrorUnavailable, "Хранилище вложений недоступно") +} +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return appError(domain.ErrorInvalid, "Некорректный filename") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return appError(domain.ErrorInvalid, "Недопустимые символы filename") + } + } + return nil +} + +type Original struct { + File *os.File + Size int64 + SHA256, ContentType string + Width, Height int +} + +func (o *Original) Close() { name := o.File.Name(); o.File.Close(); os.Remove(name) } + +// Spool bounds compressed bytes on disk before any decode. DecodeConfig checks the +// pixel budget before a full decode verifies CRCs/truncated scan data. +func Spool(ctx context.Context, body io.Reader, declared string) (*Original, error) { + media, _, err := mime.ParseMediaType(declared) + if err != nil || (media != "image/png" && media != "image/jpeg") { + return nil, appError("unsupported_media_type", "Допустимы PNG и JPEG") + } + f, err := os.CreateTemp("", "messenger-upload-*") + if err != nil { + return nil, unavailable() + } + o := &Original{File: f} + good := false + defer func() { + if !good { + o.Close() + } + }() + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hash), io.LimitReader(&contextReader{ctx: ctx, r: body}, MaxBytes+1)) + if n > MaxBytes { + return nil, appError("payload_too_large", "Файл превышает 10 MiB") + } + if err != nil { + return nil, appError(domain.ErrorInvalid, "Загрузка прервана") + } + o.Size = n + o.SHA256 = hex.EncodeToString(hash.Sum(nil)) + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + cfg, format, err := image.DecodeConfig(f) + if err != nil { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + actual := "image/" + format + if actual != media { + return nil, appError("unsupported_media_type", "MIME не соответствует содержимому") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return nil, appError("payload_too_large", "Изображение превышает 40 миллионов пикселей") + } + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + decoded, format, err := image.Decode(f) + if err != nil || decoded == nil || "image/"+format != actual { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + if decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height { + return nil, appError("unsupported_media_type", "Некорректные размеры изображения") + } + decoded = nil + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + o.ContentType = actual + o.Width = cfg.Width + o.Height = cfg.Height + good = true + return o, nil +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r *contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + n, err := r.r.Read(p) + if errors.Is(err, context.Canceled) { + return n, err + } + return n, err +} diff --git a/internal/attachment/image_test.go b/internal/attachment/image_test.go new file mode 100644 index 0000000..30700b7 --- /dev/null +++ b/internal/attachment/image_test.go @@ -0,0 +1,139 @@ +package attachment + +import ( + "bytes" + "context" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func pngFixture(t *testing.T) []byte { + t.Helper() + m := image.NewRGBA(image.Rect(0, 0, 10, 8)) + m.Set(1, 1, color.RGBA{R: 255, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, m); err != nil { + t.Fatal(err) + } + return b.Bytes() +} +func code(t *testing.T, err error, want domain.ErrorCode) { + t.Helper() + app, ok := domain.AsApplicationError(err) + if !ok || app.Code != want { + t.Fatalf("want %s, got %v", want, err) + } +} +func TestSpoolOriginalAndFullValidation(t *testing.T) { + original := pngFixture(t) + good, err := Spool(context.Background(), bytes.NewReader(original), "image/png") + if err != nil { + t.Fatal(err) + } + defer good.Close() + got, err := io.ReadAll(good.File) + if err != nil || !bytes.Equal(got, original) { + t.Fatal("original changed") + } + if good.Width != 10 || good.Height != 8 || good.Size != int64(len(original)) || len(good.SHA256) != 64 { + t.Fatalf("wrong metadata %+v", good) + } + _, err = Spool(context.Background(), bytes.NewReader(original), "image/jpeg") + code(t, err, "unsupported_media_type") + _, err = Spool(context.Background(), bytes.NewReader(original[:len(original)-12]), "image/png") + code(t, err, "unsupported_media_type") + corrupted := bytes.Clone(original) + corrupted[len(corrupted)-18] ^= 0x80 + _, err = Spool(context.Background(), bytes.NewReader(corrupted), "image/png") + code(t, err, "unsupported_media_type") + var jpegBytes bytes.Buffer + if err = jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 12, 9)), nil); err != nil { + t.Fatal(err) + } + jpegOriginal, err := Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()), "image/jpeg") + if err != nil { + t.Fatal(err) + } + jpegOriginal.Close() + _, err = Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()[:jpegBytes.Len()/2]), "image/jpeg") + code(t, err, "unsupported_media_type") +} +func TestSpoolLimitsBeforeDecode(t *testing.T) { + // Oversized compressed input is stopped after exactly MaxBytes+1 reads. + source := &countingReader{remaining: MaxBytes + 100_000} + _, err := Spool(context.Background(), source, "image/png") + code(t, err, "payload_too_large") + if source.read != MaxBytes+1 { + t.Fatalf("read %d bytes", source.read) + } + bomb := bytes.Clone(pngFixture(t)) + binary.BigEndian.PutUint32(bomb[16:20], 40001) + binary.BigEndian.PutUint32(bomb[20:24], 1000) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + _, err = Spool(context.Background(), bytes.NewReader(bomb), "image/png") + code(t, err, "payload_too_large") +} + +type countingReader struct{ remaining, read int64 } + +func (c *countingReader) Read(p []byte) (int, error) { + if c.remaining == 0 { + return 0, io.EOF + } + n := min(int64(len(p)), c.remaining) + clear(p[:n]) + c.remaining -= n + c.read += n + return int(n), nil +} +func TestFilenameAndCancelledUpload(t *testing.T) { + for _, name := range []string{"", "../x.png", "x\\photo.jpg", "a\x00b", "a\nb", "\t"} { + code(t, ValidateFilename(name), domain.ErrorInvalid) + } + for _, name := range []string{"фото.png", "image.jpg", ".."} { + if err := ValidateFilename(name); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Spool(ctx, bytes.NewReader(pngFixture(t)), "image/png") + code(t, err, domain.ErrorInvalid) +} +func TestReconcileRejectsUnsafeGrace(t *testing.T) { + s := &Service{} + if _, err := s.Reconcile(context.Background(), true, 89*time.Second); err == nil { + t.Fatal("unsafe grace accepted") + } + if MinimumGrace <= UploadTimeout { + t.Fatal("grace does not cover in-flight PUT") + } +} + +func TestValid16BitWidePNG(t *testing.T) { + // Exercise the same RGBA64 + scanline allocation path as an extreme-aspect + // 40M image, at 1/1000 of that pixel count to keep the unit suite small. + img := image.NewNRGBA64(image.Rect(0, 0, 40000, 1)) + img.SetNRGBA64(1, 0, color.NRGBA64{R: 65535, G: 12345, B: 4, A: 32000}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, img); err != nil { + t.Fatal(err) + } + original, err := Spool(context.Background(), bytes.NewReader(encoded.Bytes()), "image/png") + if err != nil { + t.Fatal(err) + } + defer original.Close() + if original.Width != 40000 || original.Height != 1 { + t.Fatal("wide PNG dimensions changed") + } +} diff --git a/internal/attachment/json.go b/internal/attachment/json.go new file mode 100644 index 0000000..cb3958f --- /dev/null +++ b/internal/attachment/json.go @@ -0,0 +1,18 @@ +package attachment + +import ( + "encoding/json" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +// MarshalJSON uses the API's fixed microsecond precision instead of time.Time's +// variable fractional precision, which Python 3.9 clients cannot always parse. +func (a Attachment) MarshalJSON() ([]byte, error) { + type fields Attachment + return json.Marshal(struct { + fields + CreatedAt dto.Timestamp `json:"created_at"` + UpdatedAt dto.Timestamp `json:"updated_at"` + }{fields: fields(a), CreatedAt: dto.Timestamp(a.CreatedAt), UpdatedAt: dto.Timestamp(a.UpdatedAt)}) +} diff --git a/internal/attachment/json_test.go b/internal/attachment/json_test.go new file mode 100644 index 0000000..4812d60 --- /dev/null +++ b/internal/attachment/json_test.go @@ -0,0 +1,37 @@ +package attachment + +import ( + "encoding/json" + "testing" + "time" +) + +func TestAttachmentTimestampsUseFixedMicroseconds(t *testing.T) { + value := Attachment{ID: "attachment-id", CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 123400000, time.UTC), UpdatedAt: time.Date(2026, 1, 2, 6, 4, 5, 0, time.FixedZone("MSK", 3*60*60)), Key: "original/private", Variants: []any{}} + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err = json.Unmarshal(encoded, &fields); err != nil { + t.Fatal(err) + } + if fields["created_at"] != "2026-01-02T03:04:05.123400Z" { + t.Fatalf("created_at=%v", fields["created_at"]) + } + if fields["updated_at"] != "2026-01-02T03:04:05.000000Z" { + t.Fatalf("updated_at=%v", fields["updated_at"]) + } + if fields["id"] != value.ID { + t.Fatal("serialization lost metadata") + } + if _, exposed := fields["Key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if _, exposed := fields["key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if fields["error_code"] != nil { + t.Fatal("ready error_code must be null") + } +} diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 0000000..755338f --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,225 @@ +package attachment + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" +) + +type Summary struct { + Apply bool `json:"apply"` + GraceSeconds float64 `json:"grace_seconds"` + ObjectsScanned int `json:"objects_scanned"` + MetadataScanned int `json:"metadata_scanned"` + BytesScanned int64 `json:"bytes_scanned"` + ListRequestsEstimate int `json:"list_requests_estimate"` + StatRequests int `json:"stat_requests"` + DBQueries int `json:"db_queries"` + Orphans int `json:"orphans"` + Missing int `json:"missing"` + FreshSkipped int `json:"fresh_skipped"` + LockedSkipped int `json:"locked_skipped"` + ReferencedMissing int `json:"referenced_missing"` + Deleted int `json:"deleted"` + MarkedFailed int `json:"marked_failed"` + DurationMS int64 `json:"duration_ms"` + Actions []Action `json:"actions"` +} +type Action struct { + Kind string `json:"kind"` + Key string `json:"key"` + Applied bool `json:"applied"` +} + +// Reconcile scans a fixed cutoff and rechecks each candidate while holding the +// same transaction-scoped advisory lock as upload. Metadata, including message +// references, is never deleted. Missing originals become failed tombstones. +func (s *Service) Reconcile(ctx context.Context, apply bool, grace time.Duration) (result Summary, err error) { + started := time.Now() + result = Summary{Apply: apply, GraceSeconds: grace.Seconds(), Actions: []Action{}} + defer func() { result.DurationMS = time.Since(started).Milliseconds() }() + if grace < MinimumGrace { + return result, fmt.Errorf("grace must be at least %s (upload timeout %s)", MinimumGrace, UploadTimeout) + } + cutoff := started.Add(-grace) + for obj := range s.Store.Client.ListObjects(ctx, s.Store.Bucket, minio.ListObjectsOptions{Prefix: "original/", Recursive: true, MaxKeys: 1000}) { + if obj.Err != nil { + return result, fmt.Errorf("S3 listing failed") + } + result.ObjectsScanned++ + result.BytesScanned += obj.Size + if result.ObjectsScanned%1000 == 1 { + result.ListRequestsEstimate++ + } + if !obj.LastModified.Before(cutoff) { + result.FreshSkipped++ + continue + } + err = s.reconcileObject(ctx, obj.Key, cutoff, apply, &result) + if err != nil { + return result, err + } + } + // Bounded keyset pages avoid retaining the entire bucket or a DB cursor while + // making per-object transactions. A later run picks up concurrent new rows. + last := "00000000-0000-0000-0000-000000000000" + for { + result.DBQueries++ + rows, e := s.Pool.Query(ctx, `SELECT id::text,object_key,created_at,status FROM attachments WHERE id>$1 ORDER BY id LIMIT 100`, last) + if e != nil { + return result, fmt.Errorf("metadata scan failed") + } + type metadata struct { + id, key, status string + created time.Time + } + page := []metadata{} + for rows.Next() { + var m metadata + if e = rows.Scan(&m.id, &m.key, &m.created, &m.status); e != nil { + rows.Close() + return result, e + } + page = append(page, m) + } + e = rows.Err() + rows.Close() + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, m := range page { + last = m.id + result.MetadataScanned++ + if m.status != "ready" { + continue + } + if !m.created.Before(cutoff) { + result.FreshSkipped++ + continue + } + if e = s.reconcileMetadata(ctx, m.id, m.key, cutoff, apply, &result); e != nil { + return result, e + } + } + } + if ctx.Err() != nil { + return result, ctx.Err() + } + return result, nil +} +func (s *Service) locked(ctx context.Context, key string, r *Summary) (pgx.Tx, bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return nil, false, fmt.Errorf("DB unavailable") + } + var locked bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, key).Scan(&locked); err != nil { + tx.Rollback(context.Background()) + return nil, false, fmt.Errorf("lock failed") + } + if !locked { + r.LockedSkipped++ + tx.Rollback(context.Background()) + return nil, false, nil + } + return tx, true, nil +} +func (s *Service) reconcileObject(ctx context.Context, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var exists bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR $1 LIKE 'original/'||id::text||'/v'||pipeline_version::text||'/%')`, key).Scan(&exists); err != nil { + return fmt.Errorf("metadata recheck failed") + } + if exists { + return nil + } + r.StatRequests++ + obj, err := s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if IsMissing(err) { + return nil + } + if err != nil { + return fmt.Errorf("object recheck failed") + } + if !obj.LastModified.Before(cutoff) { + r.FreshSkipped++ + return nil + } + r.Orphans++ + if apply { + if err = s.Store.Client.RemoveObject(ctx, s.Store.Bucket, key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("orphan deletion failed") + } + r.Deleted++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "orphan", Key: key, Applied: apply}) + } + return nil +} +func (s *Service) reconcileMetadata(ctx context.Context, id, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var created time.Time + var status string + r.DBQueries++ + err = tx.QueryRow(ctx, `SELECT created_at,status FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&created, &status) + if err == pgx.ErrNoRows { + return nil + } + if err != nil { + return fmt.Errorf("metadata recheck failed") + } + if status != "ready" || !created.Before(cutoff) { + return nil + } + r.StatRequests++ + _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if err == nil { + return nil + } + if !IsMissing(err) { + return fmt.Errorf("object stat failed (no mutation)") + } + var referenced bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE $1::uuid=ANY(attachment_ids))`, id).Scan(&referenced); err != nil { + return fmt.Errorf("reference check failed") + } + r.Missing++ + if referenced { + r.ReferencedMissing++ + } + if apply { + r.DBQueries++ + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing',updated_at=clock_timestamp() WHERE id=$1 AND status='ready'`, id); err != nil { + return fmt.Errorf("mark missing failed") + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='failed',last_error='original_missing',fence=fence+1,updated_at=clock_timestamp() WHERE attachment_id=$1 AND state='ready'`, id); err != nil { + return fmt.Errorf("mark job missing failed") + } + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit missing failed") + } + r.MarkedFailed++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "missing", Key: key, Applied: apply}) + } + return nil +} diff --git a/internal/attachment/render.go b/internal/attachment/render.go new file mode 100644 index 0000000..e6cebce --- /dev/null +++ b/internal/attachment/render.go @@ -0,0 +1,267 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "math" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +var ErrInvalidImage = errors.New("invalid image") + +type oriented struct { + image.Image + orientation int +} + +func (o oriented) Bounds() image.Rectangle { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + if o.orientation >= 5 && o.orientation <= 8 { + w, h = h, w + } + return image.Rect(0, 0, w, h) +} +func (o oriented) At(x, y int) color.Color { + b := o.Image.Bounds() + w, h := b.Dx(), b.Dy() + sx, sy := x, y + switch o.orientation { + case 2: + sx = w - 1 - x + case 3: + sx, sy = w-1-x, h-1-y + case 4: + sy = h - 1 - y + case 5: + sx, sy = y, x + case 6: + sx, sy = y, h-1-x + case 7: + sx, sy = w-1-y, h-1-x + case 8: + sx, sy = w-1-y, x + } + r, g, blue, a := o.Image.At(b.Min.X+sx, b.Min.Y+sy).RGBA() + white := uint32(65535) - a + return color.RGBA{uint8((r + white) / 257), uint8((g + white) / 257), uint8((blue + white) / 257), 255} +} + +// EXIF IFD0 orientation; bounded segment parsing does not follow client offsets outside APP1. +func tiffOrientation(t []byte) int { + if len(t) < 8 { + return 1 + } + var order binary.ByteOrder + if string(t[:2]) == "II" { + order = binary.LittleEndian + } else if string(t[:2]) == "MM" { + order = binary.BigEndian + } else { + return 1 + } + if order.Uint16(t[2:4]) != 42 { + return 1 + } + offset := uint64(order.Uint32(t[4:8])) + if offset+2 > uint64(len(t)) { + return 1 + } + count := uint64(order.Uint16(t[offset : offset+2])) + for k := uint64(0); k < count; k++ { + p := offset + 2 + k*12 + if p+12 > uint64(len(t)) { + break + } + entry := t[p : p+12] + if order.Uint16(entry[:2]) == 0x112 && order.Uint16(entry[2:4]) == 3 && order.Uint32(entry[4:8]) == 1 { + v := int(order.Uint16(entry[8:10])) + if v >= 1 && v <= 8 { + return v + } + } + } + return 1 +} +func Orientation(data []byte) int { + if len(data) >= 8 && bytes.Equal(data[:8], []byte{137, 80, 78, 71, 13, 10, 26, 10}) { + for i := 8; i+12 <= len(data); { + n := uint64(binary.BigEndian.Uint32(data[i : i+4])) + if n > uint64(len(data)-i-12) { + return 1 + } + end := i + 8 + int(n) + if string(data[i+4:i+8]) == "eXIf" { + return tiffOrientation(data[i+8 : end]) + } + i = end + 4 + } + return 1 + } + if len(data) < 4 || data[0] != 255 || data[1] != 216 { + return 1 + } + for i := 2; i+4 <= len(data); { + if data[i] != 255 { + return 1 + } + for i+1 < len(data) && data[i+1] == 255 { + i++ + } + if i+3 >= len(data) { + return 1 + } + marker := data[i+1] + i += 2 + if marker == 0xda || marker == 0xd9 { + return 1 + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + n := int(binary.BigEndian.Uint16(data[i : i+2])) + if n < 2 || i+n > len(data) { + return 1 + } + body := data[i+2 : i+n] + i += n + if marker == 0xe1 && len(body) >= 14 && bytes.Equal(body[:6], []byte("Exif\x00\x00")) { + return tiffOrientation(body[6:]) + } + } + return 1 +} + +func fit(w, h, maxSide int) (int, int) { + if w <= maxSide && h <= maxSide { + return w, h + } + if w >= h { + return maxSide, max(1, int(int64(h)*int64(maxSide)/int64(w))) + } + return max(1, int(int64(w)*int64(maxSide)/int64(h))), maxSide +} + +func resize(src image.Image, region image.Rectangle, w, h int) *image.RGBA { + out := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + fy := (float64(y)+.5)*float64(region.Dy())/float64(h) - .5 + fy = math.Max(0, fy) + y0 := min(int(fy), region.Dy()-1) + y1 := min(y0+1, region.Dy()-1) + dy := fy - float64(y0) + for x := 0; x < w; x++ { + fx := (float64(x)+.5)*float64(region.Dx())/float64(w) - .5 + fx = math.Max(0, fx) + x0 := min(int(fx), region.Dx()-1) + x1 := min(x0+1, region.Dx()-1) + dx := fx - float64(x0) + colors := [4]color.RGBA{color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y0)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x0, region.Min.Y+y1)).(color.RGBA), color.RGBAModel.Convert(src.At(region.Min.X+x1, region.Min.Y+y1)).(color.RGBA)} + blend := func(a, b, c, d uint8) uint8 { + return uint8(math.Round(float64(a)*(1-dx)*(1-dy) + float64(b)*dx*(1-dy) + float64(c)*(1-dx)*dy + float64(d)*dx*dy)) + } + out.SetRGBA(x, y, color.RGBA{blend(colors[0].R, colors[1].R, colors[2].R, colors[3].R), blend(colors[0].G, colors[1].G, colors[2].G, colors[3].G), blend(colors[0].B, colors[1].B, colors[2].B, colors[3].B), 255}) + } + } + return out +} + +// Render runs only inside a deadline-killed child, never in the HTTP process. +func Render(ctx context.Context, source, dir, expectedSHA string) error { + raw, err := os.ReadFile(source) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + if int64(len(raw)) > MaxBytes || hex.EncodeToString(sum[:]) != expectedSHA { + return ErrInvalidImage + } + cfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return ErrInvalidImage + } + if os.Getenv("PIPELINE_FAILPOINT") == "renderer_timeout" { + fifo := filepath.Join(dir, "blocked-input.png") + if err = syscall.Mkfifo(fifo, 0600); err != nil { + return err + } + return exec.CommandContext(ctx, "cwebp", "-quiet", fifo, "-o", filepath.Join(dir, "blocked.webp")).Run() + } + decoded, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return ErrInvalidImage + } + src := oriented{decoded, Orientation(raw)} + bounds := src.Bounds() + tw, th := fit(bounds.Dx(), bounds.Dy(), 256) + ow, oh := fit(bounds.Dx(), bounds.Dy(), 1280) + variants := []Variant{} + for _, name := range []string{"thumbnail", "crop", "optimized"} { + w, h := tw, th + region := bounds + if name == "crop" { + w, h = 128, 128 + side := min(bounds.Dx(), bounds.Dy()) + left, top := (bounds.Dx()-side)/2, (bounds.Dy()-side)/2 + region = image.Rect(left, top, left+side, top+side) + } + if name == "optimized" { + w, h = ow, oh + } + img := resize(src, region, w, h) + path := filepath.Join(dir, name) + if name == "optimized" { + input := filepath.Join(dir, "webp-input.png") + f, e := os.Create(input) + if e != nil { + return e + } + e = png.Encode(f, img) + f.Close() + if e != nil { + return e + } + command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", path) + if err = command.Run(); err != nil { + return err + } + } else { + f, e := os.Create(path) + if e != nil { + return e + } + e = jpeg.Encode(f, img, &jpeg.Options{Quality: 85}) + f.Close() + if e != nil { + return e + } + } + data, e := os.ReadFile(path) + if e != nil { + return e + } + hash := sha256.Sum256(data) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + variants = append(variants, Variant{Name: name, ContentType: mime, Width: w, Height: h, Size: int64(len(data)), SHA256: hex.EncodeToString(hash[:])}) + } + data, err := json.Marshal(variants) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "manifest.json"), data, 0600) +} diff --git a/internal/attachment/render_test.go b/internal/attachment/render_test.go new file mode 100644 index 0000000..a5ab246 --- /dev/null +++ b/internal/attachment/render_test.go @@ -0,0 +1,81 @@ +package attachment + +import ( + "bytes" + "encoding/binary" + "image" + "image/color" + "testing" +) + +func TestFitGeometry(t *testing.T) { + for _, c := range []struct{ w, h, limit, x, y int }{{320, 200, 256, 256, 160}, {513, 257, 256, 256, 128}, {257, 513, 256, 128, 256}, {12, 8, 256, 12, 8}, {1, 40000, 256, 1, 256}} { + w, h := fit(c.w, c.h, c.limit) + if w != c.x || h != c.y { + t.Fatalf("fit(%d,%d)=%d,%d", c.w, c.h, w, h) + } + } +} +func TestOrientationsAndAlpha(t *testing.T) { + img := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 3; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(1 + x + y*3), A: 255}) + } + } + expected := [][]uint8{{1, 2, 3, 4, 5, 6}, {3, 2, 1, 6, 5, 4}, {6, 5, 4, 3, 2, 1}, {4, 5, 6, 1, 2, 3}, {1, 4, 2, 5, 3, 6}, {4, 1, 5, 2, 6, 3}, {6, 3, 5, 2, 4, 1}, {3, 6, 2, 5, 1, 4}} + for i, want := range expected { + src := oriented{img, i + 1} + got := []uint8{} + for y := 0; y < src.Bounds().Dy(); y++ { + for x := 0; x < src.Bounds().Dx(); x++ { + got = append(got, src.At(x, y).(color.RGBA).R) + } + } + if !bytes.Equal(got, want) { + t.Fatalf("orientation %d: %v want %v", i+1, got, want) + } + } + img.SetNRGBA(0, 0, color.NRGBA{R: 255, A: 128}) + got := oriented{img, 1}.At(0, 0).(color.RGBA) + if got != (color.RGBA{255, 127, 127, 255}) { + t.Fatalf("white alpha: %v", got) + } +} +func TestEXIFOrientation(t *testing.T) { + for _, order := range []binary.ByteOrder{binary.LittleEndian, binary.BigEndian} { + for v := 1; v <= 8; v++ { + tiff := make([]byte, 26) + copy(tiff, "II") + if order == binary.BigEndian { + copy(tiff, "MM") + } + order.PutUint16(tiff[2:], 42) + order.PutUint32(tiff[4:], 8) + order.PutUint16(tiff[8:], 1) + order.PutUint16(tiff[10:], 0x112) + order.PutUint16(tiff[12:], 3) + order.PutUint32(tiff[14:], 1) + order.PutUint16(tiff[18:], uint16(v)) + data := append([]byte{255, 216, 255, 225, 0, 34}, append([]byte("Exif\x00\x00"), tiff...)...) + if Orientation(data) != v { + t.Fatalf("EXIF value %d", v) + } + pngData := []byte{137, 80, 78, 71, 13, 10, 26, 10} + size := make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(tiff))) + pngData = append(pngData, size...) + pngData = append(pngData, []byte("eXIf")...) + pngData = append(pngData, tiff...) + pngData = append(pngData, 0, 0, 0, 0) + if Orientation(pngData) != v { + t.Fatalf("PNG EXIF value %d", v) + } + } + } + for _, b := range [][]byte{nil, {}, {255, 216, 255, 225, 255, 255}, {255, 216, 255, 255, 225, 0, 2}} { + if Orientation(b) != 1 { + t.Fatal("malformed EXIF should be ignored") + } + } +} diff --git a/internal/attachment/s3.go b/internal/attachment/s3.go new file mode 100644 index 0000000..317cdea --- /dev/null +++ b/internal/attachment/s3.go @@ -0,0 +1,45 @@ +package attachment + +import ( + "context" + "crypto/tls" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "net" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Store struct { + Client *minio.Client + Bucket string +} + +func NewStore(endpoint, bucket, access, secret string, secure bool) (*Store, error) { + transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 3 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, IdleConnTimeout: 30 * time.Second, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 5 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}} + c, err := minio.New(endpoint, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: secure, Region: "us-east-1", Transport: observability.Transport{Base: transport}, MaxRetries: 1}) + if err != nil { + return nil, err + } + return &Store{Client: c, Bucket: bucket}, nil +} +func (s *Store) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + for item := range s.Client.ListObjects(ctx, s.Bucket, minio.ListObjectsOptions{Prefix: "original/", MaxKeys: 1, Recursive: true}) { + if item.Err != nil { + return unavailable() + } + return nil + } + if ctx.Err() != nil { + return unavailable() + } + return nil +} +func IsMissing(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchObject" +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 0000000..fb0a441 --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,138 @@ +package attachment + +import ( + "context" + "encoding/json" + "errors" + "io" + "log" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const UploadTimeout = 45 * time.Second +const MinimumGrace = 90 * time.Second + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Key string `json:"-"` + PipelineVersion int `json:"-"` +} + +type Service struct { + Pool *pgxpool.Pool + Store *Store + slots chan struct{} + Async bool + Options Options +} + +func New(pool *pgxpool.Pool, store *Store) *Service { + return &Service{Pool: pool, Store: store, slots: make(chan struct{}, 1)} +} +func (s *Service) Ready(ctx context.Context) error { return s.Store.Ready(ctx) } +func (s *Service) uploadSync(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "ready", Variants: []any{}} + a.Key = "original/" + a.ID + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // Keep the object lock through PUT, metadata insert and COMMIT. A crash releases + // the lock; the grace protects a still-running client PUT at the S3 server. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, a.Key); err != nil { + return Attachment{}, unavailable() + } + _, err = s.Store.Client.PutObject(ctx, s.Store.Bucket, a.Key, original.File, a.Size, minio.PutObjectOptions{ContentType: a.ContentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": a.SHA256}}) + if err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING created_at`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height).Scan(&a.CreatedAt) + if err != nil { + log.Print("attachment metadata failed; orphan retained for grace-period reconciliation") + return Attachment{}, unavailable() + } + // Never DELETE on an ambiguous COMMIT failure: the DB may have committed. A + // later reconciler checks both stores under the same lock before any cleanup. + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + a.UpdatedAt = a.CreatedAt + return a, nil +} +func (s *Service) Get(ctx context.Context, id, userID string) (Attachment, error) { + parsed, err := uuid.Parse(id) + if err != nil { + return Attachment{}, appError(domain.ErrorInvalid, "Некорректный attachment_id") + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + a := Attachment{Variants: []any{}} + var variants []byte + err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at,a.updated_at,a.variants,a.pipeline_version FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, &variants, &a.PipelineVersion) + if errors.Is(err, pgx.ErrNoRows) { + return Attachment{}, appError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return Attachment{}, unavailable() + } + if err = json.Unmarshal(variants, &a.Variants); err != nil { + return Attachment{}, unavailable() + } + // The legacy fixture service preserves lab4 errors; production Async exposes all states. + if a.Status != "ready" && !s.Async { + return Attachment{}, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return a, nil +} +func (s *Service) Content(ctx context.Context, a Attachment) (*minio.Object, error) { + if a.Status != "ready" { + return nil, appError(domain.ErrorConflict, "Вложение не готово") + } + obj, err := s.Store.Client.GetObject(ctx, s.Store.Bucket, a.Key, minio.GetObjectOptions{}) + if err != nil { + return nil, unavailable() + } + if _, err = obj.Stat(); err != nil { + obj.Close() + if IsMissing(err) { + return nil, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return nil, unavailable() + } + return obj, nil +} diff --git a/internal/attachment/worker.go b/internal/attachment/worker.go new file mode 100644 index 0000000..57110a4 --- /dev/null +++ b/internal/attachment/worker.go @@ -0,0 +1,658 @@ +package attachment + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/observability" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "io" + "log" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/semaphore" +) + +const Stream = "attachments:jobs:v1" +const Group = "image-workers-v1" + +type JobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Width int `json:"width"` + Height int `json:"height"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` +} +type claimed struct { + JobMessage + Fence int64 + Stage, Original, MIME, SHA string + Width, Height int + Size int64 +} + +func (s *Service) put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + ctx, span := observability.Start(ctx, "s3 PUT", trace.SpanKindClient) + defer span.End() + _, err := s.Store.Client.PutObject(ctx, s.Store.Bucket, key, r, size, minio.PutObjectOptions{ContentType: mime, DisableMultipart: true}) + observability.Result(span, err) + return err +} +func (s *Service) Publish(ctx context.Context, r *redis.Client) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT j.id::text,j.attachment_id::text,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.state='queued' ORDER BY o.available_at LIMIT 32 FOR UPDATE OF o SKIP LOCKED`) + if err != nil { + return err + } + messages := []JobMessage{} + for rows.Next() { + m := JobMessage{SchemaVersion: 1, Type: "attachment.process.v1"} + if err = rows.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &m.Traceparent, &m.Tracestate); err != nil { + rows.Close() + return err + } + m.CreatedAt = m.CreatedAt.UTC() + messages = append(messages, m) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, m := range messages { + parent := observability.Extract(ctx, m.Traceparent, m.Tracestate) + publishCtx, span := observability.Start(parent, "attachments publish", trace.SpanKindProducer) + m.Traceparent, m.Tracestate = observability.Inject(publishCtx) + raw, _ := json.Marshal(m) + id, e := r.XAdd(publishCtx, &redis.XAddArgs{Stream: Stream, Values: map[string]interface{}{"job": string(raw)}}).Result() + observability.Result(span, e) + observability.Log(publishCtx, "job.published", map[string]any{"job_type": m.Type, "attempt": m.Attempt}) + span.End() + if e != nil { + return e + } + if _, e = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=clock_timestamp(),stream_id=$2 WHERE job_id=$1`, m.JobID, id); e != nil { + return e + } + } + return tx.Commit(ctx) +} +func (s *Service) recover(ctx context.Context) error { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + rows, err := tx.Query(ctx, `SELECT id::text,attachment_id::text,attempts,max_attempts FROM attachment_jobs WHERE state='processing' AND lease_until= l.max { + state = "failed" + v := "retry_exhausted" + code = &v + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state=$2,lease_until=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),last_error='worker_lost' WHERE id=$1`, l.id, state); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,updated_at=clock_timestamp() WHERE id=$1`, l.att, state, code); err != nil { + return err + } + if state == "queued" { + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, l.id); err != nil { + return err + } + } + } + return tx.Commit(ctx) +} +func (s *Service) claim(ctx context.Context, m JobMessage) (claimed, bool, error) { + j := claimed{JobMessage: m} + tx, err := s.Pool.Begin(ctx) + if err != nil { + return j, false, err + } + defer tx.Rollback(context.Background()) + var state string + var attempts, max, version int + var available time.Time + err = tx.QueryRow(ctx, `SELECT state,attempts,max_attempts,available_at,pipeline_version FROM attachment_jobs WHERE id=$1 AND attachment_id=$2 FOR UPDATE`, m.JobID, m.AttachmentID).Scan(&state, &attempts, &max, &available, &version) + if errors.Is(err, pgx.ErrNoRows) { + return j, false, nil + } + if err != nil { + return j, false, err + } + if state == "ready" { + var key, mime string + var size int64 + var raw []byte + if err = tx.QueryRow(ctx, `SELECT object_key,content_type,size_bytes,variants FROM attachments WHERE id=$1 AND status='ready' AND pipeline_version=$2`, m.AttachmentID, version).Scan(&key, &mime, &size, &raw); err != nil { + return j, false, err + } + var variants []Variant + if err = json.Unmarshal(raw, &variants); err != nil || len(variants) != 3 { + return j, false, fmt.Errorf("completed effect metadata incomplete") + } + objects := map[string]int64{key: size} + for _, v := range variants { + objects[effectKey(m.AttachmentID, version, v.Name)] = v.Size + } + for object, n := range objects { + info, e := s.Store.Client.StatObject(ctx, s.Store.Bucket, object, minio.StatObjectOptions{}) + if e != nil { + return j, false, e + } + if info.Size != n { + return j, false, fmt.Errorf("completed effect size mismatch") + } + } + return j, false, nil + } + if state != "queued" || available.After(time.Now()) || m.Attempt != attempts+1 || version != m.PipelineVersion { + return j, false, nil + } + if attempts >= max { + return j, false, fmt.Errorf("attempt invariant violated") + } + err = tx.QueryRow(ctx, `UPDATE attachment_jobs SET state='processing',attempts=attempts+1,fence=fence+1,lease_until=clock_timestamp()+$2*interval '1 second',updated_at=clock_timestamp() WHERE id=$1 RETURNING fence`, m.JobID, s.Options.Lease.Seconds()).Scan(&j.Fence) + if err != nil { + return j, false, err + } + err = tx.QueryRow(ctx, `UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=$1 AND pipeline_version=$2 AND status IN ('queued','processing') RETURNING staging_key,object_key,content_type,size_bytes,sha256,width,height`, m.AttachmentID, version).Scan(&j.Stage, &j.Original, &j.MIME, &j.Size, &j.SHA, &j.Width, &j.Height) + if err != nil { + return j, false, err + } + return j, true, tx.Commit(ctx) +} + +// finish publishes metadata only for the current, unexpired fencing token. +func (s *Service) finish(ctx context.Context, j claimed, variants []Variant, code string, permanent bool) (bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var max int + err = tx.QueryRow(ctx, `SELECT max_attempts FROM attachment_jobs WHERE id=$1 AND fence=$2 AND state='processing' AND lease_until>clock_timestamp() AND pipeline_version=$3 FOR UPDATE`, j.JobID, j.Fence, j.PipelineVersion).Scan(&max) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + lastError := code + state := "ready" + var failure *string + available := time.Now() + if code != "" { + state = "queued" + if permanent || j.Attempt >= max { + state = "failed" + if !permanent { + code = "retry_exhausted" + } + failure = &code + } + delay := time.Duration(1< 0 { + op, cancel = context.WithTimeout(ctx, time.Second) + streams, e := r.XReadGroup(op, &redis.XReadGroupArgs{Group: Group, Consumer: consumer, Streams: []string{Stream, ">"}, Count: int64(room), Block: 100 * time.Millisecond}).Result() + cancel() + if e == nil { + for _, stream := range streams { + for _, m := range stream.Messages { + select { + case queue <- m: + case <-ctx.Done(): + return nil + } + } + } + } + } + if time.Since(lastCleanup) >= 5*time.Second { + lastCleanup = time.Now() + op, cancel = context.WithTimeout(ctx, 5*time.Second) + s.cleanStages(op) + cancel() + } + } +} +func ack(ctx context.Context, r *redis.Client, id string) { + if r.XAck(ctx, Stream, Group, id).Err() == nil { + r.XDel(ctx, Stream, id) + } +} + +// Replay is an explicit operator action, serialized with admission and job claims. +// It preserves job/effect identity and refuses already successful work. +func (s *Service) Replay(ctx context.Context, id, operator, reason string) error { + if uuid.Validate(id) != nil || len(operator) < 1 || len(operator) > 100 || len(reason) < 1 || len(reason) > 300 { + return fmt.Errorf("job UUID, operator and reason are required") + } + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(578903551)`); err != nil { + return err + } + var pending int + if err = tx.QueryRow(ctx, `SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&pending); err != nil { + return err + } + if pending >= s.Options.MaxOutstanding { + return fmt.Errorf("admission limit reached") + } + var att, state, previous string + err = tx.QueryRow(ctx, `SELECT attachment_id::text,state,coalesce(last_error,'processing_failed') FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id).Scan(&att, &state, &previous) + if err != nil { + return err + } + if state != "failed" { + return fmt.Errorf("only failed jobs may be replayed") + } + var stage *string + if err = tx.QueryRow(ctx, `SELECT staging_key FROM attachments WHERE id=$1 FOR UPDATE`, att).Scan(&stage); err != nil { + return err + } + if stage == nil { + return fmt.Errorf("staging retention expired") + } + if _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, *stage, minio.StatObjectOptions{}); err != nil { + return fmt.Errorf("staging unavailable") + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,operator,reason,previous_error) VALUES($1,$2,$3,$4)`, id, operator, reason, previous); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempts=0,fence=fence+1,last_error=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=$1`, id); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, att); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=$1`, id); err != nil { + return err + } + return tx.Commit(ctx) +} + +// Keep failed stages for seven days for explicit replay; ready stages are garbage. +func (s *Service) cleanStages(ctx context.Context) { + rows, err := s.Pool.Query(ctx, `SELECT id::text FROM attachments WHERE (status='ready' AND staging_key IS NOT NULL) OR (status='failed' AND updated_at max { + return 0, fmt.Errorf("invalid %s", name) + } + return n, nil +} +func secret(name string) (string, error) { + if file := os.Getenv(name + "_FILE"); file != "" { + data, err := os.ReadFile(file) + if err != nil { + return "", fmt.Errorf("cannot read %s_FILE", name) + } + if s := strings.TrimSpace(string(data)); s != "" { + return s, nil + } + } + if s := os.Getenv(name); s != "" { + return s, nil + } + return "", fmt.Errorf("%s or %s_FILE is required", name, name) +} + +func Load() (Config, error) { + var c Config + port, err := positive("PORT", 8080, 65535) + if err != nil { + return c, err + } + c.Port = strconv.Itoa(port) + pgPort, err := positive("POSTGRES_PORT", 5432, 65535) + if err != nil { + return c, err + } + c.PostgresPort = uint16(pgPort) + max, err := positive("POSTGRES_MAX_CONNS", 10, 100) + if err != nil { + return c, err + } + c.PostgresMaxConns = int32(max) + ttl, err := positive("SESSION_TTL_SECONDS", 3600, 86400*30) + if err != nil { + return c, err + } + c.SessionTTL = time.Duration(ttl) * time.Second + timeout, err := positive("QUERY_TIMEOUT_MS", 2000, 10000) + if err != nil { + return c, err + } + c.QueryTimeout = time.Duration(timeout) * time.Millisecond + c.PostgresHost = value("POSTGRES_HOST", "localhost") + c.PostgresUser = value("POSTGRES_USER", "messenger") + c.PostgresDatabase = value("POSTGRES_DB", "messenger") + c.RedisAddr = value("REDIS_ADDR", "localhost:6379") + c.PostgresPassword, err = secret("POSTGRES_PASSWORD") + if err != nil { + return c, err + } + key, err := secret("CURSOR_KEY") + if err != nil { + return c, err + } + decoded, err := hex.DecodeString(key) + if err != nil || len(decoded) != 32 { + return c, fmt.Errorf("CURSOR_KEY must be 32 random bytes encoded as hex") + } + copy(c.CursorKey[:], decoded) + c.S3Endpoint = value("S3_ENDPOINT", "minio:9000") + c.S3Bucket = value("S3_BUCKET", "attachments") + c.S3UseSSL, err = strconv.ParseBool(value("S3_USE_SSL", "false")) + if err != nil { + return c, fmt.Errorf("invalid S3_USE_SSL") + } + c.S3AccessKey, err = secret("S3_ACCESS_KEY") + if err != nil { + return c, err + } + c.S3SecretKey, err = secret("S3_SECRET_KEY") + if err != nil { + return c, err + } + return c, nil +} diff --git a/internal/observability/db.go b/internal/observability/db.go new file mode 100644 index 0000000..4f3e730 --- /dev/null +++ b/internal/observability/db.go @@ -0,0 +1,41 @@ +package observability + +import ( + "context" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "strings" + "sync/atomic" +) + +type DBTracer struct{} + +func (DBTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context { + op := "query" + if words := strings.Fields(d.SQL); len(words) > 0 { + switch strings.ToUpper(words[0]) { + case "SELECT", "UPDATE", "INSERT", "DELETE", "BEGIN", "COMMIT", "ROLLBACK": + op = strings.ToUpper(words[0]) + } + } + ctx, span := Start(ctx, "postgresql "+op, trace.SpanKindClient) + span.SetAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op)) + return ctx +} +func (DBTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryEndData) { + span := trace.SpanFromContext(ctx) + Result(span, d.Err) + span.End() +} + +var waiting atomic.Int64 + +func (DBTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context { + waiting.Add(1) + return ctx +} +func (DBTracer) TraceAcquireEnd(context.Context, *pgxpool.Pool, pgxpool.TraceAcquireEndData) { + waiting.Add(-1) +} diff --git a/internal/observability/http.go b/internal/observability/http.go new file mode 100644 index 0000000..a5ce72a --- /dev/null +++ b/internal/observability/http.go @@ -0,0 +1,26 @@ +package observability + +import ( + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "net/http" +) + +// Transport records dependency timing without URLs, headers, object names or credentials. +type Transport struct{ Base http.RoundTripper } + +func (t Transport) RoundTrip(r *http.Request) (*http.Response, error) { + ctx, span := Start(r.Context(), "s3 "+r.Method, trace.SpanKindClient) + defer span.End() + span.SetAttributes(attribute.String("rpc.system", "s3"), attribute.String("http.request.method", r.Method)) + response, err := t.Base.RoundTrip(r.Clone(ctx)) + Result(span, err) + if response != nil { + span.SetAttributes(attribute.Int("http.response.status_code", response.StatusCode)) + if response.StatusCode >= 500 { + span.SetStatus(codes.Error, "storage unavailable") + } + } + return response, err +} diff --git a/internal/observability/telemetry.go b/internal/observability/telemetry.go new file mode 100644 index 0000000..5c6137e --- /dev/null +++ b/internal/observability/telemetry.go @@ -0,0 +1,205 @@ +// Package observability exports bounded, content-free OTLP signals. +package observability + +import ( + "context" + "encoding/json" + "log" + "os" + "runtime" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/labstack/echo/v4" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +var service string +var logger otellog.Logger +var busy atomic.Int64 +var requests metric.Int64Counter +var duration, jobsDuration, readyDuration metric.Float64Histogram +var jobs metric.Int64Counter +var telemetryErrors atomic.Int64 + +func Init(ctx context.Context, name string) (func(), error) { + service = name + otel.SetTextMapPropagator(propagation.TraceContext{}) + otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { telemetryErrors.Add(1) })) + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return func() {}, nil + } + instance, _ := os.Hostname() + res := resource.NewWithAttributes("", attribute.String("service.name", name), attribute.String("service.version", "lab6"), attribute.String("service.instance.id", instance), attribute.String("deployment.environment.name", "lab")) + te, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(2*time.Second)) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithBatcher(te, sdktrace.WithMaxQueueSize(512), sdktrace.WithMaxExportBatchSize(128), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(2*time.Second))) + otel.SetTracerProvider(tp) + me, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(2*time.Second)) + if err != nil { + return nil, err + } + mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me, sdkmetric.WithInterval(5*time.Second), sdkmetric.WithTimeout(2*time.Second))), sdkmetric.WithView(sdkmetric.NewView(sdkmetric.Instrument{Kind: sdkmetric.InstrumentKindHistogram}, sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{Boundaries: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60, 120}}}))) + otel.SetMeterProvider(mp) + le, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(2*time.Second)) + if err != nil { + return nil, err + } + lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(le, sdklog.WithMaxQueueSize(512), sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(2*time.Second)))) + logger = lp.Logger("messenger") + m := otel.Meter("messenger") + requests, _ = m.Int64Counter("messenger_http_requests") + duration, _ = m.Float64Histogram("messenger_http_duration_seconds", metric.WithUnit("s")) + jobs, _ = m.Int64Counter("messenger_jobs") + jobsDuration, _ = m.Float64Histogram("messenger_job_duration_seconds", metric.WithUnit("s")) + readyDuration, _ = m.Float64Histogram("messenger_attachment_ready_seconds", metric.WithUnit("s")) + return func() { + c, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = lp.Shutdown(c) + _ = mp.Shutdown(c) + _ = tp.Shutdown(c) + }, nil +} + +func Start(ctx context.Context, name string, kind trace.SpanKind) (context.Context, trace.Span) { + return otel.Tracer("messenger").Start(ctx, name, trace.WithSpanKind(kind)) +} +func Result(span trace.Span, err error) { + if err != nil { + span.SetStatus(codes.Error, "dependency unavailable") + } else { + span.SetStatus(codes.Ok, "") + } +} +func Inject(ctx context.Context) (string, string) { + c := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, c) + return c.Get("traceparent"), c.Get("tracestate") +} +func Extract(ctx context.Context, parent, state string) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}) +} +func Log(ctx context.Context, event string, fields map[string]any) { + sc := trace.SpanContextFromContext(ctx) + values := map[string]any{"timestamp": time.Now().UTC().Format(time.RFC3339Nano), "severity": "INFO", "service.name": service, "event": event, "trace_id": sc.TraceID().String(), "span_id": sc.SpanID().String()} + for k, v := range fields { + values[k] = v + } + raw, _ := json.Marshal(values) + log.Print(string(raw)) + if logger != nil { + var rec otellog.Record + rec.SetTimestamp(time.Now()) + rec.SetSeverity(otellog.SeverityInfo) + rec.SetBody(otellog.StringValue(string(raw))) + logger.Emit(ctx, rec) + } +} +func Middleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + route := c.Path() + if route == "" { + route = "unmatched" + } + route = strings.ReplaceAll(strings.ReplaceAll(route, ":chat_id", "{chat_id}"), ":attachment_id", "{attachment_id}") + ctx, span := Start(otel.GetTextMapPropagator().Extract(c.Request().Context(), propagation.HeaderCarrier(c.Request().Header)), c.Request().Method+" "+route, trace.SpanKindServer) + defer span.End() + c.Response().Header().Set("X-Trace-Id", span.SpanContext().TraceID().String()) + c.SetRequest(c.Request().WithContext(ctx)) + start := time.Now() + err := next(c) + if err != nil { + c.Error(err) + } + status := c.Response().Status + class := string(rune('0'+status/100)) + "xx" + attrs := []attribute.KeyValue{attribute.String("method", c.Request().Method), attribute.String("route", route), attribute.String("status_class", class)} + span.SetAttributes(append(attrs, attribute.Int("http.response.status_code", status))...) + if status >= 500 { + span.SetStatus(codes.Error, "server error") + } + if !strings.HasPrefix(route, "/health/") && !strings.HasPrefix(route, "/swagger") && requests != nil { + requests.Add(ctx, 1, metric.WithAttributes(attrs...)) + duration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) + } + Log(ctx, "http.request", map[string]any{"method": c.Request().Method, "route": route, "status": status, "duration_seconds": time.Since(start).Seconds(), "request_id": c.Response().Header().Get("X-Request-Id")}) + return nil + } +} +func JobStart() { busy.Add(1) } +func JobEnd(ctx context.Context, start, accepted time.Time, outcome string) { + busy.Add(-1) + if jobs == nil { + return + } + a := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome)) + jobs.Add(ctx, 1, a) + jobsDuration.Record(ctx, time.Since(start).Seconds(), a) + if outcome == "ready" { + readyDuration.Record(ctx, time.Since(accepted).Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"))) + } +} + +func ObservePool(pool *pgxpool.Pool, capacity int) { + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return + } + m := otel.Meter("messenger") + names := []string{"messenger_db_pool_used", "messenger_db_pool_waiting", "messenger_db_pool_max", "messenger_worker_busy", "messenger_worker_capacity", "messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_process_memory_bytes", "messenger_process_cpu_seconds_total", "messenger_telemetry_export_errors"} + gauges := make([]metric.Float64ObservableGauge, len(names)) + for i, n := range names { + gauges[i], _ = m.Float64ObservableGauge(n) + } + _, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + stat := pool.Stat() + vals := []float64{float64(stat.AcquiredConns()), float64(waiting.Load()), float64(stat.MaxConns()), float64(busy.Load()), float64(capacity)} + for i, v := range vals { + o.ObserveFloat64(gauges[i], v) + } + // Only API replicas report the authoritative queue. Sum counts would duplicate it; dashboards use max. + c, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + var depth int64 + var age float64 + if pool.QueryRow(c, `SELECT count(*),coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) FROM attachment_jobs WHERE state IN ('queued','processing')`).Scan(&depth, &age) == nil { + a := metric.WithAttributes(attribute.String("queue", "attachments")) + o.ObserveFloat64(gauges[5], float64(depth), a) + o.ObserveFloat64(gauges[6], age, a) + } + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + o.ObserveFloat64(gauges[7], float64(mem.Sys)) + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil { + o.ObserveFloat64(gauges[8], float64(usage.Utime.Sec+usage.Stime.Sec)+float64(usage.Utime.Usec+usage.Stime.Usec)/1e6) + } + o.ObserveFloat64(gauges[9], float64(telemetryErrors.Load())) + return nil + }, observable(gauges)...) +} +func observable(gs []metric.Float64ObservableGauge) []metric.Observable { + r := make([]metric.Observable, len(gs)) + for i, g := range gs { + r[i] = g + } + return r +} diff --git a/internal/observability/telemetry_test.go b/internal/observability/telemetry_test.go new file mode 100644 index 0000000..19626ef --- /dev/null +++ b/internal/observability/telemetry_test.go @@ -0,0 +1,99 @@ +package observability + +import ( + "context" + "github.com/jackc/pgx/v5" + "github.com/labstack/echo/v4" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func testProvider(t *testing.T) *tracetest.SpanRecorder { + t.Helper() + old := otel.GetTracerProvider() + oldProp := otel.GetTextMapPropagator() + rec := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) + otel.SetTracerProvider(provider) + otel.SetTextMapPropagator(propagation.TraceContext{}) + t.Cleanup(func() { + otel.SetTracerProvider(old) + otel.SetTextMapPropagator(oldProp) + _ = provider.Shutdown(context.Background()) + }) + return rec +} +func TestHTTPRouteAndTraceCorrelation(t *testing.T) { + rec := testProvider(t) + e := echo.New() + e.Use(Middleware) + e.POST("/api/v1/chats/:chat_id/messages", func(c echo.Context) error { return c.NoContent(http.StatusCreated) }) + r := httptest.NewRequest(http.MethodPost, "/api/v1/chats/secret-id/messages", strings.NewReader("private-message")) + r.Header.Set("Authorization", "Bearer secret-token") + w := httptest.NewRecorder() + e.ServeHTTP(w, r) + spans := rec.Ended() + if w.Code != 201 || len(spans) != 1 { + t.Fatalf("status %d, spans %d", w.Code, len(spans)) + } + span := spans[0] + if w.Header().Get("X-Trace-Id") != span.SpanContext().TraceID().String() { + t.Fatal("HTTP trace correlation missing") + } + if span.Name() != "POST /api/v1/chats/{chat_id}/messages" { + t.Fatalf("unbounded route: %s", span.Name()) + } + for _, a := range span.Attributes() { + v := a.Value.Emit() + if strings.Contains(v, "secret") || strings.Contains(v, "private-message") { + t.Fatal("private request data exported") + } + } +} +func TestDurableContextKeepsParentAcrossPublishAndRetry(t *testing.T) { + rec := testProvider(t) + ctx, server := Start(context.Background(), "upload", trace.SpanKindServer) + parent, state := Inject(ctx) + server.End() + for attempt := 0; attempt < 2; attempt++ { + publish, producer := Start(Extract(context.Background(), parent, state), "publish", trace.SpanKindProducer) + p, s := Inject(publish) + producer.End() + _, consumer := Start(Extract(context.Background(), p, s), "consume", trace.SpanKindConsumer) + consumer.End() + } + spans := rec.Ended() + if len(spans) != 5 { + t.Fatalf("got %d spans", len(spans)) + } + for _, s := range spans { + if s.SpanContext().TraceID() != server.SpanContext().TraceID() { + t.Fatal("trace detached across queue/retry") + } + } + if spans[2].Parent().SpanID() != spans[1].SpanContext().SpanID() || spans[4].Parent().SpanID() != spans[3].SpanContext().SpanID() { + t.Fatal("consumer missing producer parent") + } +} +func TestDatabaseTracerDoesNotExportSQLOrArguments(t *testing.T) { + rec := testProvider(t) + tracer := DBTracer{} + ctx := tracer.TraceQueryStart(context.Background(), nil, pgx.TraceQueryStartData{SQL: "INSERT INTO messages(body) VALUES ('private-message')", Args: []any{"secret-token"}}) + tracer.TraceQueryEnd(ctx, nil, pgx.TraceQueryEndData{}) + spans := rec.Ended() + if len(spans) != 1 || spans[0].Name() != "postgresql INSERT" { + t.Fatal("database operation span missing") + } + for _, a := range spans[0].Attributes() { + if strings.Contains(a.Value.Emit(), "private") || strings.Contains(a.Value.Emit(), "secret") { + t.Fatal("SQL or argument leak") + } + } +} diff --git a/internal/pkg/domain/chat.go b/internal/pkg/domain/chat.go new file mode 100644 index 0000000..2de7174 --- /dev/null +++ b/internal/pkg/domain/chat.go @@ -0,0 +1,10 @@ +package domain + +import "time" + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} diff --git a/internal/pkg/domain/errors.go b/internal/pkg/domain/errors.go new file mode 100644 index 0000000..a8447d2 --- /dev/null +++ b/internal/pkg/domain/errors.go @@ -0,0 +1,32 @@ +package domain + +import "errors" + +type ErrorCode string + +const ( + ErrorUnavailable ErrorCode = "unavailable" + ErrorInvalid ErrorCode = "invalid_request" + ErrorUnauthorized ErrorCode = "unauthorized" + ErrorNotFound ErrorCode = "not_found" + ErrorConflict ErrorCode = "conflict" +) + +type ApplicationError struct { + Code ErrorCode + Message string +} + +func (e *ApplicationError) Error() string { + return e.Message +} + +func NewApplicationError(code ErrorCode, message string) error { + return &ApplicationError{Code: code, Message: message} +} + +func AsApplicationError(err error) (*ApplicationError, bool) { + var applicationError *ApplicationError + ok := errors.As(err, &applicationError) + return applicationError, ok +} diff --git a/internal/pkg/domain/message.go b/internal/pkg/domain/message.go new file mode 100644 index 0000000..3487996 --- /dev/null +++ b/internal/pkg/domain/message.go @@ -0,0 +1,13 @@ +package domain + +import "time" + +type Message struct { + IdempotencyKey string + ID string + ChatID string + SenderID string + AttachmentIDs []string + Text string + CreatedAt time.Time +} diff --git a/internal/pkg/domain/pagination.go b/internal/pkg/domain/pagination.go new file mode 100644 index 0000000..d3ab68f --- /dev/null +++ b/internal/pkg/domain/pagination.go @@ -0,0 +1,23 @@ +package domain + +import "time" + +type Position struct { + CreatedAt time.Time + ID string +} + +type Pagination struct { + After *Position + Limit int +} + +type ChatPage struct { + Items []Chat + HasMore bool +} + +type MessagePage struct { + Items []Message + HasMore bool +} diff --git a/internal/pkg/domain/session.go b/internal/pkg/domain/session.go new file mode 100644 index 0000000..50029b9 --- /dev/null +++ b/internal/pkg/domain/session.go @@ -0,0 +1,14 @@ +package domain + +import "time" + +type Session struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Login struct { + Token string + ExpiresAt time.Time + User User +} diff --git a/internal/pkg/domain/user.go b/internal/pkg/domain/user.go new file mode 100644 index 0000000..34d5235 --- /dev/null +++ b/internal/pkg/domain/user.go @@ -0,0 +1,10 @@ +package domain + +import "time" + +type User struct { + ID string + Username string + DisplayName string + CreatedAt time.Time +} diff --git a/internal/pkg/password/password.go b/internal/pkg/password/password.go new file mode 100644 index 0000000..1280dca --- /dev/null +++ b/internal/pkg/password/password.go @@ -0,0 +1,65 @@ +package password + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "strings" + + "golang.org/x/crypto/argon2" +) + +// Bound simultaneous KDF work to 4 * 19 MiB, including concurrent registrations. +var slots = make(chan struct{}, 4) + +const prefix = "$argon2id$v=19$m=19456,t=2,p=1$" + +func derive(ctx context.Context, password string, salt []byte) ([]byte, error) { + select { + case slots <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-slots }() + if err := ctx.Err(); err != nil { + return nil, err + } + return argon2.IDKey([]byte(password), salt, 2, 19456, 1, 32), nil +} + +func Hash(ctx context.Context, password string) (string, error) { + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return "", err + } + key, err := derive(ctx, password, salt) + if err != nil { + return "", err + } + return prefix + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} + +func Verify(ctx context.Context, password, encoded string) (bool, error) { + if !strings.HasPrefix(encoded, prefix) { + return false, errors.New("unsupported password hash") + } + parts := strings.Split(strings.TrimPrefix(encoded, prefix), "$") + if len(parts) != 2 { + return false, errors.New("invalid password hash") + } + salt, err := base64.RawStdEncoding.DecodeString(parts[0]) + if err != nil || len(salt) != 16 { + return false, errors.New("invalid salt") + } + expected, err := base64.RawStdEncoding.DecodeString(parts[1]) + if err != nil || len(expected) != 32 { + return false, errors.New("invalid hash") + } + actual, err := derive(ctx, password, salt) + return subtle.ConstantTimeCompare(actual, expected) == 1, err +} + +// A valid fixed encoding for equal-cost verification when username does not exist. +const DummyHash = prefix + "AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" diff --git a/internal/pkg/password/password_test.go b/internal/pkg/password/password_test.go new file mode 100644 index 0000000..6a0ef96 --- /dev/null +++ b/internal/pkg/password/password_test.go @@ -0,0 +1,35 @@ +package password_test + +import ( + "context" + "strings" + "testing" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +func TestUnicodeAndSalt(t *testing.T) { + ctx := context.Background() + secret := strings.Repeat("🔑", 128) + first, err := password.Hash(ctx, secret) + if err != nil { + t.Fatal(err) + } + second, err := password.Hash(ctx, secret) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("password hashes reused salt") + } + for _, hash := range []string{first, second} { + ok, err := password.Verify(ctx, secret, hash) + if err != nil || !ok { + t.Fatal("128-code-point UTF-8 password rejected") + } + ok, err = password.Verify(ctx, strings.Repeat("🔑", 127)+"x", hash) + if err != nil || ok { + t.Fatal("password suffix was ignored") + } + } +} diff --git a/internal/pkg/repository/memory.go b/internal/pkg/repository/memory.go new file mode 100644 index 0000000..1c26901 --- /dev/null +++ b/internal/pkg/repository/memory.go @@ -0,0 +1,202 @@ +package repository + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/google/uuid" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +// Memory is the lab1 in-memory Repository implementation. +type Memory struct { + mu sync.RWMutex + users map[string]domain.User + hashes map[string]string + userByUsername map[string]string + chats map[string]domain.Chat + messages map[string][]domain.Message +} + +var _ chatusecase.Repository = (*Memory)(nil) + +func NewMemory() *Memory { + return &Memory{ + users: make(map[string]domain.User), + hashes: make(map[string]string), + userByUsername: make(map[string]string), + chats: make(map[string]domain.Chat), + messages: make(map[string][]domain.Message), + } +} + +func (m *Memory) CreateUser(_ context.Context, username, displayName, hash string) (domain.User, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.userByUsername[username]; exists { + return domain.User{}, chatusecase.ErrUsernameTaken + } + user := domain.User{ + ID: uuid.NewString(), + Username: username, + DisplayName: displayName, + CreatedAt: now(), + } + m.users[user.ID] = user + m.hashes[user.ID] = hash + m.userByUsername[user.Username] = user.ID + return user, nil +} + +func (m *Memory) User(_ context.Context, id string) (domain.User, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + user, ok := m.users[id] + return user, ok, nil +} + +func (m *Memory) CreateChat(_ context.Context, title string, memberIDs []string) (domain.Chat, error) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, memberID := range memberIDs { + if _, exists := m.users[memberID]; !exists { + return domain.Chat{}, chatusecase.ErrUserNotFound + } + } + chat := domain.Chat{ + ID: uuid.NewString(), + Title: title, + MemberIDs: append([]string(nil), memberIDs...), + CreatedAt: now(), + } + m.chats[chat.ID] = chat + return cloneChat(chat), nil +} + +func (m *Memory) ChatForUser(_ context.Context, chatID, userID string) (domain.Chat, error) { + m.mu.RLock() + defer m.mu.RUnlock() + chat, exists := m.chats[chatID] + if !exists { + return domain.Chat{}, chatusecase.ErrChatNotFound + } + if !contains(chat.MemberIDs, userID) { + return domain.Chat{}, chatusecase.ErrNotMember + } + return cloneChat(chat), nil +} + +func (m *Memory) ListChats(_ context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make([]domain.Chat, 0) + for _, chat := range m.chats { + if contains(chat.MemberIDs, userID) && isAfter(chat.CreatedAt, chat.ID, after) { + result = append(result, cloneChat(chat)) + } + } + sort.Slice(result, func(i, j int) bool { + return less(result[i].CreatedAt, result[i].ID, result[j].CreatedAt, result[j].ID) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func (m *Memory) CreateMessage(_ context.Context, chatID, senderID, text, key string) (domain.Message, error) { + m.mu.Lock() + defer m.mu.Unlock() + + chat, exists := m.chats[chatID] + if !exists { + return domain.Message{}, chatusecase.ErrChatNotFound + } + if !contains(chat.MemberIDs, senderID) { + return domain.Message{}, chatusecase.ErrNotMember + } + if key != "" { + for _, previous := range m.messages[chatID] { + if previous.SenderID == senderID && previous.IdempotencyKey == key { + if previous.Text != text { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") + } + return previous, nil + } + } + } + message := domain.Message{ + IdempotencyKey: key, + ID: uuid.NewString(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: now(), + } + m.messages[chatID] = append(m.messages[chatID], message) + return message, nil +} + +func (m *Memory) ListMessages(_ context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + values := m.messages[chatID] + result := make([]domain.Message, 0, min(len(values), limit)) + for _, message := range values { + if isAfter(message.CreatedAt, message.ID, after) { + result = append(result, message) + } + } + sort.Slice(result, func(i, j int) bool { + return less(result[i].CreatedAt, result[i].ID, result[j].CreatedAt, result[j].ID) + }) + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +func cloneChat(chat domain.Chat) domain.Chat { + chat.MemberIDs = append([]string(nil), chat.MemberIDs...) + return chat +} + +func now() time.Time { + return time.Now().UTC().Truncate(time.Second) +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func isAfter(createdAt time.Time, id string, after *domain.Position) bool { + return after == nil || createdAt.After(after.CreatedAt) || createdAt.Equal(after.CreatedAt) && id > after.ID +} + +func less(leftTime time.Time, leftID string, rightTime time.Time, rightID string) bool { + if leftTime.Equal(rightTime) { + return leftID < rightID + } + return leftTime.Before(rightTime) +} + +func (m *Memory) Ready(context.Context) error { return nil } +func (m *Memory) UserByUsername(_ context.Context, username string) (domain.User, string, bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + id, ok := m.userByUsername[username] + return m.users[id], m.hashes[id], ok, nil +} diff --git a/internal/pkg/repository/postgres/chats.go b/internal/pkg/repository/postgres/chats.go new file mode 100644 index 0000000..60c9051 --- /dev/null +++ b/internal/pkg/repository/postgres/chats.go @@ -0,0 +1,78 @@ +package postgres + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +func (s *Storage) CreateChat(ctx context.Context, title string, members []string) (domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + tx, err := s.pool.Begin(ctx) + if err != nil { + return domain.Chat{}, dbError(err) + } + defer tx.Rollback(context.WithoutCancel(ctx)) + chat := domain.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), members...)} + err = tx.QueryRow(ctx, `INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at`, chat.ID, title).Scan(&chat.CreatedAt) + if err != nil { + return domain.Chat{}, dbError(err) + } + for i, member := range members { + if _, err = tx.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id,ordinal) VALUES($1,$2,$3)`, chat.ID, member, i); err != nil { + return domain.Chat{}, dbError(err) + } + } + if err = tx.Commit(ctx); err != nil { + return domain.Chat{}, dbError(err) + } + return chat, nil +} + +const chatColumns = `c.id::text,c.title,c.created_at,ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id=c.id ORDER BY m.ordinal)` + +func (s *Storage) ChatForUser(ctx context.Context, chatID, userID string) (domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var chat domain.Chat + err := s.pool.QueryRow(ctx, `SELECT `+chatColumns+` FROM chats c + WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=$2)`, chatID, userID). + Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + if errors.Is(err, pgx.ErrNoRows) { + return chat, chatusecase.ErrChatNotFound + } + return chat, dbError(err) +} + +func (s *Storage) ListChats(ctx context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + query := `SELECT ` + chatColumns + ` FROM chats c JOIN chat_members member ON member.chat_id=c.id + WHERE member.user_id=$1` + args := []any{userID, limit} + if after != nil { + query += ` AND (c.created_at,c.id)>($3,$4)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY c.created_at,c.id LIMIT $2` + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + chats := make([]domain.Chat, 0) + for rows.Next() { + var chat domain.Chat + if err = rows.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, dbError(err) + } + chats = append(chats, chat) + } + return chats, dbError(rows.Err()) +} diff --git a/internal/pkg/repository/postgres/lab4_test.go b/internal/pkg/repository/postgres/lab4_test.go new file mode 100644 index 0000000..c4adf4e --- /dev/null +++ b/internal/pkg/repository/postgres/lab4_test.go @@ -0,0 +1,346 @@ +package postgres_test + +import ( + "bufio" + "bytes" + "context" + "encoding/xml" + "fmt" + "image" + "image/png" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type fakeObject struct { + data []byte + modified time.Time +} +type fakeS3 struct { + mu sync.Mutex + objects map[string]fakeObject + failPUT bool +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "GET" && r.URL.Path == "/attachments/" || r.Method == "GET" && r.URL.Path == "/attachments" { + type item struct { + Key string + LastModified string + ETag string + Size int + } + type listing struct { + XMLName xml.Name `xml:"ListBucketResult"` + Xmlns string `xml:"xmlns,attr"` + Name string + Prefix string + MaxKeys int + IsTruncated bool + Contents []item + } + l := listing{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", Name: "attachments", Prefix: r.URL.Query().Get("prefix"), MaxKeys: 1000} + for k, v := range f.objects { + if strings.HasPrefix(k, l.Prefix) { + l.Contents = append(l.Contents, item{Key: k, LastModified: v.modified.UTC().Format("2006-01-02T15:04:05.000Z"), ETag: `"abc"`, Size: len(v.data)}) + } + } + w.Header().Set("Content-Type", "application/xml") + xml.NewEncoder(w).Encode(l) + return + } + if r.Method == "PUT" { + if f.failPUT { + w.WriteHeader(503) + fmt.Fprint(w, `ServiceUnavailable`) + return + } + var data []byte + var err error + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") { + reader := bufio.NewReader(r.Body) + for { + line, e := reader.ReadString('\n') + if e != nil { + err = e + break + } + size, e := strconv.ParseInt(strings.Split(strings.TrimSpace(line), ";")[0], 16, 64) + if e != nil { + err = e + break + } + if size == 0 { + break + } + chunk := make([]byte, size) + if _, e = io.ReadFull(reader, chunk); e != nil { + err = e + break + } + data = append(data, chunk...) + if _, e = reader.Discard(2); e != nil { + err = e + break + } + } + } else { + data, err = io.ReadAll(r.Body) + } + if err != nil { + w.WriteHeader(400) + return + } + f.objects[key] = fakeObject{data: data, modified: time.Now().UTC()} + w.Header().Set("ETag", `"900150983cd24fb0d6963f7d28e17f72"`) + w.WriteHeader(200) + return + } + obj, ok := f.objects[key] + if !ok { + w.WriteHeader(404) + if r.Method != "HEAD" { + fmt.Fprint(w, `NoSuchKey`) + } + return + } + if r.Method == "DELETE" { + delete(f.objects, key) + w.WriteHeader(204) + return + } + w.Header().Set("Last-Modified", obj.modified.UTC().Format(http.TimeFormat)) + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Content-Length", fmt.Sprint(len(obj.data))) + w.Header().Set("Content-Type", "image/png") + if r.Method == "GET" { + w.Write(obj.data) + } +} +func lab4Backend(t *testing.T, pool *pgxpool.Pool, repo *postgres.Storage, user domain.User, chat domain.Chat) { + ctx := context.Background() + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + service := attachment.New(pool, &attachment.Store{Client: client, Bucket: "attachments"}) + var imageBytes bytes.Buffer + if err = png.Encode(&imageBytes, image.NewRGBA(image.Rect(0, 0, 10, 8))); err != nil { + t.Fatal(err) + } + upload := func(name string) attachment.Attachment { + t.Helper() + a, e := service.Upload(ctx, chat.ID, user.ID, name, "image/png", bytes.NewReader(imageBytes.Bytes())) + if e != nil { + t.Fatal(e) + } + return a + } + first := upload("first.png") + second := upload("second.png") + requireCode := func(err error, want domain.ErrorCode) { + t.Helper() + e, ok := domain.AsApplicationError(err) + if !ok || e.Code != want { + t.Fatalf("expected %s, got %v", want, err) + } + } + outside, err := repo.CreateUser(ctx, "attachment_outsider", "Outside", "hash") + if err != nil { + t.Fatal(err) + } + _, err = service.Get(ctx, first.ID, outside.ID) + requireCode(err, domain.ErrorNotFound) + obj, err := service.Content(ctx, first) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(obj) + obj.Close() + if err != nil || !bytes.Equal(data, imageBytes.Bytes()) { + t.Fatalf("original mismatch: %v", err) + } + otherChat, err := repo.CreateChat(ctx, "Other", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + _, err = repo.CreateMessageWithAttachments(ctx, otherChat.ID, user.ID, "", "cross-chat", []string{first.ID}) + requireCode(err, domain.ErrorNotFound) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "unknown", []string{uuid.NewString()}) + requireCode(err, domain.ErrorNotFound) + sent, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil { + t.Fatal(err) + } + replay, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || sent.ID != replay.ID { + t.Fatalf("replay failed: %v", err) + } + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{second.ID, first.ID}) + requireCode(err, domain.ErrorConflict) + textOnly, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", nil) + if err != nil { + t.Fatal(err) + } + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", []string{}) + if err != nil || textOnly.ID != replay.ID { + t.Fatalf("absent/empty differs: %v", err) + } + + const callers = 12 + outcomes := make(chan domain.Message, callers) + failures := make(chan error, callers) + var concurrent sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < callers; i++ { + concurrent.Add(1) + go func() { + defer concurrent.Done() + <-start + m, e := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "concurrent-attachments", []string{first.ID, second.ID}) + if e != nil { + failures <- e + } else { + outcomes <- m + } + }() + } + close(start) + concurrent.Wait() + close(outcomes) + close(failures) + for e := range failures { + t.Fatal(e) + } + identity := "" + for m := range outcomes { + if identity == "" { + identity = m.ID + } + if m.ID != identity { + t.Fatal("duplicate concurrent attachment message") + } + } + var effects int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id=$1 AND idempotency_key='concurrent-attachments'`, chat.ID).Scan(&effects); err != nil || effects != 1 { + t.Fatalf("concurrent effects=%d %v", effects, err) + } + t.Log("ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed") + + old := time.Now().Add(-3 * time.Minute) + orphan := "original/" + uuid.NewString() + fresh := "original/" + uuid.NewString() + locked := "original/" + uuid.NewString() + fake.mu.Lock() + delete(fake.objects, first.Key) + v := fake.objects[second.Key] + v.modified = old + fake.objects[second.Key] = v + fake.objects[orphan] = fakeObject{data: []byte("orphan"), modified: old} + fake.objects[fresh] = fakeObject{data: []byte("fresh"), modified: time.Now()} + fake.objects[locked] = fakeObject{data: []byte("locked"), modified: old} + fake.objects["original/legacy-staging"] = fakeObject{data: []byte("legacy stage"), modified: old} + fake.mu.Unlock() + if _, err = pool.Exec(ctx, `UPDATE attachments SET staging_key='original/legacy-staging' WHERE id=$1`, second.ID); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `UPDATE attachments SET created_at=$1 WHERE id=ANY($2::uuid[])`, old, []string{first.ID, second.ID}); err != nil { + t.Fatal(err) + } + hold, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer hold.Rollback(ctx) + if _, err = hold.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, locked); err != nil { + t.Fatal(err) + } + dry, err := service.Reconcile(ctx, false, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if dry.Orphans != 1 || dry.Missing != 1 || dry.ReferencedMissing != 1 || dry.LockedSkipped != 1 || dry.FreshSkipped < 1 || dry.Deleted != 0 || dry.MarkedFailed != 0 { + t.Fatalf("bad dry-run %+v", dry) + } + var status string + if err = pool.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1`, first.ID).Scan(&status); err != nil || status != "ready" { + t.Fatalf("dry-run mutated metadata %s %v", status, err) + } + applied, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if applied.Deleted != 1 || applied.MarkedFailed != 1 { + t.Fatalf("bad apply %+v", applied) + } + fake.mu.Lock() + _, hasOrphan := fake.objects[orphan] + _, hasFresh := fake.objects[fresh] + _, hasLinked := fake.objects[second.Key] + _, hasLocked := fake.objects[locked] + _, hasLegacyStage := fake.objects["original/legacy-staging"] + fake.mu.Unlock() + if hasOrphan || !hasFresh || !hasLinked || !hasLocked || !hasLegacyStage { + t.Fatal("unsafe object cleanup") + } + _, err = service.Get(ctx, first.ID, user.ID) + requireCode(err, domain.ErrorConflict) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "failed-attachment", []string{first.ID}) + requireCode(err, domain.ErrorConflict) + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || replay.ID != sent.ID { + t.Fatalf("historical replay changed after missing: %v", err) + } + repeated, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if repeated.Deleted != 0 || repeated.MarkedFailed != 0 || repeated.Orphans != 0 || repeated.Missing != 0 { + t.Fatalf("not repeatable %+v", repeated) + } + t.Logf("dry-run=%+v apply=%+v repeat=%+v; fresh, locked and referenced objects retained", dry, applied, repeated) +} + +func TestS3WireFixture(t *testing.T) { + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + payload := []byte("exact object bytes") + _, err = client.PutObject(context.Background(), "attachments", "original/test", bytes.NewReader(payload), int64(len(payload)), minio.PutObjectOptions{ContentType: "image/png", DisableMultipart: true}) + if err != nil { + t.Fatal(err) + } + obj, err := client.GetObject(context.Background(), "attachments", "original/test", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } + defer obj.Close() + got, err := io.ReadAll(obj) + if err != nil || !bytes.Equal(got, payload) { + t.Fatalf("wire fixture did not store original: %q %v", got, err) + } +} diff --git a/internal/pkg/repository/postgres/messages.go b/internal/pkg/repository/postgres/messages.go new file mode 100644 index 0000000..da695fa --- /dev/null +++ b/internal/pkg/repository/postgres/messages.go @@ -0,0 +1,96 @@ +package postgres + +import ( + "context" + "errors" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "slices" +) + +func (s *Storage) CreateMessage(ctx context.Context, chatID, senderID, text, key string) (domain.Message, error) { + return s.CreateMessageWithAttachments(ctx, chatID, senderID, text, key, nil) +} + +func (s *Storage) CreateMessageWithAttachments(ctx context.Context, chatID, senderID, text, key string, ids []string) (domain.Message, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if ids == nil { + ids = []string{} + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return domain.Message{}, dbError(err) + } + defer tx.Rollback(context.Background()) + // Serialize this idempotency scope before validating attachments. A replay remains a + // replay even when reconciliation has since marked an original as missing. + if key != "" { + _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, senderID+":"+chatID+":"+key) + if err != nil { + return domain.Message{}, dbError(err) + } + old := domain.Message{ChatID: chatID, SenderID: senderID} + err = tx.QueryRow(ctx, `SELECT id::text,text,attachment_ids::text[],created_at FROM messages WHERE sender_id=$1 AND chat_id=$2 AND idempotency_key=$3`, senderID, chatID, key).Scan(&old.ID, &old.Text, &old.AttachmentIDs, &old.CreatedAt) + if err == nil { + if old.Text != text || !slices.Equal(old.AttachmentIDs, ids) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") + } + return old, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, dbError(err) + } + } + // Lock each metadata row against reconciler updates until the link is committed. + for _, id := range ids { + var status string + err = tx.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE`, id, chatID).Scan(&status) + if errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return domain.Message{}, dbError(err) + } + if status == "failed" { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Вложение не готово") + } + } + m := domain.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text, AttachmentIDs: ids} + var storedKey *string + if key != "" { + storedKey = &key + } + err = tx.QueryRow(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,idempotency_key,attachment_ids) VALUES($1,$2,$3,$4,$5,$6) RETURNING created_at`, m.ID, chatID, senderID, text, storedKey, ids).Scan(&m.CreatedAt) + if err != nil { + return domain.Message{}, dbError(err) + } + return m, dbError(tx.Commit(ctx)) +} + +func (s *Storage) ListMessages(ctx context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + query := `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.attachment_ids::text[],m.created_at FROM messages m WHERE m.chat_id=$1` + args := []any{chatID, limit} + if after != nil { + query += ` AND (m.created_at,m.id)>($3,$4)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY m.created_at,m.id LIMIT $2` + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, dbError(err) + } + defer rows.Close() + result := []domain.Message{} + for rows.Next() { + var m domain.Message + if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.AttachmentIDs, &m.CreatedAt); err != nil { + return nil, dbError(err) + } + result = append(result, m) + } + return result, dbError(rows.Err()) +} diff --git a/internal/pkg/repository/postgres/storage.go b/internal/pkg/repository/postgres/storage.go new file mode 100644 index 0000000..22a93b1 --- /dev/null +++ b/internal/pkg/repository/postgres/storage.go @@ -0,0 +1,51 @@ +package postgres + +import ( + "context" + "errors" + "log" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +type Storage struct { + pool *pgxpool.Pool + timeout time.Duration +} + +var _ chatusecase.Repository = (*Storage)(nil) + +func New(pool *pgxpool.Pool, timeout time.Duration) *Storage { + return &Storage{pool: pool, timeout: timeout} +} + +func (s *Storage) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return dbError(s.pool.Ping(ctx)) +} + +func dbError(err error) error { + if err == nil { + return nil + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": + return chatusecase.ErrUsernameTaken + case "23503": + return chatusecase.ErrUserNotFound + } + log.Printf("postgres operation failed: sqlstate=%s", pgErr.Code) + } else if !errors.Is(err, pgx.ErrNoRows) { + log.Print("postgres operation failed") + } + return domain.NewApplicationError(domain.ErrorUnavailable, "Хранилище недоступно") +} diff --git a/internal/pkg/repository/postgres/storage_test.go b/internal/pkg/repository/postgres/storage_test.go new file mode 100644 index 0000000..f179747 --- /dev/null +++ b/internal/pkg/repository/postgres/storage_test.go @@ -0,0 +1,292 @@ +package postgres_test + +import ( + "context" + "errors" + "net" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + chatusecase "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat" +) + +func TestIntegration(t *testing.T) { + if os.Getenv("INTEGRATION_TESTS") != "1" { + t.Skip("run make test-integration against the Compose stack") + } + ctx := context.Background() + cfg, err := config.Load() + if err != nil { + t.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + t.Fatal(err) + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.RuntimeParams["timezone"] = "UTC" + pc.MaxConns = 4 + admin, err := pgxpool.NewWithConfig(ctx, pc.Copy()) + if err != nil { + t.Fatal("connect failed") + } + defer admin.Close() + // Identifier is generated by the test, never from an API request. + schema := "test_" + uuid.New().String()[:8] + if _, err = admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil { + t.Fatal(err) + } + defer admin.Exec(ctx, `DROP SCHEMA `+schema+` CASCADE`) + pc.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + runGoose := func(t *testing.T, command string, params ...string) { + t.Helper() + migrationDir := os.Getenv("GOOSE_MIGRATION_DIR") + if migrationDir == "" { + migrationDir, err = filepath.Abs("../../../../migrations") + if err != nil { + t.Fatal(err) + } + } + connection := url.URL{Scheme: "postgres", User: url.User(cfg.PostgresUser), Host: net.JoinHostPort(cfg.PostgresHost, strconv.Itoa(int(cfg.PostgresPort))), Path: cfg.PostgresDatabase} + connection.RawQuery = url.Values{"sslmode": {"disable"}, "search_path": {schema}, "connect_timeout": {"5"}}.Encode() + commandCtx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + args := append([]string{"-env", "/dev/null", "-dir", migrationDir, "-timeout", "30s", command}, params...) + commandExec := exec.CommandContext(commandCtx, "goose", args...) + commandExec.Env = append(os.Environ(), "GOOSE_DRIVER=postgres", "GOOSE_DBSTRING="+connection.String(), "PGPASSWORD="+cfg.PostgresPassword) + output, err := commandExec.CombinedOutput() + if err != nil { + t.Fatalf("goose %s: %v: %s", command, err, strings.ReplaceAll(string(output), cfg.PostgresPassword, "[redacted]")) + } + t.Logf("goose %s: %s", command, output) + } + runGoose(t, "up") + runGoose(t, "up") + var versions int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM goose_db_version WHERE version_id=1 AND is_applied`).Scan(&versions); err != nil || versions != 1 { + t.Fatalf("migration versions=%d error=%v", versions, err) + } + repo := postgres.New(pool, 2*time.Second) + t.Run("registration_race", func(t *testing.T) { + const count = 12 + results := make(chan error, count) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := repo.CreateUser(ctx, "same_username", "Same", "test-hash") + results <- err + }() + } + close(start) + wg.Wait() + close(results) + successes, conflicts := 0, 0 + for err := range results { + if err == nil { + successes++ + } else if errors.Is(err, chatusecase.ErrUsernameTaken) { + conflicts++ + } else { + t.Fatal(err) + } + } + if successes != 1 || conflicts != count-1 { + t.Fatalf("successes=%d conflicts=%d", successes, conflicts) + } + t.Logf("12 concurrent INSERTs: %d success, %d unique conflicts", successes, conflicts) + }) + user, err := repo.CreateUser(ctx, "history_owner", "Owner", "test-hash") + if err != nil { + t.Fatal(err) + } + t.Run("rollback", func(t *testing.T) { + _, err := repo.CreateChat(ctx, "must rollback", []string{user.ID, uuid.NewString()}) + if !errors.Is(err, chatusecase.ErrUserNotFound) { + t.Fatalf("unexpected error %v", err) + } + var chats, members int + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM chats),(SELECT count(*) FROM chat_members)`).Scan(&chats, &members); err != nil { + t.Fatal(err) + } + if chats != 0 || members != 0 { + t.Fatalf("partial transaction: chats=%d members=%d", chats, members) + } + t.Log("failed second member INSERT left 0 chats and 0 members") + }) + chat, err := repo.CreateChat(ctx, "History", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + t.Run("lab4_attachments_and_reconciliation", func(t *testing.T) { + isolated, e := repo.CreateChat(ctx, "Lab4", []string{user.ID}) + if e != nil { + t.Fatal(e) + } + lab4Backend(t, pool, repo, user, isolated) + }) + _, err = pool.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT md5(i::text)::uuid,$1,$2,'synthetic message '||i, + timestamptz '2026-01-01 00:00:00+00' + (i / 10) * interval '1 millisecond' + FROM generate_series(1,10000) i`, chat.ID, user.ID) + if err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `ANALYZE messages`); err != nil { + t.Fatal(err) + } + t.Run("keyset_and_explain", func(t *testing.T) { + seen := make(map[string]bool) + var after *domain.Position + for { + page, err := repo.ListMessages(ctx, chat.ID, after, 97) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + for _, message := range page { + if seen[message.ID] { + t.Fatal("duplicate on page boundary") + } + seen[message.ID] = true + } + last := page[len(page)-1] + after = &domain.Position{CreatedAt: last.CreatedAt, ID: last.ID} + } + if len(seen) != 10000 { + t.Fatalf("history count=%d", len(seen)) + } + rows, err := pool.Query(ctx, `EXPLAIN (ANALYZE, BUFFERS) + SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.created_at FROM messages m + WHERE m.chat_id=$1 AND (m.created_at,m.id)>($2,$3) ORDER BY m.created_at,m.id LIMIT $4`, + chat.ID, time.Date(2026, 1, 1, 0, 0, 0, 900000000, time.UTC), "00000000-0000-0000-0000-000000000000", 51) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + t.Log("EXPLAIN on 10,000 messages, cursor near the end (9,000):") + for rows.Next() { + var line string + if err = rows.Scan(&line); err != nil { + t.Fatal(err) + } + t.Log(line) + } + if err = rows.Err(); err != nil { + t.Fatal(err) + } + t.Log("keyset traversal returned 10,000 distinct messages, including equal timestamps") + }) + t.Run("idempotency_race", func(t *testing.T) { + isolated, err := repo.CreateChat(ctx, "Idempotency", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + const count = 16 + results := make(chan domain.Message, count) + failures := make(chan error, count) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + message, err := repo.CreateMessage(ctx, isolated.ID, user.ID, "exactly once", "shared-key") + if err != nil { + failures <- err + return + } + results <- message + }() + } + close(start) + wg.Wait() + close(results) + close(failures) + for err := range failures { + t.Fatal(err) + } + var first domain.Message + for message := range results { + if first.ID == "" { + first = message + } + if message.ID != first.ID || !message.CreatedAt.Equal(first.CreatedAt) || message.Text != first.Text { + t.Fatal("concurrent replay changed result") + } + } + var rows int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id=$1`, isolated.ID).Scan(&rows); err != nil || rows != 1 { + t.Fatalf("expected one durable effect, got %d: %v", rows, err) + } + _, err = repo.CreateMessage(ctx, isolated.ID, user.ID, "different", "shared-key") + applicationError, ok := domain.AsApplicationError(err) + if !ok || applicationError.Code != domain.ErrorConflict { + t.Fatalf("expected conflict, got %v", err) + } + t.Log("16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts") + }) + t.Run("goose_legacy_and_down", func(t *testing.T) { + // This is the disposable test schema, never the application's public schema. + const digestSQL = `SELECT md5(string_agg(ROW(id,chat_id,sender_id,text,created_at)::text,',' ORDER BY id)) FROM messages` + var before, after string + if err = pool.QueryRow(ctx, digestSQL).Scan(&before); err != nil { + t.Fatal(err) + } + // Model the actual lab4 physical schema before adopting its legacy journal. + runGoose(t, "down-to", "4") + if _, err = pool.Exec(ctx, `DROP TABLE goose_db_version; + CREATE TABLE schema_migrations(version text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now()); + INSERT INTO schema_migrations(version) VALUES('001_initial.sql')`); err != nil { + t.Fatal(err) + } + runGoose(t, "up") + runGoose(t, "up") + if err = pool.QueryRow(ctx, digestSQL).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("legacy migration changed message data") + } + t.Log("adopted legacy schema: all existing messages are unchanged") + runGoose(t, "reset") + var tables int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM information_schema.tables WHERE table_schema=$1 AND table_name IN ('users','chats','chat_members','messages','schema_migrations')`, schema).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("down left %d application tables: %v", tables, err) + } + runGoose(t, "up") + if err = pool.QueryRow(ctx, `SELECT count(*) FROM messages`).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("up after down failed: %v", err) + } + t.Log("down and up succeeded in isolated schema") + }) + +} diff --git a/internal/pkg/repository/postgres/users.go b/internal/pkg/repository/postgres/users.go new file mode 100644 index 0000000..ad3f8c1 --- /dev/null +++ b/internal/pkg/repository/postgres/users.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func (s *Storage) CreateUser(ctx context.Context, username, displayName, hash string) (domain.User, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + err := s.pool.QueryRow(ctx, `INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) + RETURNING id::text,username,display_name,created_at`, uuid.NewString(), username, displayName, hash). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + return user, dbError(err) +} + +func (s *Storage) User(ctx context.Context, id string) (domain.User, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + err := s.pool.QueryRow(ctx, `SELECT id::text,username,display_name,created_at FROM users WHERE id=$1`, id). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return user, false, nil + } + return user, err == nil, dbError(err) +} + +func (s *Storage) UserByUsername(ctx context.Context, username string) (domain.User, string, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var user domain.User + var hash string + err := s.pool.QueryRow(ctx, `SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1`, username). + Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt, &hash) + if errors.Is(err, pgx.ErrNoRows) { + return user, "", false, nil + } + return user, hash, err == nil, dbError(err) +} diff --git a/internal/pkg/session/redis.go b/internal/pkg/session/redis.go new file mode 100644 index 0000000..13721b4 --- /dev/null +++ b/internal/pkg/session/redis.go @@ -0,0 +1,71 @@ +package session + +import ( + "context" + "encoding/json" + "errors" + "log" + "time" + + "github.com/redis/go-redis/v9" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +type Redis struct { + client *redis.Client + timeout time.Duration +} + +func NewRedis(client *redis.Client, timeout time.Duration) *Redis { + return &Redis{client: client, timeout: timeout} +} + +func unavailable(err error) error { + if err == nil { + return nil + } + log.Print("redis operation failed") + return domain.NewApplicationError(domain.ErrorUnavailable, "Хранилище сессий недоступно") +} + +func (s *Redis) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + return unavailable(s.client.Ping(ctx).Err()) +} + +func (s *Redis) Put(ctx context.Context, key string, value domain.Session) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + data, err := json.Marshal(value) + if err != nil { + return err + } + // Absolute expiry prevents TTL extension by network delay or restart. + return unavailable(s.client.Do(ctx, "SET", "session:"+key, data, "PXAT", value.ExpiresAt.UnixMilli()).Err()) +} + +func (s *Redis) Get(ctx context.Context, key string) (domain.Session, bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + var value domain.Session + data, err := s.client.Get(ctx, "session:"+key).Bytes() + if errors.Is(err, redis.Nil) { + return value, false, nil + } + if err != nil { + return value, false, unavailable(err) + } + if err = json.Unmarshal(data, &value); err != nil { + return value, false, unavailable(err) + } + return value, time.Now().Before(value.ExpiresAt), nil +} + +func (s *Redis) Delete(ctx context.Context, key string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + count, err := s.client.Del(ctx, "session:"+key).Result() + return count > 0, unavailable(err) +} diff --git a/internal/usecase/chat/repository.go b/internal/usecase/chat/repository.go new file mode 100644 index 0000000..004d1f6 --- /dev/null +++ b/internal/usecase/chat/repository.go @@ -0,0 +1,28 @@ +package chat + +import ( + "context" + "errors" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +var ( + ErrUsernameTaken = errors.New("username is already taken") + ErrUserNotFound = errors.New("user not found") + ErrChatNotFound = errors.New("chat not found") + ErrNotMember = errors.New("user is not a chat member") +) + +// Repository is the persistence port required by the chat use case. +type Repository interface { + Ready(ctx context.Context) error + UserByUsername(ctx context.Context, username string) (domain.User, string, bool, error) + CreateUser(ctx context.Context, username, displayName, passwordHash string) (domain.User, error) + User(ctx context.Context, id string) (domain.User, bool, error) + CreateChat(ctx context.Context, title string, memberIDs []string) (domain.Chat, error) + ChatForUser(ctx context.Context, chatID, userID string) (domain.Chat, error) + ListChats(ctx context.Context, userID string, after *domain.Position, limit int) ([]domain.Chat, error) + CreateMessage(ctx context.Context, chatID, senderID, text, key string) (domain.Message, error) + ListMessages(ctx context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) +} diff --git a/internal/usecase/chat/service.go b/internal/usecase/chat/service.go new file mode 100644 index 0000000..0ca9425 --- /dev/null +++ b/internal/usecase/chat/service.go @@ -0,0 +1,182 @@ +package chat + +import ( + "context" + "errors" + "strings" + "time" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +type UseCase struct { + repository Repository + sessions Sessions + sessionTTL time.Duration +} + +func New(repository Repository, sessions Sessions, ttl time.Duration) *UseCase { + return &UseCase{repository: repository, sessions: sessions, sessionTTL: ttl} +} + +func (u *UseCase) CreateUser(ctx context.Context, user domain.User, secret string) (domain.User, error) { + if !validUsername(user.Username) { + return domain.User{}, invalid("username должен соответствовать [a-z0-9_]{3,32}") + } + if !validRequiredText(user.DisplayName, 100) { + return domain.User{}, invalid("display_name должен содержать от 1 до 100 символов") + } + if !validPassword(secret) { + return domain.User{}, invalid("password должен содержать от 12 до 128 символов") + } + hash, err := password.Hash(ctx, secret) + if err != nil { + return domain.User{}, err + } + created, err := u.repository.CreateUser(ctx, user.Username, user.DisplayName, hash) + if errors.Is(err, ErrUsernameTaken) { + return domain.User{}, domain.NewApplicationError(domain.ErrorConflict, "Пользователь с таким username уже существует") + } + return created, err +} + +func (u *UseCase) CreateChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) { + if !validRequiredText(chat.Title, 100) { + return domain.Chat{}, invalid("title должен содержать от 1 до 100 символов") + } + if len(chat.MemberIDs) < 1 || len(chat.MemberIDs) > 100 { + return domain.Chat{}, invalid("member_ids должен содержать от 1 до 100 участников") + } + + normalizedIDs := make([]string, len(chat.MemberIDs)) + seen := make(map[string]struct{}, len(chat.MemberIDs)) + callerIncluded := false + for index, memberID := range chat.MemberIDs { + normalized, valid := normalizeUUID(memberID) + if !valid { + return domain.Chat{}, invalid("member_ids содержит некорректный UUID") + } + if _, duplicate := seen[normalized]; duplicate { + return domain.Chat{}, invalid("member_ids содержит повторяющиеся UUID") + } + normalizedIDs[index] = normalized + seen[normalized] = struct{}{} + callerIncluded = callerIncluded || normalized == callerID + } + if !callerIncluded { + return domain.Chat{}, invalid("текущий пользователь должен входить в member_ids") + } + + created, err := u.repository.CreateChat(ctx, chat.Title, normalizedIDs) + if errors.Is(err, ErrUserNotFound) { + return domain.Chat{}, invalid("member_ids содержит неизвестного пользователя") + } + return created, err +} + +func (u *UseCase) GetChat(ctx context.Context, callerID string, chat domain.Chat) (domain.Chat, error) { + chatID, valid := normalizeUUID(chat.ID) + if !valid { + return domain.Chat{}, invalid("Некорректный chat_id") + } + return u.chatForUser(ctx, chatID, callerID) +} + +func (u *UseCase) ListChats(ctx context.Context, callerID string, pagination domain.Pagination) (domain.ChatPage, error) { + if pagination.Limit < 1 || pagination.Limit > 100 { + return domain.ChatPage{}, invalid("limit должен быть целым числом от 1 до 100") + } + items, err := u.repository.ListChats(ctx, callerID, pagination.After, pagination.Limit+1) + if err != nil { + return domain.ChatPage{}, err + } + page := domain.ChatPage{Items: items, HasMore: len(items) > pagination.Limit} + if page.HasMore { + page.Items = page.Items[:pagination.Limit] + } + return page, nil +} + +func (u *UseCase) CreateMessage(ctx context.Context, callerID string, message domain.Message) (domain.Message, error) { + chatID, valid := normalizeUUID(message.ChatID) + if !valid { + return domain.Message{}, invalid("Некорректный chat_id") + } + if _, err := u.chatForUser(ctx, chatID, callerID); err != nil { + return domain.Message{}, err + } + if !utf8.ValidString(message.Text) || utf8.RuneCountInString(message.Text) > 4000 || (strings.TrimSpace(message.Text) == "" && len(message.AttachmentIDs) == 0) { + return domain.Message{}, invalid("text должен содержать от 1 до 4000 символов") + } + if len(message.AttachmentIDs) > 10 { + return domain.Message{}, invalid("Не более 10 вложений") + } + seen := map[string]bool{} + for i, id := range message.AttachmentIDs { + normalized, ok := normalizeUUID(id) + if !ok || seen[normalized] { + return domain.Message{}, invalid("Некорректные или повторные attachment_ids") + } + seen[normalized] = true + message.AttachmentIDs[i] = normalized + } + var created domain.Message + var err error + if repository, ok := u.repository.(interface { + CreateMessageWithAttachments(context.Context, string, string, string, string, []string) (domain.Message, error) + }); ok { + created, err = repository.CreateMessageWithAttachments(ctx, chatID, callerID, message.Text, message.IdempotencyKey, message.AttachmentIDs) + } else if len(message.AttachmentIDs) > 0 { + return domain.Message{}, notFound() + } else { + created, err = u.repository.CreateMessage(ctx, chatID, callerID, message.Text, message.IdempotencyKey) + } + if errors.Is(err, ErrChatNotFound) || errors.Is(err, ErrNotMember) { + return domain.Message{}, notFound() + } + return created, err +} + +func (u *UseCase) ListMessages(ctx context.Context, callerID string, chat domain.Chat, pagination domain.Pagination) (domain.MessagePage, error) { + chatID, valid := normalizeUUID(chat.ID) + if !valid { + return domain.MessagePage{}, invalid("Некорректный chat_id") + } + if _, err := u.chatForUser(ctx, chatID, callerID); err != nil { + return domain.MessagePage{}, err + } + if pagination.Limit < 1 || pagination.Limit > 100 { + return domain.MessagePage{}, invalid("limit должен быть целым числом от 1 до 100") + } + items, err := u.repository.ListMessages(ctx, chatID, pagination.After, pagination.Limit+1) + if err != nil { + return domain.MessagePage{}, err + } + page := domain.MessagePage{Items: items, HasMore: len(items) > pagination.Limit} + if page.HasMore { + page.Items = page.Items[:pagination.Limit] + } + return page, nil +} + +func (u *UseCase) chatForUser(ctx context.Context, chatID, callerID string) (domain.Chat, error) { + chat, err := u.repository.ChatForUser(ctx, chatID, callerID) + if errors.Is(err, ErrChatNotFound) || errors.Is(err, ErrNotMember) { + return domain.Chat{}, notFound() + } + return chat, err +} + +func invalid(message string) error { + return domain.NewApplicationError(domain.ErrorInvalid, message) +} + +func unauthorized() error { + return domain.NewApplicationError(domain.ErrorUnauthorized, "Требуется действительная сессия") +} + +func notFound() error { + return domain.NewApplicationError(domain.ErrorNotFound, "Объект не найден") +} diff --git a/internal/usecase/chat/session.go b/internal/usecase/chat/session.go new file mode 100644 index 0000000..07d1668 --- /dev/null +++ b/internal/usecase/chat/session.go @@ -0,0 +1,102 @@ +package chat + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "strings" + "time" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" +) + +type Sessions interface { + Put(context.Context, string, domain.Session) error + Get(context.Context, string) (domain.Session, bool, error) + Delete(context.Context, string) (bool, error) + Ready(context.Context) error +} + +func validPassword(value string) bool { + n := utf8.RuneCountInString(value) + return utf8.ValidString(value) && n >= 12 && n <= 128 && strings.TrimSpace(value) != "" +} + +func tokenKey(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func (u *UseCase) Authenticate(ctx context.Context, token string) (domain.User, error) { + decoded, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(decoded) != 32 { + return domain.User{}, unauthorized() + } + session, exists, err := u.sessions.Get(ctx, tokenKey(token)) + if err != nil { + return domain.User{}, err + } + if !exists || !time.Now().Before(session.ExpiresAt) { + return domain.User{}, unauthorized() + } + user, exists, err := u.repository.User(ctx, session.UserID) + if err != nil { + return domain.User{}, err + } + if !exists { + return domain.User{}, unauthorized() + } + return user, nil +} + +func (u *UseCase) CreateSession(ctx context.Context, username, secret string) (domain.Login, error) { + if !validUsername(username) || !validPassword(secret) { + return domain.Login{}, unauthorized() + } + user, hash, exists, err := u.repository.UserByUsername(ctx, username) + if err != nil { + return domain.Login{}, err + } + if !exists { + hash = password.DummyHash + } + valid, err := password.Verify(ctx, secret, hash) + if err != nil { + return domain.Login{}, err + } + if !valid || !exists { + return domain.Login{}, unauthorized() + } + var raw [32]byte + if _, err = rand.Read(raw[:]); err != nil { + return domain.Login{}, err + } + token := base64.RawURLEncoding.EncodeToString(raw[:]) + expires := time.Now().UTC().Add(u.sessionTTL) + if err = u.sessions.Put(ctx, tokenKey(token), domain.Session{UserID: user.ID, ExpiresAt: expires}); err != nil { + return domain.Login{}, err + } + return domain.Login{Token: token, ExpiresAt: expires, User: user}, nil +} + +func (u *UseCase) DeleteSession(ctx context.Context, token string) error { + deleted, err := u.sessions.Delete(ctx, tokenKey(token)) + if err != nil { + return err + } + if !deleted { + return unauthorized() + } + return nil +} + +func (u *UseCase) Ready(ctx context.Context) error { + if err := u.repository.Ready(ctx); err != nil { + return err + } + return u.sessions.Ready(ctx) +} diff --git a/internal/usecase/chat/session_test.go b/internal/usecase/chat/session_test.go new file mode 100644 index 0000000..5096e1b --- /dev/null +++ b/internal/usecase/chat/session_test.go @@ -0,0 +1,19 @@ +package chat + +import ( + "strings" + "testing" +) + +func TestPasswordContract(t *testing.T) { + for _, value := range []string{"", "too-short", strings.Repeat(" ", 12), strings.Repeat("x", 129), strings.Repeat("x", 12) + string([]byte{0xff})} { + if validPassword(value) { + t.Fatal("invalid password accepted") + } + } + for _, value := range []string{strings.Repeat("🔑", 128), "valid:password", " password with spaces "} { + if !validPassword(value) { + t.Fatal("valid Unicode password rejected") + } + } +} diff --git a/internal/usecase/chat/validation.go b/internal/usecase/chat/validation.go new file mode 100644 index 0000000..5fe43df --- /dev/null +++ b/internal/usecase/chat/validation.go @@ -0,0 +1,37 @@ +package chat + +import ( + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +func normalizeUUID(value string) (string, bool) { + parsed, err := uuid.Parse(value) + if err != nil { + return "", false + } + normalized := parsed.String() + if !strings.EqualFold(value, normalized) { + return "", false + } + return normalized, true +} + +func validUsername(value string) bool { + if len(value) < 3 || len(value) > 32 { + return false + } + for _, char := range value { + if char != '_' && (char < 'a' || char > 'z') && (char < '0' || char > '9') { + return false + } + } + return true +} + +func validRequiredText(value string, maxRunes int) bool { + length := utf8.RuneCountInString(value) + return utf8.ValidString(value) && length >= 1 && length <= maxRunes && strings.TrimSpace(value) != "" +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..2433b27 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,54 @@ +-- +goose Up +-- +goose StatementBegin +DO $$ +BEGIN + -- Adopt the schema created by the previous runner without changing its data. + IF to_regclass('schema_migrations') IS NOT NULL THEN + IF NOT EXISTS (SELECT 1 FROM schema_migrations WHERE version = '001_initial.sql') + OR to_regclass('users') IS NULL OR to_regclass('chats') IS NULL + OR to_regclass('chat_members') IS NULL OR to_regclass('messages') IS NULL THEN + RAISE EXCEPTION 'incomplete legacy schema: cannot adopt migration 1'; + END IF; + RETURN; + END IF; + CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ); + CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ); + CREATE INDEX chats_history_idx ON chats (created_at, id); + CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + ordinal integer NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, ordinal) + ); + CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) + ); + CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +END; +$$; +-- +goose StatementEnd + +-- +goose Down +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; +-- The old journal must not claim that the dropped schema is still applied. +DROP TABLE IF EXISTS schema_migrations; diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..fb02f93 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +-- +goose Up +-- Additive: old lab2 writers can still insert NULL keys during migration. +ALTER TABLE messages ADD COLUMN IF NOT EXISTS idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); +CREATE UNIQUE INDEX IF NOT EXISTS messages_idempotency_idx + ON messages(sender_id, chat_id, idempotency_key); + +-- +goose Down +DROP INDEX messages_idempotency_idx; +ALTER TABLE messages DROP COLUMN idempotency_key; diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..a8b812a --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + uploader_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready','failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)) +); + +-- +goose Down +DROP TABLE IF EXISTS attachments; diff --git a/migrations/004_message_attachments.sql b/migrations/004_message_attachments.sql new file mode 100644 index 0000000..1fd41a9 --- /dev/null +++ b/migrations/004_message_attachments.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE messages + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD COLUMN IF NOT EXISTS attachment_ids uuid[] NOT NULL DEFAULT '{}'; + +-- +goose Down +-- Attachment-only messages cannot exist in lab3; retain an explicit placeholder when downgrading. +UPDATE messages SET text='[attachment]' WHERE text=''; +ALTER TABLE messages + DROP COLUMN IF EXISTS attachment_ids, + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000); diff --git a/migrations/005_async_attachments.sql b/migrations/005_async_attachments.sql new file mode 100644 index 0000000..e8deb84 --- /dev/null +++ b/migrations/005_async_attachments.sql @@ -0,0 +1,18 @@ +-- +goose Up +ALTER TABLE attachments + DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check CHECK (status IN ('queued','processing','ready','failed')), + DROP CONSTRAINT attachments_check, + ADD CONSTRAINT attachments_check CHECK ((status='failed' AND error_code IS NOT NULL) OR (status<>'failed' AND error_code IS NULL)), + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text; + +-- +goose Down +UPDATE attachments SET status='failed',error_code='pipeline_downgrade' WHERE status IN ('queued','processing'); +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check CHECK (status IN ('ready','failed')), + DROP CONSTRAINT attachments_check, + ADD CONSTRAINT attachments_check CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)), + DROP COLUMN variants, DROP COLUMN updated_at, DROP COLUMN pipeline_version, DROP COLUMN staging_key; diff --git a/migrations/006_attachment_jobs.sql b/migrations/006_attachment_jobs.sql new file mode 100644 index 0000000..96fe871 --- /dev/null +++ b/migrations/006_attachment_jobs.sql @@ -0,0 +1,18 @@ +-- +goose Up +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1, + state text NOT NULL CHECK (state IN ('queued','processing','ready','failed')), + attempts integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 20), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_until timestamptz, + fence bigint NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_jobs; diff --git a/migrations/007_attachment_outbox.sql b/migrations/007_attachment_outbox.sql new file mode 100644 index 0000000..4aa021c --- /dev/null +++ b/migrations/007_attachment_outbox.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE attachment_outbox ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + stream_id text +); + +-- +goose Down +DROP TABLE attachment_outbox; diff --git a/migrations/008_attachment_replays.sql b/migrations/008_attachment_replays.sql new file mode 100644 index 0000000..2587f83 --- /dev/null +++ b/migrations/008_attachment_replays.sql @@ -0,0 +1,12 @@ +-- +goose Up +CREATE TABLE attachment_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + operator text NOT NULL CHECK (char_length(operator) BETWEEN 1 AND 100), + reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 300), + previous_error text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- +goose Down +DROP TABLE attachment_replays; diff --git a/migrations/009_backfill_pipeline.sql b/migrations/009_backfill_pipeline.sql new file mode 100644 index 0000000..ac2b71e --- /dev/null +++ b/migrations/009_backfill_pipeline.sql @@ -0,0 +1,12 @@ +-- +goose Up +-- Re-process accepted lab4 originals; original keys remain durable staging until ready. +INSERT INTO attachment_jobs(id,attachment_id,state,max_attempts) +SELECT id,id,'queued',5 FROM attachments WHERE status='ready' AND variants='[]'::jsonb; +UPDATE attachments SET staging_key=object_key, + object_key='original/'||id::text||'/v1/original',status='queued',updated_at=clock_timestamp() +WHERE id IN (SELECT attachment_id FROM attachment_jobs); +INSERT INTO attachment_outbox(job_id) SELECT id FROM attachment_jobs; + +-- +goose Down +-- Do not erase accepted processing jobs during a downgrade. Schema Down is destructive. +SELECT 1; diff --git a/migrations/010_storage_cleanup.sql b/migrations/010_storage_cleanup.sql new file mode 100644 index 0000000..62cbac5 --- /dev/null +++ b/migrations/010_storage_cleanup.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE attachments ADD COLUMN storage_cleaned_at timestamptz; + +-- +goose Down +ALTER TABLE attachments DROP COLUMN storage_cleaned_at; diff --git a/migrations/011_job_trace_context.sql b/migrations/011_job_trace_context.sql new file mode 100644 index 0000000..51d4185 --- /dev/null +++ b/migrations/011_job_trace_context.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE attachment_jobs ADD COLUMN traceparent text NOT NULL DEFAULT '', ADD COLUMN tracestate text NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE attachment_jobs DROP COLUMN traceparent, DROP COLUMN tracestate; diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..3a18367 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +chmod 700 secrets +for name in postgres_password cursor_key s3_root_secret_key s3_secret_key; do + if [ ! -s "secrets/$name" ]; then + openssl rand -hex 32 > "secrets/$name" + fi +done +for name in s3_root_access_key s3_access_key; do + if [ ! -s "secrets/$name" ]; then + openssl rand -hex 8 > "secrets/$name" + fi +done +# Compose mounts files directly on Linux; the unprivileged API must read them. +# The host directory remains private (0700). +chmod 444 secrets/postgres_password secrets/cursor_key secrets/s3_root_access_key secrets/s3_root_secret_key secrets/s3_access_key secrets/s3_secret_key +printf '%s\n' 'Local secrets are ready.' diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 0000000..530b087 --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if docker compose version >/dev/null 2>&1; then + exec docker compose --env-file /dev/null -f compose.yaml "$@" +fi +exec docker-compose --env-file /dev/null -f compose.yaml "$@" diff --git a/scripts/observability_backend.py b/scripts/observability_backend.py new file mode 100644 index 0000000..71f522d --- /dev/null +++ b/scripts/observability_backend.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Measure actual trace retention and bounded buffering during a backend outage.""" +import datetime +import json +import subprocess +import sys +import time + +from observability_common import ROOT, compose, get_json, ready, save, scalar, start_services +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + + +def snapshot(): + return {'failed_spans': scalar('sum(otelcol_exporter_send_failed_spans_total)'), + 'refused_spans': scalar('sum(otelcol_receiver_refused_spans_total)'), + 'queued_batches': scalar('otelcol_exporter_queue_size{exporter="otlphttp/tempo"}'), + 'rss_bytes': scalar('otelcol_process_memory_rss_bytes')} + + +def main(): + ready() + c = Client('http://127.0.0.1:18086') + actor = c.user('backend') + chat = c.call('POST', '/api/v1/chats', 201, actor=actor, + body={'title': 'Backend outage', 'member_ids': [actor['id']]}, schema='Chat')['json'] + report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'before': snapshot()} + trace_ids, timeline = [], [] + try: + compose('stop', 'tempo') + started = time.monotonic() + for index in range(35): + response = c.message(actor, chat, 'synthetic backend outage') + trace_ids.append(response['headers']['X-Trace-Id']) + if index % 5 == 0: + timeline.append({'seconds': round(time.monotonic()-started, 2), **snapshot()}) + time.sleep(1) + report['during'] = timeline + report['successful_writes'] = len(trace_ids) + finally: + start_services('tempo') + for _ in range(24): + time.sleep(2) + if snapshot()['queued_batches'] == 0: + break + time.sleep(6) + report['after'] = snapshot() + retained = 0 + trace_results = [] + for trace_id in trace_ids: + found = False + try: + d = get_json('http://127.0.0.1:13200/api/traces/' + trace_id) + found = bool(d.get('batches') or d.get('resourceSpans')) + except Exception: + pass + retained += int(found) + trace_results.append({'trace_id': trace_id, 'found': found}) + report['request_trace_results'] = trace_results + report['retained_request_traces'] = retained + report['missing_request_traces'] = len(trace_ids)-retained + report['exporter_failed_spans_delta'] = report['after']['failed_spans']-report['before']['failed_spans'] + report['receiver_refused_spans_delta'] = report['after']['refused_spans']-report['before']['refused_spans'] + info = json.loads(subprocess.check_output(['docker', 'inspect', '--format', + '{"oom":{{json .State.OOMKilled}},"memory_limit":{{.HostConfig.Memory}}}', 'mtusi2026lab6-collector-1'], text=True)) + report['collector'] = info + assert len(trace_ids) == 35, 'business writes failed' + assert not info['oom'], 'Collector exceeded memory cgroup' + assert max(x['queued_batches'] for x in timeline) <= 256, 'unbounded export queue' + assert report['after']['queued_batches'] == 0, 'export queue did not drain' + assert max(x['rss_bytes'] for x in timeline) < info['memory_limit'], 'RSS exceeded cgroup' + report['status'] = 'PASS' + report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + save('evidence/lab6-backend-outage.json', report) + print(json.dumps({k: report[k] for k in ('status', 'successful_writes', 'retained_request_traces', 'missing_request_traces', 'exporter_failed_spans_delta')})) + + +if __name__ == '__main__': + main() diff --git a/scripts/observability_common.py b/scripts/observability_common.py new file mode 100644 index 0000000..5e76f38 --- /dev/null +++ b/scripts/observability_common.py @@ -0,0 +1,58 @@ +"""Local lab6 evidence helpers. Never persist credentials or response bodies.""" +import json +import subprocess +import time +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, check=True): + return subprocess.run([str(ROOT / 'scripts/compose.sh'), *args], cwd=ROOT, + check=check, capture_output=True, text=True) + + +def start_services(*services): + # Compose `start` may start dependencies as well. Fault injection must keep + # an intentionally stopped dependency stopped until its explicit recovery. + for service in services: + ids = compose('ps', '-a', '-q', service).stdout.split() + if not ids: + raise RuntimeError('Service container not created: ' + service) + subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL) + + +def get_json(url, timeout=10): + with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=timeout) as r: + return json.load(r) + + +def prom(query): + result = get_json('http://127.0.0.1:19090/api/v1/query?' + urllib.parse.urlencode({'query': query})) + if result.get('status') != 'success': + raise RuntimeError('Prometheus query failed') + return result['data']['result'] + + +def scalar(query): + return sum(float(x['value'][1]) for x in prom(query)) + + +def ready(base='http://127.0.0.1:18086', timeout=120): + end = time.monotonic() + timeout + while time.monotonic() < end: + try: + if get_json(base + '/health/ready'): + return + except Exception: + pass + time.sleep(1) + raise RuntimeError('API readiness deadline exceeded') + + +def save(path, value): + path = ROOT / path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') diff --git a/scripts/observability_experiment.py b/scripts/observability_experiment.py new file mode 100644 index 0000000..fc01ddc --- /dev/null +++ b/scripts/observability_experiment.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Track 1: fixed-rate, equal-sized trials with measured sampling retention.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import datetime +import json +import math +import os +import subprocess +import sys +import threading +import time +import uuid + +from observability_common import ROOT, compose, get_json, prom, ready, save +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + + +def percentile(values, p): + return round(sorted(values)[max(0, math.ceil(len(values) * p) - 1)] * 1000, 3) if values else None + + +def snapshot(): + queries = { + 'exported_spans': 'sum(otelcol_exporter_sent_spans_total)', + 'failed_spans': 'sum(otelcol_exporter_send_failed_spans_total)', + 'accepted_spans': 'sum(otelcol_receiver_accepted_spans_total)', + 'refused_spans': 'sum(otelcol_receiver_refused_spans_total)', + 'tempo_ingested_bytes': 'sum(tempo_distributor_bytes_received_total)', + 'loki_ingested_bytes': 'sum(loki_distributor_bytes_received_total)', + 'prometheus_samples': 'sum(prometheus_tsdb_head_samples_appended_total)', + } + result = {} + for key, query in queries.items(): + rows = prom(query) + result[key] = sum(float(x['value'][1]) for x in rows) if rows else None + return result + + +def samples(stop, output): + while not stop.is_set(): + result = subprocess.run(['docker', 'stats', '--no-stream', '--format', '{{json .}}'], capture_output=True, text=True) + for line in result.stdout.splitlines(): + row = json.loads(line) + if row['Name'].startswith('mtusi2026lab6-'): + output.append({'at': time.time(), **{k: row[k] for k in ('Name', 'CPUPerc', 'MemUsage')}}) + stop.wait(1) + + +def configure(disabled, ratio): + # Compose interpolation only; no secrets or .env files are read. + os.environ['OTEL_SDK_DISABLED'] = str(disabled).lower() + os.environ['OTEL_TRACES_SAMPLER_ARG'] = str(ratio) + compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker') + ready() + time.sleep(8) + + +def trial(name, disabled, ratio, args, repetition): + configure(disabled, ratio) + c = Client(args.base_url) + actor = c.user('research') + chat = c.call('POST', '/api/v1/chats', 201, actor=actor, + body={'title': 'Synthetic research', 'member_ids': [actor['id']]}, schema='Chat')['json'] + # Equal warmup before every trial, including telemetry off. + for _ in range(20): + c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + time.sleep(6) + before = snapshot() + stop, resource = threading.Event(), [] + sampler = threading.Thread(target=samples, args=(stop, resource), daemon=True) + sampler.start() + known, latency, errors, uploads = [], [], [], [] + lock = threading.Lock() + started = time.monotonic() + + def request(index): + scheduled = started + index / args.rate + time.sleep(max(0, scheduled - time.monotonic())) + client = Client(args.base_url) + trace_id = None + headers = {} + begin = time.monotonic() + outcome, kind = 'ok', 'read' + try: + if index % 20 == 0: + kind = 'upload' + headers['Content-Type'] = 'image/png' + response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', + 202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), + headers=headers, schema='Attachment') + att = response['json'] + with lock: + uploads.append(att) + elif index % 20 == 1: + kind = 'client_error' + response = client.call('GET', '/api/v1/users/me', 401, headers=headers) + elif index % 5 == 0: + kind = 'write' + headers['Idempotency-Key'] = uuid.uuid4().hex + response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=actor, + body={'text': 'synthetic fixed-size benchmark message'}, headers=headers, schema='Message') + else: + response = client.call('GET', '/api/v1/users/me', actor=actor, headers=headers, schema='User') + trace_id = response['headers'].get('X-Trace-Id') + except Exception as exc: + outcome = type(exc).__name__ + with lock: + elapsed_request = time.monotonic() - begin + latency.append(elapsed_request) + known.append({'trace_id': trace_id, 'kind': kind, 'outcome': outcome, 'elapsed_seconds': elapsed_request}) + if outcome != 'ok': + errors.append(outcome) + + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(request, range(args.requests))) + elapsed = time.monotonic() - started + for att in uploads: + c.wait_ready(actor, att) + stop.set() + sampler.join(timeout=10) + # Flush SDK batches, Collector and backend scrape before taking counters. + time.sleep(12) + after = snapshot() + retained = {kind: {'sent': 0, 'found': 0} for kind in ('read', 'write', 'upload', 'client_error')} + slow = {'definition': 'slowest 5 percent of this trial by measured client latency', 'sent': 0, 'found': 0} + slow_ids = {id(x) for x in sorted(known, key=lambda x: x['elapsed_seconds'], reverse=True)[:max(1, math.ceil(.05*len(known)))]} + for row in known: + retained[row['kind']]['sent'] += 1 + is_slow = id(row) in slow_ids + slow['sent'] += int(is_slow) + try: + trace = get_json('http://127.0.0.1:13200/api/traces/' + (row['trace_id'] or 'missing')) + if trace.get('batches') or trace.get('resourceSpans'): + retained[row['kind']]['found'] += 1 + slow['found'] += int(is_slow) + except Exception: + pass + return {'configuration': name, 'repetition': repetition, 'disabled': disabled, 'sampling_ratio': ratio, + 'requests': args.requests, 'target_rps': args.rate, 'concurrency_limit': args.concurrency, + 'elapsed_seconds': round(elapsed, 3), 'successful_rps': round((args.requests-len(errors))/elapsed, 3), + 'unexpected_errors': len(errors), 'error_types': errors, + 'all_request_latency_ms': {f'p{int(p*100)}': percentile(latency, p) for p in (.5, .95, .99)}, + 'ready_uploads': len(uploads), 'counter_before': before, 'counter_after': after, + 'counter_delta': {k: after[k]-before[k] if after[k] is not None and before[k] is not None else None for k in before}, + 'trace_retention': retained, 'slow_request_trace_retention': slow, 'resources': resource} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default='http://127.0.0.1:18086') + p.add_argument('--requests', type=int, default=120) + p.add_argument('--rate', type=float, default=8) + p.add_argument('--concurrency', type=int, default=4) + p.add_argument('--repetitions', type=int, default=2) + args = p.parse_args() + if args.requests < 20 or args.rate <= 0 or args.concurrency < 1 or args.repetitions < 1: + p.error('requests>=20, rate>0, concurrency>=1, repetitions>=1 required') + result = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'method': 'Fixed-rate schedule, fixed request count/mix; 20 warmups; SDK and backend flush; sampled IDs queried from Tempo.', + 'trials': []} + try: + configs = [('off', True, 1), ('full', False, 1), ('ten_percent', False, .1), ('balanced', False, .5)] + for rep in range(args.repetitions): + # Counterbalance order on second repetition. + for name, disabled, ratio in (configs if rep % 2 == 0 else list(reversed(configs))): + print('Running', name, 'repetition', rep + 1, flush=True) + row = trial(name, disabled, ratio, args, rep + 1) + result['trials'].append(row) + save('evidence/lab6-research.json', result) + print(json.dumps({k: row[k] for k in ('configuration', 'unexpected_errors', 'all_request_latency_ms', 'trace_retention')}), flush=True) + assert row['unexpected_errors'] == 0, 'Unexpected business errors during research' + found = sum(x['found'] for x in row['trace_retention'].values()) + if name == 'off': + assert found == 0, 'SDK off still exported request traces' + elif name == 'full': + assert found == args.requests, 'Full sampling lost request traces' + else: + assert 0 < found < args.requests, 'Sampling setting had no observed effect' + result['status'] = 'PASS' + finally: + configure(False, 1) + result['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + save('evidence/lab6-research.json', result) + + +if __name__ == '__main__': + main() diff --git a/scripts/observability_faults.py b/scripts/observability_faults.py new file mode 100644 index 0000000..7250d39 --- /dev/null +++ b/scripts/observability_faults.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Repeatable worker/DB/backend faults on this lab's isolated Compose project.""" +import argparse +import datetime +import json +import subprocess +import sys +import time +import urllib.parse +import uuid +from concurrent.futures import ThreadPoolExecutor + +from observability_common import ROOT, compose, get_json, prom, ready, save, start_services +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + + +def alerts(): + return get_json('http://127.0.0.1:19090/api/v1/alerts')['data']['alerts'] + + +def collect_trace(trace_id): + for _ in range(20): + try: + trace = get_json('http://127.0.0.1:13200/api/traces/' + trace_id) + if trace.get('batches') or trace.get('resourceSpans'): + save('evidence/trace-' + trace_id + '.json', trace) + query = '{service_name=~"messenger-api|messenger-worker"} | trace_id = "' + trace_id + '"' + logs = get_json('http://127.0.0.1:13100/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'since': '1h', 'limit': 100})) + if not logs['data']['result']: + raise RuntimeError('correlated logs not yet available') + save('evidence/logs-' + trace_id + '.json', logs) + return {'trace_id': trace_id, 'trace_file': 'evidence/trace-' + trace_id + '.json', + 'log_streams': len(logs['data']['result']), 'logs_file': 'evidence/logs-' + trace_id + '.json'} + except Exception: + pass + time.sleep(2) + raise RuntimeError('trace/log correlation not found in backend') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default='http://127.0.0.1:18086') + args = p.parse_args() + ready(args.base_url) + c = Client(args.base_url) + actor = c.user('incidents') + chat = c.call('POST', '/api/v1/chats', 201, actor=actor, + body={'title': 'Synthetic incidents', 'member_ids': [actor['id']]}, schema='Chat')['json'] + report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'cases': {}} + traces = [] + try: + print('Worker outage', flush=True) + compose('stop', 'worker') + attachments = [] + started = time.monotonic() + for _ in range(6): + trace_id = uuid.uuid4().hex + result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', + 202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), + headers={'Content-Type': 'image/png', 'traceparent': '00-' + trace_id + '-1234567890123456-01'}, + schema='Attachment') + attachments.append(result['json']) + traces.append(trace_id) + timeline = [] + for _ in range(6): + time.sleep(5) + timeline.append({'seconds': round(time.monotonic()-started, 2), 'alerts': alerts(), + 'queue': prom('messenger_queue_depth'), 'age': prom('messenger_queue_oldest_age_seconds')}) + # Keep S3 unavailable for the first attempt; the same durable trace must + # cover the failed attempt, retry publication and eventual ready. + compose('stop', 'minio') + start_services('worker') + time.sleep(10) + report['cases']['s3_retry'] = {'alerts_during': alerts(), 'jobs': prom('messenger_jobs_total')} + start_services('minio') + ready(args.base_url) + for att in attachments: + c.wait_ready(actor, att) + time.sleep(35) + report['cases']['worker_stop'] = {'queued': len(attachments), 'ready_after_recovery': len(attachments), + 'timeline': timeline, 'resolved': alerts(), + 'queue_after': prom('messenger_queue_depth')} + save('evidence/lab6-faults.json', report) + print('DB lock', flush=True) + lock = subprocess.Popen([str(ROOT / 'scripts/compose.sh'), 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', + '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c', + 'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(45); COMMIT;'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(2) + observations = [] + def write(index): + trace_id = uuid.uuid4().hex + result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500, 503], actor=actor, + body={'text': 'synthetic incident'}, headers={'Idempotency-Key': uuid.uuid4().hex, + 'traceparent': '00-' + trace_id + '-1234567890123456-01'}) + return {'status': result['status'], 'elapsed': result['elapsed'], 'trace_id': trace_id} + with ThreadPoolExecutor(max_workers=2) as pool: + for block in range(7): + observations.extend(pool.map(write, range(4))) + report['cases']['db_lock'] = {'requests': observations, 'alerts_during': alerts()} + save('evidence/lab6-faults.json', report) + lock.wait(timeout=60) + c.message(actor, chat, 'synthetic recovery') + recovery_timeline = [] + for _ in range(24): + time.sleep(5) + state = alerts() + recovery_timeline.append(state) + if not any(a['state'] == 'firing' for a in state): + break + c.message(actor, chat, 'synthetic recovery probe') + report['cases']['db_lock']['recovery_alerts'] = recovery_timeline + report['cases']['db_lock']['alerts_after'] = alerts() + failed = next((x for x in observations if x['status'] >= 500), None) + if failed: + traces.append(failed['trace_id']) + report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])] + print('Telemetry backend outage', flush=True) + compose('stop', 'tempo') + successful = 0 + for _ in range(30): + c.message(actor, chat, 'synthetic telemetry outage') + successful += 1 + time.sleep(.5) + report['cases']['tempo_stop'] = {'successful_writes': successful, + 'exporter_failed': prom('otelcol_exporter_send_failed_spans_total'), + 'queue_size': prom('otelcol_exporter_queue_size'), + 'collector_memory': prom('otelcol_process_memory_rss_bytes')} + start_services('tempo') + time.sleep(20) + report['cases']['tempo_stop']['after'] = {'queue_size': prom('otelcol_exporter_queue_size'), + 'collector_memory': prom('otelcol_process_memory_rss_bytes')} + assert any(a['labels']['alertname'] == 'QueueLagGrowing' and a['state'] == 'firing' + for row in report['cases']['worker_stop']['timeline'] for a in row['alerts']), 'queue alert never fired' + assert any(a['labels']['alertname'] == 'APIHighErrorRate' and a['state'] == 'firing' + for a in report['cases']['db_lock']['alerts_during']), 'API alert never fired' + assert not any(a['state'] == 'firing' for a in report['cases']['db_lock']['alerts_after']), 'alerts did not resolve' + assert all(float(x['value'][1]) == 0 for x in report['cases']['worker_stop']['queue_after']), 'backlog not empty' + report['status'] = 'PASS' + finally: + start_services('worker', 'tempo', 'minio') + report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + save('evidence/lab6-faults.json', report) + print(json.dumps({'status': report['status'], 'cases': list(report['cases'])}), flush=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d614508 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec ./scripts/compose.sh restart postgres redis minio worker api gateway collector prometheus loki tempo grafana diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..d0f2807 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?target instance is required}" +for container in $(./scripts/compose.sh ps -q api); do + alias=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 8 "$container" >/dev/null + exit 0 + fi +done +printf '%s\n' 'No matching API replica in this project' >&2 +exit 1 diff --git a/scripts/summarize_observability.py b/scripts/summarize_observability.py new file mode 100644 index 0000000..d770380 --- /dev/null +++ b/scripts/summarize_observability.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Create a compact, reproducible table from measured track-1 JSON.""" +import csv +import json +import re +import statistics + +from observability_common import ROOT + + +def mib(value): + match = re.fullmatch(r'([0-9.]+)([A-Za-z]+)', value.strip()) + if not match: + raise ValueError('Unrecognized Docker memory unit') + factor = {'B': 1/1048576, 'KiB': 1/1024, 'MiB': 1, 'GiB': 1024, + 'kB': 1000/1048576, 'MB': 1000000/1048576, 'GB': 1000000000/1048576} + return float(match[1]) * factor[match[2]] + + +def summarize(row): + by_container = {} + for resource in row['resources']: + group = by_container.setdefault(resource['Name'], {'cpu': [], 'mem': []}) + group['cpu'].append(float(resource['CPUPerc'].rstrip('%'))) + group['mem'].append(mib(resource['MemUsage'].split('/')[0])) + app = [v for k, v in by_container.items() if '-api-' in k or '-worker-' in k] + telemetry = [v for k, v in by_container.items() if any('-' + service + '-' in k for service in ('collector', 'prometheus', 'loki', 'tempo', 'grafana'))] + def resources(items): + return round(sum(statistics.mean(v['cpu']) for v in items), 3), round(sum(max(v['mem']) for v in items), 3) + app_cpu, app_ram = resources(app) + telemetry_cpu, telemetry_ram = resources(telemetry) + sent = sum(v['sent'] for v in row['trace_retention'].values()) + found = sum(v['found'] for v in row['trace_retention'].values()) + return {'configuration': row['configuration'], 'repetition': row['repetition'], + 'rps': row['successful_rps'], 'unexpected_errors': row['unexpected_errors'], + **row['all_request_latency_ms'], 'app_cpu_percent_sum_mean': app_cpu, + 'app_ram_mib_sum_peak': app_ram, 'telemetry_cpu_percent_sum_mean': telemetry_cpu, + 'telemetry_ram_mib_sum_peak': telemetry_ram, + 'sent_requests': sent, 'retained_request_traces': found, + 'missing_request_traces': sent-found, + **{k: v for k, v in row['counter_delta'].items()}} + + +def main(): + report = json.loads((ROOT / 'evidence/lab6-research.json').read_text()) + rows = [summarize(row) for row in report['trials']] + if not rows: + raise RuntimeError('No completed trials') + with (ROOT / 'evidence/lab6-research.csv').open('w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print('| Config | Repeat | RPS | p50 ms | p95 ms | p99 ms | App CPU % | App MiB | Telemetry CPU % | Telemetry MiB | Request traces |') + print('| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |') + for row in rows: + print('| {configuration} | {repetition} | {rps} | {p50} | {p95} | {p99} | {app_cpu_percent_sum_mean} | {app_ram_mib_sum_peak} | {telemetry_cpu_percent_sum_mean} | {telemetry_ram_mib_sum_peak} | {retained_request_traces}/{sent_requests} |'.format(**row)) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_observability.py b/scripts/verify_observability.py new file mode 100644 index 0000000..94b80b1 --- /dev/null +++ b/scripts/verify_observability.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify live provisioned sources, dashboard queries and metric label policy.""" +import datetime +import json +import re +import urllib.parse + +from observability_common import get_json, prom, save + + +def main(): + targets = get_json('http://127.0.0.1:19090/api/v1/targets')['data']['activeTargets'] + assert targets and all(t['health'] == 'up' for t in targets), 'Prometheus target is down' + dashboard = get_json('http://127.0.0.1:13000/api/dashboards/uid/messenger')['dashboard'] + assert dashboard['uid'] == 'messenger' and len(dashboard['panels']) >= 10 + results = [] + for panel in dashboard['panels']: + if panel.get('datasource', {}).get('uid') != 'prometheus': + continue + for target in panel.get('targets', []): + query = target['expr'].replace('$__rate_interval', '1m') + rows = prom(query) + results.append({'panel': panel['title'], 'query': query, 'series': len(rows)}) + query = '{__name__=~"messenger_.*"}' + series = get_json('http://127.0.0.1:19090/api/v1/series?' + urllib.parse.urlencode({'match[]': query}))['data'] + prohibited = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_target'} + for row in series: + assert not prohibited.intersection(row), 'Forbidden metric label' + assert not re.search(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', row.get('route', '')), 'Raw UUID in route' + required = {'messenger_http_requests_total', 'messenger_http_duration_seconds_bucket', 'messenger_jobs_total', + 'messenger_job_duration_seconds_bucket', 'messenger_attachment_ready_seconds_bucket', + 'messenger_queue_depth', 'messenger_queue_oldest_age_seconds', 'messenger_db_pool_used', + 'messenger_db_pool_waiting', 'messenger_worker_capacity', 'messenger_worker_busy'} + names = {row['__name__'] for row in series} + assert required <= names, 'Required metrics absent: ' + str(sorted(required-names)) + report = {'at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'status': 'PASS', + 'targets': {t['labels']['job']: t['health'] for t in targets}, 'dashboard_uid': dashboard['uid'], + 'panel_count': len(dashboard['panels']), 'queries': results, + 'metric_series_checked': len(series), 'required_metrics': sorted(required), 'label_policy': 'PASS'} + save('evidence/lab6-provisioning.json', report) + print(json.dumps({'status': 'PASS', 'panels': report['panel_count'], 'queries': len(results), 'label_policy': 'PASS'})) + + +if __name__ == '__main__': + main() diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..f2496c1 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Wait for the local gateway without disclosing application configuration.""" +import argparse +import time +import urllib.request +p=argparse.ArgumentParser();p.add_argument('--base-url',default='http://localhost:18086');args=p.parse_args() +deadline=time.monotonic()+120 +while time.monotonic()