From dc7745064dfc18c4fcb3b49b4138ff6119c4ac01 Mon Sep 17 00:00:00 2001 From: gamayunov2004 Date: Fri, 20 Nov 2026 21:52:10 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=206:=20OpenTelemetry,=20Prometheus?= =?UTF-8?q?,=20Loki,=20Tempo=20=D0=B8=20Grafana=20=E2=80=94=20=D0=BD=D0=B0?= =?UTF-8?q?=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 30 +- .gitattributes | 5 + .gitignore | 2 + Dockerfile | 12 + REPORT.md | 121 ++++- compose.yaml | 236 +++++++++ evidence/grade3/README.md | 24 + evidence/grade3/baseline.txt | 3 + evidence/grade3/build-api.txt | 207 ++++++++ evidence/grade3/build-tools.txt | 223 +++++++++ evidence/grade3/compose-config.txt | 1 + evidence/grade3/compose-ps.txt | 15 + evidence/grade3/container-inspect.txt | 14 + evidence/grade3/course-sha256.txt | 19 + evidence/grade3/docker-stats.txt | 15 + evidence/grade3/environment.txt | 71 +++ evidence/grade3/failover.txt | 175 +++++++ evidence/grade3/git-status.txt | 26 + evidence/grade3/image-tags.txt | 14 + evidence/grade3/integrity.txt | 7 + evidence/grade3/lab5-regression.txt | 72 +++ evidence/grade3/lab5-transfer.txt | 32 ++ evidence/grade3/load.txt | 19 + .../observability-before-tempo-probe.txt | 47 ++ evidence/grade3/observability-initial.txt | 36 ++ evidence/grade3/observability.txt | 64 +++ evidence/grade3/offline.txt | 10 + evidence/grade3/persistence-after.txt | 12 + evidence/grade3/persistence-before.txt | 12 + evidence/grade3/persistence.txt | 5 + evidence/grade3/runtime-validation.txt | 10 + evidence/grade3/smoke.txt | 4 + evidence/grade3/startup.txt | 122 +++++ infra/Dockerfile.tools | 15 + infra/grafana/dashboards/lab6.json | 176 +++++++ .../grafana/provisioning/dashboards/lab6.yaml | 8 + .../provisioning/datasources/lab6.yaml | 30 ++ infra/haproxy.cfg | 29 ++ infra/otel/collector.yaml | 51 ++ infra/otel/loki.yaml | 39 ++ infra/otel/prometheus.yaml | 8 + infra/otel/tempo.yaml | 24 + infra/requirements-tools.lock | 20 + migrations/001_initial.sql | 36 ++ migrations/002_message_idempotency.sql | 9 + migrations/003_attachments.sql | 35 ++ migrations/004_attachment_jobs.sql | 32 ++ requirements.lock | 41 ++ requirements.txt | 12 + scripts/init-env.ps1 | 79 +++ scripts/lab-containers.sh | 18 + scripts/restart.sh | 20 + scripts/stop-one.sh | 9 + scripts/test-failover-both.ps1 | 39 ++ src/__init__.py | 1 + src/attachments.py | 165 +++++++ src/config.py | 16 + src/db.py | 40 ++ src/errors.py | 34 ++ src/jobs.py | 81 ++++ src/main.py | 247 ++++++++++ src/models.py | 171 +++++++ src/pagination.py | 74 +++ src/s3.py | 92 ++++ src/security.py | 47 ++ src/store.py | 174 +++++++ src/telemetry.py | 96 ++++ src/thumbnail.py | 25 + src/worker.py | 199 ++++++++ student-tests/README.md | 12 + student-tests/grade3.py | 454 ++++++++++++++++++ student-tests/observability.py | 405 ++++++++++++++++ 72 files changed, 4684 insertions(+), 44 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/grade3/README.md create mode 100644 evidence/grade3/baseline.txt create mode 100644 evidence/grade3/build-api.txt create mode 100644 evidence/grade3/build-tools.txt create mode 100644 evidence/grade3/compose-config.txt create mode 100644 evidence/grade3/compose-ps.txt create mode 100644 evidence/grade3/container-inspect.txt create mode 100644 evidence/grade3/course-sha256.txt create mode 100644 evidence/grade3/docker-stats.txt create mode 100644 evidence/grade3/environment.txt create mode 100644 evidence/grade3/failover.txt create mode 100644 evidence/grade3/git-status.txt create mode 100644 evidence/grade3/image-tags.txt create mode 100644 evidence/grade3/integrity.txt create mode 100644 evidence/grade3/lab5-regression.txt create mode 100644 evidence/grade3/lab5-transfer.txt create mode 100644 evidence/grade3/load.txt create mode 100644 evidence/grade3/observability-before-tempo-probe.txt create mode 100644 evidence/grade3/observability-initial.txt create mode 100644 evidence/grade3/observability.txt create mode 100644 evidence/grade3/offline.txt create mode 100644 evidence/grade3/persistence-after.txt create mode 100644 evidence/grade3/persistence-before.txt create mode 100644 evidence/grade3/persistence.txt create mode 100644 evidence/grade3/runtime-validation.txt create mode 100644 evidence/grade3/smoke.txt create mode 100644 evidence/grade3/startup.txt create mode 100644 infra/Dockerfile.tools create mode 100644 infra/grafana/dashboards/lab6.json create mode 100644 infra/grafana/provisioning/dashboards/lab6.yaml create mode 100644 infra/grafana/provisioning/datasources/lab6.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/loki.yaml create mode 100644 infra/otel/prometheus.yaml create mode 100644 infra/otel/tempo.yaml create mode 100644 infra/requirements-tools.lock create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_message_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_attachment_jobs.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100644 scripts/init-env.ps1 create mode 100755 scripts/lab-containers.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/test-failover-both.ps1 create mode 100644 src/__init__.py create mode 100644 src/attachments.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/jobs.py create mode 100644 src/main.py create mode 100644 src/models.py create mode 100644 src/pagination.py create mode 100644 src/s3.py create mode 100644 src/security.py create mode 100644 src/store.py create mode 100644 src/telemetry.py create mode 100644 src/thumbnail.py create mode 100644 src/worker.py create mode 100644 student-tests/README.md create mode 100644 student-tests/grade3.py create mode 100644 student-tests/observability.py diff --git a/.env.example b/.env.example index 2b32655..edc701a 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,8 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Local Compose overrides. Generate .env and private secret files with scripts/init-env.ps1. +LAB6_PORT=8086 +GRAFANA_PORT=3006 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Database, S3, RabbitMQ and Grafana credentials are read from ./secrets via Docker. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e16d172 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.sql text eol=lf +*.ps1 text eol=lf diff --git a/.gitignore b/.gitignore index b5a7189..72a60fe 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +/PR_GRADE3_DRAFT.md +/lab6-grade3-review.zip diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..52b847e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.10-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 +WORKDIR /app +COPY requirements.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock && useradd --create-home --uid 10001 appuser +COPY --chown=appuser:appuser src ./src +COPY --chown=appuser:appuser migrations ./migrations +USER appuser +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=5 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).read()" || exit 1 +CMD ["sh", "-c", "exec uvicorn src.main:app --host \"$APP_HOST\" --port \"$APP_PORT\" --no-access-log --timeout-graceful-shutdown 3"] diff --git a/REPORT.md b/REPORT.md index b97ae42..6ba8856 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,122 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор и группа — Гамаюнов Артём, БВТ2301. +- Целевая оценка — 3. Трек 5 — не заявлен. +- Ветка — `solution/bvt2301-gamayunov`. Исходники, отчёт и доказательства подготовлены для публикации в PR преподавательского репозитория. +- Основа — рабочая ЛР5, преподавательский skeleton ЛР6. Codex помог с переносом, реализацией OTLP, экспериментами Docker и отчётом. [Контракт наблюдаемости](contracts/observability.md) · [Collector](https://opentelemetry.io/docs/collector/configuration/) · [Prometheus exporter 0.127.0](https://raw.githubusercontent.com/open-telemetry/opentelemetry-collector-contrib/v0.127.0/exporter/prometheusexporter/README.md) · [OTLP в Loki](https://grafana.com/docs/loki/latest/send-data/otel/) · [Tempo 2.8](https://raw.githubusercontent.com/grafana/tempo/v2.8.0/example/docker-compose/local/tempo.yaml) ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Windows PowerShell, Docker Desktop в режиме Linux containers, Compose с поддержкой `--wait`, доступ к реестрам образов и зависимостей при сборке. Для запуска тестов на хосте Python и GNU Make не нужны. Команды ниже выполняются из корня рабочей копии. + +```powershell +docker version +docker compose version +docker info --format 'CPU={{.NCPU}} RAM_bytes={{.MemTotal}}' +docker system df +& ./scripts/init-env.ps1 +docker compose -p mtusi-lab6-grade3 config --quiet +docker compose -p mtusi-lab6-grade3 --profile test build migrate tools +docker compose -p mtusi-lab6-grade3 up -d --wait --wait-timeout 150 +docker compose -p mtusi-lab6-grade3 ps -a +curl.exe -fsS http://127.0.0.1:8086/health/ready +curl.exe -fsS http://127.0.0.1:3006/api/health +``` + +Bootstrap создаёт только отсутствующие файлы конфигурации, генерирует случайные локальные пароли, проверяет соответствие паролей и DSN и ограничивает доступ к каталогу `secrets` текущим Windows-пользователем и SYSTEM. `migrate` применяет последовательные SQL-миграции с учётом уже применённых версий, `bootstrap` готовит приватный S3 bucket. Оба контейнера завершаются с кодом 0. На чистом стенде для Grafana используется логин `admin`, пароль берётся локально из `secrets/grafana_admin_password`. Перейдите к [dashboard ЛР6 в Grafana](http://127.0.0.1:3006/d/lab6-messenger). В первые секунды после запуска имеет смысл дождаться первых событий и двух scrape, для данных worker загрузить изображение или выполнить сценарий ниже. + +```powershell +docker compose run --rm --no-deps tools make check validate +docker compose run --rm --no-deps tools make test BASE_URL=http://lb:8080 +docker compose run --rm --no-deps tools make test-persistence BASE_URL=http://lb:8080 ACTION_SCRIPT=scripts/restart.sh +& ./scripts/test-failover-both.ps1 +docker compose run --rm --no-deps tools python student-tests/grade3.py +docker compose run --rm --no-deps tools python student-tests/observability.py +docker compose run --rm --no-deps tools python tests/load.py --base-url http://lb:8080 --duration 8 --concurrency 2 +``` + +Для работы из Linux tools выполните `docker compose run --rm --no-deps tools sh` в подготовленном стенде. Затем запускайте `make check validate`, `make test BASE_URL=http://lb:8080`, `make test-persistence BASE_URL=http://lb:8080 ACTION_SCRIPT=scripts/restart.sh` и `make test-failover BASE_URL=http://lb:8080 ACTION_SCRIPT=scripts/stop-one.sh`. Восстановите остановленную реплику после каждого прогона failover с хоста. Скрипт PowerShell выше проверяет обе реплики и восстанавливает их автоматически. Профиль tools подключает Docker socket для сценариев отказа. Запускайте проверки отказов последовательно на собственном стенде с синтетическими данными. + +```powershell +# Штатная остановка с сохранением данных +docker compose -p mtusi-lab6-grade3 stop +# Явное удаление учебных данных только проекта ЛР6, при намеренном сбросе +docker compose -p mtusi-lab6-grade3 down --volumes +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP loopback → HAProxy → api1 / api2 → PostgreSQL + ├→ RustFS: staging → original / JPEG + └→ RabbitMQ → worker → PostgreSQL + RustFS + +api1 / api2 / worker → OTLP HTTP protobuf → OTel Collector + metrics → /metrics :8889 → Prometheus + logs → OTLP HTTP /otlp → Loki + traces → OTLP gRPC :4317 → Tempo + Grafana → запросы к Prometheus, Loki и Tempo +``` + +| Роль | Версия | Готовность | Данные и доступ | +| --- | --- | --- | --- | +| `lb` | HAProxy 3.0.11-alpine | проверяет `/health/ready` у обеих реплик | порт `127.0.0.1:8086`; API внутри сети | +| `api1`, `api2`, `worker` | Python 3.12.10, FastAPI 0.115.12, Pika 1.3.2, Pillow 12.2.0; OTel SDK и OTLP HTTP 1.34.1, semantic conventions 0.55b1 | API проверяет PostgreSQL и S3; worker после healthy RabbitMQ регистрирует consumer | порты только внутри сети; DSN, AMQP и S3 credentials в mounts secrets | +| `postgres` | 16.8-bookworm | `pg_isready` | `postgres_data`, секрет PostgreSQL | +| `s3` | RustFS 1.0.0-alpha.85 с digest | `/health` | `s3_data`, приватный bucket, ключи из файлов | +| `broker` | RabbitMQ 4.1.4-alpine | AMQP listener :5672 | `broker_data`, пароль из файла | +| `otel-collector` | contrib 0.127.0 | HTTP `:13133/` и проверка pipeline | `:4318` OTLP и `:8889` scrape только в сети Compose | +| `prometheus` | 3.4.1 | `/-/ready` | `prometheus_data`, до 2 дней, лимит retention 256 MB | +| `loki` | 3.5.1 | `/ready` | `loki_data`, TSDB v13, WAL, retention 48 ч | +| `tempo` | 2.8.0 | `/ready` | `tempo_data`, локальные блоки и WAL, retention 48 ч | +| `grafana` | 12.0.2 | `/api/health` | `grafana_data`, пароль администратора из файла, `127.0.0.1:3006` | +| `tools` | docker:27.5.1-cli, Python в Alpine | одноразовый запуск | профиль test, Docker socket и локальный пароль Grafana для диагностики | + +`.env.example` содержит порты и параметры, которые Compose передаёт приложению: TTL сессий, bucket, верхний предел попыток и OTLP endpoint. Имена service и instance, внутренние адреса и пути к файлам секретов заданы в Compose. `X-Instance-Id` — `api1` или `api2`. `X-Request-Id` генерируется сервером и сохраняется в trace и структурном логе ответа. + +PostgreSQL хранит пользователей, Argon2id-хэши, хэши Bearer-токенов, сообщения, ключи идемпотентности, состояния вложений и бюджет попыток. RustFS хранит байты. API возвращает `202 queued` после записи в S3, фиксации метаданных и подтверждения durable-карриера RabbitMQ. Worker удерживает advisory lock на job, фиксирует claim, сохраняет и проверяет вариант, атомарно фиксирует ready в БД, затем подтверждает сообщение. Retry сохраняет номер попытки до повторной публикации. Повторная доставка после ready/failed сохраняет итог. JPEG ограничен квадратом 256×256 с сохранением пропорций: квадратное изображение даёт 256×256, fixture 256×160. Миграции и обработчики ЛР5 сохранены, изменения приложения сосредоточены в инструментировании. + +## Сигналы и поиск + +| SDK | Prometheus | Единицы и измерения | +| --- | --- | --- | +| `messenger.http.requests` | `messenger_http_requests_total` | число ответов, `method, route, status_class` | +| `messenger.http.duration` | `messenger_http_duration_seconds_{bucket,sum,count}` | секунды до получения ответа от обработчика, те же измерения | +| `messenger.job.outcomes` | `messenger_job_outcomes_total` | терминальные исходы, `job_type, outcome` | +| `messenger.job.duration` | `messenger_job_duration_seconds_{bucket,sum,count}` | длительность финальной попытки без ожидания в очереди/backoff, `job_type, outcome` | + +Prometheus добавляет `job` и `instance`. `route` берётся из шаблона роутера, запросы за его пределами получают `unmatched`. API-панели строят RPS, 4xx/5xx раздельно и среднюю длительность из `sum/count` по окну в минуту, исключая штатные health endpoints. Worker показывает число завершённых jobs и среднюю длительность по `success` и `fail` за жизнь процесса. Фактические PromQL из Grafana и точки с метками сохранены в `evidence/grade3/observability.txt`. Временные retry видны как `job.outcome` в Loki и как spans с ошибкой в Tempo. Counters обновляются после commit терминального состояния. + +В Explore выберите Loki. Для ошибки с заголовком `X-Request-Id` используйте `{service_name="messenger-api"} | request_id="" | event="http.completed"`. Для задания используйте `{service_name="messenger-worker"} | attachment_id="" | event="job.outcome"`. В полях записи есть `trace_id` и `span_id`. Ссылка Trace открывает Tempo, тот же идентификатор работает с внутренним `GET /api/traces/`. Сопоставляйте оба ID и временную метку записи с интервалом соответствующего SERVER или CONSUMER span. + +В логах передаются серверный request ID, технические ID job и вложения и фиксированные статусы или причины. Приложение экспортирует только события выделенного logger, не записывает исключения автоматически и не передаёт HTTP-заголовки, тела, query или сырые URL. Resource SDK содержит `service.name`, `service.version`, `service.instance.id`, `deployment.environment.name=lab`. В Loki индексируется только имя сервиса. SDK использует фоновые выгрузки и двухсекундные сетевые таймауты, Collector ограничивает память и очереди. Готовность бизнес-API определяется при доступности PostgreSQL и RustFS. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Результат | Команда и доказательство | +| --- | --- | --- | +| Сборка, Compose, конфигурация Collector | PASS | `docker compose build --no-cache migrate`, `docker compose up -d --wait`, `docker compose config --quiet`, `/otelcol-contrib validate`. [startup.txt](evidence/grade3/startup.txt) · [runtime-validation.txt](evidence/grade3/runtime-validation.txt) | +| Офлайн-проверки и публичный smoke | PASS | `make check validate`, `make test BASE_URL=http://lb:8080`. [offline.txt](evidence/grade3/offline.txt) · [smoke.txt](evidence/grade3/smoke.txt) | +| Бизнес-состояние и обе реплики | PASS | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` и `./scripts/test-failover-both.ps1`. Прежняя сессия и объект пережили restart. В отдельных прогонах остановлены api1 и api2, подтверждено 114 и 125 записей соответственно, 0 ошибок отправки. [persistence.txt](evidence/grade3/persistence.txt) · [failover.txt](evidence/grade3/failover.txt) | +| Метрики и Grafana | PASS | `GET /api/v1/query` и Grafana `/api/ds/query`. На API content-шаблоне видны 2xx = 7 для api1 и 2 для api2, 5xx = 6 для api1. Для worker `success=1, fail=1`; histogram count/sum/buckets проверены. Пять PromQL-панелей вернули ненулевые точки, панель Loki вернула 286 логов. [observability.txt](evidence/grade3/observability.txt) | +| Связь Loki и Tempo | PASS | Loki `/loki/api/v1/query_range` и Tempo `/api/traces/`. Для API 201 `6da5fd7eed430a7e1312e9c747d67a61` / `12e525cf65db522c`, для worker ready `9e838857cbb81f9149e42b5d85f5aa00` / `28237be6723a2529`. Совпали trace ID, span ID, сервис, outcome и временной интервал. Оба span получены также через datasource proxy Grafana. [observability.txt](evidence/grade3/observability.txt) | +| Сохранность истории | PASS | Перезапущены Prometheus, Loki, Tempo, Grafana. Повтор исторического запроса дал тот же sample, Loki вернул те же записи, Tempo те же spans, Grafana тот же UID. Готовность всех повторных запросов через 7,291 с после завершения команды restart. [observability.txt](evidence/grade3/observability.txt) | +| Отсутствие утечек в выборке | PASS | Сканирование 286 Loki-записей, 493 Prometheus series и восьми ответов по trace. Совпадений по canary и секретам 0. [observability.txt](evidence/grade3/observability.txt) | +| Регрессия ЛР5 | PASS | `student-tests/grade3.py`. Среди проверок приватный S3, ACL, дедупликация, durable RabbitMQ, потеря и восстановление worker/broker, настоящий JPEG. [lab5-regression.txt](evidence/grade3/lab5-regression.txt) | +| CI форка | NOT TESTED | Локально выполнены Docker-сценарии. | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Остальные команды и привязки к журналам собраны в [индексе результатов](evidence/grade3/README.md). ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Эксперименты выполнялись в Docker Desktop, Linux amd64, 4 CPU, около 7,7 GiB памяти Docker VM. Использованы короткие синтетические запросы и локальные изображения. Harness отправил 48 авторизованных GET за 7,167 с, POST 201 и контролируемые 401/503. Генератор `tests/load.py` выполнил один closed-loop прогон POST без повторов и отдельного прогрева. При concurrency 2 за 8,148 с получено 783 успешных ответа и 0 ошибок, 96,095 успешных запросов/с. Задержки успешных ответов p50 / p95 / p99 — 14,284 / 46,570 / 87,944 мс. Закрытый цикл учитывает накладные расходы клиента и подвержен coordinated omission. Снимок `docker stats --no-stream` содержит CPU и RAM контейнеров. [load.txt](evidence/grade3/load.txt) · [docker-stats.txt](evidence/grade3/docker-stats.txt) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Воздействие | Что наблюдали | Восстановление | +| --- | --- | --- | +| Остановка worker | API принял PNG с 202, статус остался queued | Через 5,450 с от начала инцидента получен ready и проверен JPEG 256×256. | +| Остановка RustFS | Через внутреннюю API-реплику получен 503, trace `5a31c6bf418fe31369f700b887b07060` со `dependency.name=s3`. Worker сообщил `storage_unavailable`, после пятой попытки состояние failed с `retry_exhausted`. | Через 36,433 с от начала сценария подтверждены прежнее вложение и новый JPEG. | +| Остановка Collector и Grafana | Readiness и авторизованный GET — 200, новый upload — 202, вариант — ready | Collector и Grafana запущены обратно после проверки. | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд локальный, с единичными экземплярами PostgreSQL, RustFS, RabbitMQ, load balancer и компонентов наблюдаемости. При отказе RustFS HAProxy выводит из ротации обе реплики до восстановления readiness. Наблюдаемость показывает, что после исчерпания бюджета попыток вложение остаётся failed, а предыдущий объект и следующие загрузки доступны после восстановления. Отказоустойчивость реплик API проверена только при одиночном отказе. Очереди телеметрии в памяти, при долгом отказе pipeline или завершении процесса возможна потеря ещё не доставленных событий. История уже принятых данных хранится до указанного retention. Синтетическая нагрузка показывает работу сигналов в этом прогоне. Remote CI не запускался. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..1e6f14a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,236 @@ +name: mtusi-lab6-grade3 + +x-s3-env: &s3-env + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +x-app: &app + image: mtusi-lab6-api:grade3 + pull_policy: never + +x-api: &api + <<: *app + environment: &api-env + <<: *s3-env + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url] + depends_on: + migrate: {condition: service_completed_successfully} + bootstrap: {condition: service_completed_successfully} + restart: unless-stopped + stop_grace_period: 6s + +services: + postgres: + image: postgres:16.8-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + restart: unless-stopped + + s3: + image: rustfs/rustfs:1.0.0-alpha.85@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a + entrypoint: [sh, -ec, "unset RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY; exec /entrypoint.sh rustfs"] + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_access_key_id, s3_secret_access_key] + volumes: [s3_data:/data] + healthcheck: + test: [CMD, curl, -fsS, --max-time, "2", "http://127.0.0.1:9000/health"] + interval: 2s + timeout: 3s + retries: 20 + restart: unless-stopped + + migrate: + <<: *app + build: . + command: [python, -m, src.db] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + healthcheck: {disable: true} + depends_on: + postgres: {condition: service_healthy} + + bootstrap: + <<: *app + command: [python, -m, src.s3] + environment: *s3-env + secrets: [s3_access_key_id, s3_secret_access_key] + healthcheck: {disable: true} + depends_on: + s3: {condition: service_healthy} + + broker: + image: rabbitmq:4.1.4-alpine + hostname: broker + entrypoint: [sh, -ec, 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server'] + environment: + RABBITMQ_DEFAULT_USER: messenger + secrets: [broker_password] + volumes: [broker_data:/var/lib/rabbitmq] + healthcheck: + test: [CMD, nc, -z, -w, "2", "127.0.0.1", "5672"] + interval: 3s + timeout: 3s + start_period: 60s + retries: 15 + restart: unless-stopped + + worker: + <<: *api + command: [python, -m, src.worker] + environment: + <<: *api-env + OTEL_SERVICE_NAME: messenger-worker + INSTANCE_ID: worker + healthcheck: {disable: true} + depends_on: + migrate: {condition: service_completed_successfully} + bootstrap: {condition: service_completed_successfully} + broker: {condition: service_healthy} + + api1: + <<: *api + environment: + <<: *api-env + INSTANCE_ID: api1 + + api2: + <<: *api + environment: + <<: *api-env + INSTANCE_ID: api2 + + lb: + image: haproxy:3.0.11-alpine + ports: ["127.0.0.1:${LAB6_PORT:-8086}:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 3s + retries: 30 + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.127.0 + command: [--config=/etc/otelcol-contrib/config.yaml] + volumes: [./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro] + environment: {GOMEMLIMIT: 144MiB} + mem_limit: 192m + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.4.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=2d + - --storage.tsdb.retention.size=256MB + volumes: + - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 15 + mem_limit: 384m + restart: unless-stopped + + loki: + image: grafana/loki:3.5.1 + command: [-config.file=/etc/loki/local-config.yaml] + volumes: + - ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + environment: {GOMEMLIMIT: 320MiB, GOMAXPROCS: "2"} + mem_limit: 448m + restart: unless-stopped + + tempo: + image: grafana/tempo:2.8.0 + command: [-config.file=/etc/tempo.yaml] + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + environment: {GOMEMLIMIT: 320MiB, GOMAXPROCS: "2"} + mem_limit: 448m + restart: unless-stopped + + grafana: + image: grafana/grafana:12.0.2 + ports: ["127.0.0.1:${GRAFANA_PORT:-3006}:3000"] + environment: + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + secrets: [grafana_admin_password] + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 15 + mem_limit: 384m + restart: unless-stopped + + tools: + profiles: [test] + secrets: [grafana_admin_password] + image: mtusi-lab6-tools:grade3 + build: {context: ., dockerfile: infra/Dockerfile.tools} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + BASE_URL: http://lb:8080 + LAB6_PROJECT: ${COMPOSE_PROJECT_NAME:-mtusi-lab6-grade3} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + +secrets: + grafana_admin_password: {file: ./secrets/grafana_admin_password} + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + broker_password: {file: ./secrets/broker_password} + broker_url: {file: ./secrets/broker_url} + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + postgres_data: + s3_data: + broker_data: diff --git a/evidence/grade3/README.md b/evidence/grade3/README.md new file mode 100644 index 0000000..c6459dd --- /dev/null +++ b/evidence/grade3/README.md @@ -0,0 +1,24 @@ +# Доказательства уровня 3 + +Команды выполнялись на Windows в отдельном проекте `mtusi-lab6-grade3`. `tools` использует тот же HAProxy на внутреннем адресе `http://lb:8080`. Проверка опубликованных портов Windows входит в `environment.txt`. + +| Критерий | Команда или сценарий | Что проверяется | Доказательство | +| --- | --- | --- | --- | +| Перенос и сохранность skeleton | сравнение SHA-256 при копировании ЛР5 и перед архивацией ЛР6 | исходный index пуст, преподавательские контракты и тесты совпадают с исходными | [baseline.txt](baseline.txt), [lab5-transfer.txt](lab5-transfer.txt), [course-sha256.txt](course-sha256.txt), [integrity.txt](integrity.txt) | +| Сборка и запуск | `docker compose config --quiet`, `docker compose build --no-cache migrate`, `docker compose up -d --wait` | новый Compose-проект, семь томов, отдельные образы, health/readiness, только два опубликованных loopback-порта | [compose-config.txt](compose-config.txt), [build-api.txt](build-api.txt), [build-tools.txt](build-tools.txt), [startup.txt](startup.txt), [image-tags.txt](image-tags.txt), [compose-ps.txt](compose-ps.txt), [container-inspect.txt](container-inspect.txt), [environment.txt](environment.txt) | +| Валидация | `docker compose run --rm --no-deps tools make check validate` и `/otelcol-contrib validate` | офлайн-проверки контракта и конфигурации Collector, зависимости и hooks | [offline.txt](offline.txt), [runtime-validation.txt](runtime-validation.txt) | +| API ЛР6 | `docker compose run --rm --no-deps tools make test BASE_URL=http://lb:8080` | сквозной бизнес-API с активной телеметрией | [smoke.txt](smoke.txt) | +| Сохранность данных | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` внутри tools | прежний токен, запись, оригинал; прежние ID контейнеров с новым StartedAt | [persistence.txt](persistence.txt), [persistence-before.txt](persistence-before.txt), [persistence-after.txt](persistence-after.txt) | +| Отказы реплик | `./scripts/test-failover-both.ps1` | публичный тест останавливает каждую API-реплику по очереди и подтверждает сохранность и доступность записей | [failover.txt](failover.txt) | +| Наследованный сценарий ЛР5 | `docker compose run --rm --no-deps tools python student-tests/grade3.py` | приватность S3, queued/processing/ready/failed, ACL, изображения, дубль, retry, очередь и failover | [lab5-regression.txt](lab5-regression.txt) | +| Prometheus | `GET /api/v1/query` для восьми серий приложения | реальные положительные samples счётчиков и histogram sum/count/buckets, шаблоны маршрутов, ограниченные labels, success и fail | [observability.txt](observability.txt), строки `prometheus` | +| Loki и Tempo | `GET /loki/api/v1/query_range`, `GET /api/traces/` | конкретный SERVER/CONSUMER span, одинаковые trace/span ID, временная метка лога внутри span, ожидаемый outcome | [observability.txt](observability.txt), строки `api.*` и `worker.*` | +| Grafana | `/api/datasources`, health/прокси, dashboard UID, `/api/ds/query` | три источника, пять PromQL-панелей и панель Loki читают данные; прокси Tempo нашёл оба контрольных span | [observability.txt](observability.txt), строки `grafana.*` | +| Инцидент worker | остановка worker, upload 202, запуск и скачивание JPEG | queued во время остановки, тот же ID ready со связанными trace/log | [observability.txt](observability.txt), `worker.stop`, `worker.recovered`, `worker.ready` | +| Инцидент RustFS | отказ S3 при queued job, чтение через API, восстановление S3 | API 503 с причиной s3, retry до failed, сохранность прежнего вложения и успешная новая загрузка | [observability.txt](observability.txt), `s3.*`, `api.503`, `worker.retry`, `worker.failed` | +| История после restart | restart Prometheus, Loki, Tempo, Grafana с томами | те же историческая точка, Loki-события, Tempo spans и UID Grafana при новом StartedAt | [observability.txt](observability.txt), `storage.restart` | +| Отказ телеметрии | stop Collector и Grafana во время запросов | readiness 200, авторизованный GET, upload 202 и JPEG ready | [observability.txt](observability.txt), `telemetry.outage` | +| Проверка утечек | поиск canary, тестовых паролей, токенов и известных форматов в ответах и журнале | найдено 0 совпадений в проверенных данных | [observability.txt](observability.txt), `telemetry.scan`, [integrity.txt](integrity.txt) | +| Синтетическая нагрузка | `tests/load.py --duration 8 --concurrency 2` | сводка успешных RPS, HTTP-ошибок и задержек успешных запросов | [load.txt](load.txt), [docker-stats.txt](docker-stats.txt) | + +В [observability-initial.txt](observability-initial.txt) первая проверка buckets сравнивала текст `60` с нормализованной меткой Prometheus `60.0`. Исправлено числовым сравнением. В [observability-before-tempo-probe.txt](observability-before-tempo-probe.txt) встроенный plugin Tempo вернул 404 `plugin.notImplemented` для health. Итоговый сценарий проверяет readiness через прокси Grafana и запрашивает через него два живых trace. Оба исправления касались способа измерения. diff --git a/evidence/grade3/baseline.txt b/evidence/grade3/baseline.txt new file mode 100644 index 0000000..24d19cb --- /dev/null +++ b/evidence/grade3/baseline.txt @@ -0,0 +1,3 @@ +## solution/bvt2301-gamayunov +470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1 +Staged paths: diff --git a/evidence/grade3/build-api.txt b/evidence/grade3/build-api.txt new file mode 100644 index 0000000..b83928d --- /dev/null +++ b/evidence/grade3/build-api.txt @@ -0,0 +1,207 @@ + Image mtusi-lab6-api:grade3 Building +#1 [internal] load local bake definitions +#1 reading from stdin 537B 0.1s done +#1 DONE 0.1s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 773B 0.0s done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#3 DONE 0.1s + +#4 [internal] load .dockerignore +#4 transferring context: 216B 0.0s done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 61.38kB 0.0s done +#5 DONE 0.1s + +#6 [1/6] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#6 resolve docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db 0.1s done +#6 DONE 0.2s + +#6 [1/6] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#6 extracting sha256:61320b01ae5e0798393ef25f2dc72faf43703e60ba089b07d7170acbabbf8f62 +#6 extracting sha256:61320b01ae5e0798393ef25f2dc72faf43703e60ba089b07d7170acbabbf8f62 3.3s done +#6 DONE 3.5s + +#6 [1/6] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#6 extracting sha256:be1274d3cce09d8126f8176203ba1e91f6733a713c1185ba01ee704759b0b62d +#6 extracting sha256:be1274d3cce09d8126f8176203ba1e91f6733a713c1185ba01ee704759b0b62d 0.5s done +#6 DONE 4.0s + +#6 [1/6] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#6 extracting sha256:8991c9200d621283b06ae395129d454c998e0f55095e106055240b5956ab2a2a +#6 extracting sha256:8991c9200d621283b06ae395129d454c998e0f55095e106055240b5956ab2a2a 1.4s done +#6 DONE 5.3s + +#6 [1/6] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#6 extracting sha256:7a1cb8b882211b870d7e639e775f91e423a526092b4b3eadefbab6aa7457910d 0.0s done +#6 DONE 5.4s + +#7 [2/6] WORKDIR /app +#7 DONE 0.5s + +#8 [3/6] COPY requirements.txt requirements.lock ./ +#8 DONE 0.3s + +#9 [4/6] RUN pip install --no-cache-dir -r requirements.lock && useradd --create-home --uid 10001 appuser +#9 5.951 Collecting annotated-types==0.8.0 (from -r requirements.lock (line 1)) +#9 6.295 Downloading annotated_types-0.8.0-py3-none-any.whl.metadata (15 kB) +#9 6.484 Collecting anyio==4.15.1 (from -r requirements.lock (line 2)) +#9 6.546 Downloading anyio-4.15.1-py3-none-any.whl.metadata (4.7 kB) +#9 6.667 Collecting argon2-cffi==23.1.0 (from -r requirements.lock (line 3)) +#9 6.729 Downloading argon2_cffi-23.1.0-py3-none-any.whl.metadata (5.2 kB) +#9 6.823 Collecting argon2-cffi-bindings==26.1.0 (from -r requirements.lock (line 4)) +#9 6.879 Downloading argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl.metadata (7.5 kB) +#9 7.798 Collecting cffi==2.1.1 (from -r requirements.lock (line 5)) +#9 7.873 Downloading cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.5 kB) +#9 7.977 Collecting click==8.5.0 (from -r requirements.lock (line 6)) +#9 8.034 Downloading click-8.5.0-py3-none-any.whl.metadata (2.6 kB) +#9 8.298 Collecting fastapi==0.115.12 (from -r requirements.lock (line 7)) +#9 8.372 Downloading fastapi-0.115.12-py3-none-any.whl.metadata (27 kB) +#9 8.493 Collecting h11==0.16.0 (from -r requirements.lock (line 8)) +#9 8.548 Downloading h11-0.16.0-py3-none-any.whl.metadata (8.3 kB) +#9 8.700 Collecting idna==3.20 (from -r requirements.lock (line 9)) +#9 8.815 Downloading idna-3.20-py3-none-any.whl.metadata (7.2 kB) +#9 8.947 Collecting psycopg==3.2.6 (from -r requirements.lock (line 10)) +#9 9.005 Downloading psycopg-3.2.6-py3-none-any.whl.metadata (4.4 kB) +#9 9.697 Collecting psycopg-binary==3.2.6 (from -r requirements.lock (line 11)) +#9 9.754 Downloading psycopg_binary-3.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.8 kB) +#9 9.827 Collecting psycopg-pool==3.2.6 (from -r requirements.lock (line 12)) +#9 9.884 Downloading psycopg_pool-3.2.6-py3-none-any.whl.metadata (2.6 kB) +#9 9.956 Collecting pycparser==3.12 (from -r requirements.lock (line 13)) +#9 10.02 Downloading pycparser-3.12-py3-none-any.whl.metadata (9.7 kB) +#9 10.50 Collecting pydantic==2.11.3 (from -r requirements.lock (line 14)) +#9 10.56 Downloading pydantic-2.11.3-py3-none-any.whl.metadata (65 kB) +#9 13.24 Collecting pydantic_core==2.33.1 (from -r requirements.lock (line 15)) +#9 13.30 Downloading pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.8 kB) +#9 13.51 Collecting starlette==0.46.2 (from -r requirements.lock (line 16)) +#9 13.56 Downloading starlette-0.46.2-py3-none-any.whl.metadata (6.2 kB) +#9 13.63 Collecting typing-inspection==0.4.4 (from -r requirements.lock (line 17)) +#9 14.13 Downloading typing_inspection-0.4.4-py3-none-any.whl.metadata (2.6 kB) +#9 14.25 Collecting typing_extensions==4.16.0 (from -r requirements.lock (line 18)) +#9 14.31 Downloading typing_extensions-4.16.0-py3-none-any.whl.metadata (3.3 kB) +#9 14.44 Collecting uvicorn==0.34.2 (from -r requirements.lock (line 19)) +#9 14.49 Downloading uvicorn-0.34.2-py3-none-any.whl.metadata (6.5 kB) +#9 15.50 Collecting boto3==1.40.45 (from -r requirements.lock (line 20)) +#9 15.56 Downloading boto3-1.40.45-py3-none-any.whl.metadata (6.7 kB) +#9 16.78 Collecting botocore==1.40.45 (from -r requirements.lock (line 21)) +#9 16.84 Downloading botocore-1.40.45-py3-none-any.whl.metadata (5.7 kB) +#9 17.12 Collecting jmespath==1.0.1 (from -r requirements.lock (line 22)) +#9 17.18 Downloading jmespath-1.0.1-py3-none-any.whl.metadata (7.6 kB) +#9 17.87 Collecting Pillow==12.2.0 (from -r requirements.lock (line 23)) +#9 17.93 Downloading pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (8.8 kB) +#9 18.01 Collecting python-dateutil==2.9.0.post0 (from -r requirements.lock (line 24)) +#9 18.06 Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB) +#9 18.14 Collecting s3transfer==0.14.0 (from -r requirements.lock (line 25)) +#9 18.20 Downloading s3transfer-0.14.0-py3-none-any.whl.metadata (1.7 kB) +#9 18.28 Collecting six==1.17.0 (from -r requirements.lock (line 26)) +#9 18.34 Downloading six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) +#9 18.44 Collecting urllib3==2.6.3 (from -r requirements.lock (line 27)) +#9 18.50 Downloading urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB) +#9 18.58 Collecting pika==1.3.2 (from -r requirements.lock (line 28)) +#9 18.64 Downloading pika-1.3.2-py3-none-any.whl.metadata (13 kB) +#9 18.76 Collecting opentelemetry-api==1.34.1 (from -r requirements.lock (line 29)) +#9 18.82 Downloading opentelemetry_api-1.34.1-py3-none-any.whl.metadata (1.5 kB) +#9 18.94 Collecting opentelemetry-sdk==1.34.1 (from -r requirements.lock (line 30)) +#9 19.00 Downloading opentelemetry_sdk-1.34.1-py3-none-any.whl.metadata (1.6 kB) +#9 19.09 Collecting opentelemetry-semantic-conventions==0.55b1 (from -r requirements.lock (line 31)) +#9 19.15 Downloading opentelemetry_semantic_conventions-0.55b1-py3-none-any.whl.metadata (2.5 kB) +#9 19.23 Collecting opentelemetry-exporter-otlp-proto-http==1.34.1 (from -r requirements.lock (line 32)) +#9 19.29 Downloading opentelemetry_exporter_otlp_proto_http-1.34.1-py3-none-any.whl.metadata (2.3 kB) +#9 19.36 Collecting opentelemetry-exporter-otlp-proto-common==1.34.1 (from -r requirements.lock (line 33)) +#9 19.42 Downloading opentelemetry_exporter_otlp_proto_common-1.34.1-py3-none-any.whl.metadata (1.9 kB) +#9 19.50 Collecting opentelemetry-proto==1.34.1 (from -r requirements.lock (line 34)) +#9 19.56 Downloading opentelemetry_proto-1.34.1-py3-none-any.whl.metadata (2.4 kB) +#9 20.07 Collecting protobuf==5.29.6 (from -r requirements.lock (line 35)) +#9 20.12 Downloading protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) +#9 20.21 Collecting googleapis-common-protos==1.75.0 (from -r requirements.lock (line 36)) +#9 20.27 Downloading googleapis_common_protos-1.75.0-py3-none-any.whl.metadata (8.6 kB) +#9 20.67 Collecting importlib-metadata==8.7.1 (from -r requirements.lock (line 37)) +#9 20.73 Downloading importlib_metadata-8.7.1-py3-none-any.whl.metadata (4.7 kB) +#9 20.85 Collecting zipp==4.1.2 (from -r requirements.lock (line 38)) +#9 20.91 Downloading zipp-4.1.2-py3-none-any.whl.metadata (3.6 kB) +#9 21.45 Collecting requests==2.34.2 (from -r requirements.lock (line 39)) +#9 21.50 Downloading requests-2.34.2-py3-none-any.whl.metadata (4.8 kB) +#9 21.62 Collecting certifi==2026.10.23 (from -r requirements.lock (line 40)) +#9 21.68 Downloading certifi-2026.10.23-py3-none-any.whl.metadata (2.5 kB) +#9 22.37 Collecting charset-normalizer==3.5.2 (from -r requirements.lock (line 41)) +#9 22.44 Downloading charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (46 kB) +#9 24.18 Downloading annotated_types-0.8.0-py3-none-any.whl (13 kB) +#9 24.28 Downloading anyio-4.15.1-py3-none-any.whl (132 kB) +#9 24.46 Downloading argon2_cffi-23.1.0-py3-none-any.whl (15 kB) +#9 24.53 Downloading argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl (26 kB) +#9 24.61 Downloading cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (221 kB) +#9 25.32 Downloading click-8.5.0-py3-none-any.whl (125 kB) +#9 25.55 Downloading fastapi-0.115.12-py3-none-any.whl (95 kB) +#9 25.90 Downloading h11-0.16.0-py3-none-any.whl (37 kB) +#9 25.97 Downloading idna-3.20-py3-none-any.whl (69 kB) +#9 26.09 Downloading psycopg-3.2.6-py3-none-any.whl (199 kB) +#9 26.22 Downloading psycopg_binary-3.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.3 MB) +#9 28.50 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.3/4.3 MB 1.9 MB/s eta 0:00:00 +#9 28.56 Downloading psycopg_pool-3.2.6-py3-none-any.whl (38 kB) +#9 28.65 Downloading pycparser-3.12-py3-none-any.whl (62 kB) +#9 28.73 Downloading pydantic-2.11.3-py3-none-any.whl (443 kB) +#9 29.06 Downloading pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB) +#9 30.79 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 1.2 MB/s eta 0:00:00 +#9 30.85 Downloading starlette-0.46.2-py3-none-any.whl (72 kB) +#9 30.96 Downloading typing_inspection-0.4.4-py3-none-any.whl (14 kB) +#9 31.02 Downloading typing_extensions-4.16.0-py3-none-any.whl (45 kB) +#9 31.08 Downloading uvicorn-0.34.2-py3-none-any.whl (62 kB) +#9 31.19 Downloading boto3-1.40.45-py3-none-any.whl (139 kB) +#9 31.35 Downloading botocore-1.40.45-py3-none-any.whl (14.1 MB) +#9 38.31 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 14.1/14.1 MB 2.0 MB/s eta 0:00:00 +#9 38.37 Downloading jmespath-1.0.1-py3-none-any.whl (20 kB) +#9 38.42 Downloading pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (7.1 MB) +#9 41.76 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.1/7.1 MB 2.1 MB/s eta 0:00:00 +#9 41.81 Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +#9 42.01 Downloading s3transfer-0.14.0-py3-none-any.whl (85 kB) +#9 42.10 Downloading six-1.17.0-py2.py3-none-any.whl (11 kB) +#9 42.16 Downloading urllib3-2.6.3-py3-none-any.whl (131 kB) +#9 42.26 Downloading pika-1.3.2-py3-none-any.whl (155 kB) +#9 42.37 Downloading opentelemetry_api-1.34.1-py3-none-any.whl (65 kB) +#9 42.44 Downloading opentelemetry_sdk-1.34.1-py3-none-any.whl (118 kB) +#9 42.59 Downloading opentelemetry_semantic_conventions-0.55b1-py3-none-any.whl (196 kB) +#9 42.77 Downloading opentelemetry_exporter_otlp_proto_http-1.34.1-py3-none-any.whl (17 kB) +#9 42.82 Downloading opentelemetry_exporter_otlp_proto_common-1.34.1-py3-none-any.whl (18 kB) +#9 42.88 Downloading opentelemetry_proto-1.34.1-py3-none-any.whl (55 kB) +#9 42.95 Downloading protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl (320 kB) +#9 43.22 Downloading googleapis_common_protos-1.75.0-py3-none-any.whl (300 kB) +#9 43.45 Downloading importlib_metadata-8.7.1-py3-none-any.whl (27 kB) +#9 43.51 Downloading zipp-4.1.2-py3-none-any.whl (10 kB) +#9 43.56 Downloading requests-2.34.2-py3-none-any.whl (73 kB) +#9 43.66 Downloading certifi-2026.10.23-py3-none-any.whl (135 kB) +#9 43.77 Downloading charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (257 kB) +#9 44.19 Installing collected packages: zipp, urllib3, typing_extensions, six, pycparser, psycopg-binary, protobuf, Pillow, pika, jmespath, idna, h11, click, charset-normalizer, certifi, annotated-types, uvicorn, typing-inspection, requests, python-dateutil, pydantic_core, psycopg-pool, psycopg, opentelemetry-proto, importlib-metadata, googleapis-common-protos, cffi, anyio, starlette, pydantic, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, botocore, argon2-cffi-bindings, s3transfer, opentelemetry-semantic-conventions, fastapi, argon2-cffi, opentelemetry-sdk, boto3, opentelemetry-exporter-otlp-proto-http +#9 49.64 Successfully installed Pillow-12.2.0 annotated-types-0.8.0 anyio-4.15.1 argon2-cffi-23.1.0 argon2-cffi-bindings-26.1.0 boto3-1.40.45 botocore-1.40.45 certifi-2026.10.23 cffi-2.1.1 charset-normalizer-3.5.2 click-8.5.0 fastapi-0.115.12 googleapis-common-protos-1.75.0 h11-0.16.0 idna-3.20 importlib-metadata-8.7.1 jmespath-1.0.1 opentelemetry-api-1.34.1 opentelemetry-exporter-otlp-proto-common-1.34.1 opentelemetry-exporter-otlp-proto-http-1.34.1 opentelemetry-proto-1.34.1 opentelemetry-sdk-1.34.1 opentelemetry-semantic-conventions-0.55b1 pika-1.3.2 protobuf-5.29.6 psycopg-3.2.6 psycopg-binary-3.2.6 psycopg-pool-3.2.6 pycparser-3.12 pydantic-2.11.3 pydantic_core-2.33.1 python-dateutil-2.9.0.post0 requests-2.34.2 s3transfer-0.14.0 six-1.17.0 starlette-0.46.2 typing-inspection-0.4.4 typing_extensions-4.16.0 urllib3-2.6.3 uvicorn-0.34.2 zipp-4.1.2 +#9 49.64 WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. +#9 50.10 +#9 50.10 [notice] A new release of pip is available: 25.0.1 -> 26.3 +#9 50.10 [notice] To update, run: pip install --upgrade pip +#9 DONE 50.8s + +#10 [5/6] COPY --chown=appuser:appuser src ./src +#10 DONE 0.2s + +#11 [6/6] COPY --chown=appuser:appuser migrations ./migrations +#11 DONE 0.1s + +#12 exporting to image +#12 exporting layers +#12 exporting layers 5.2s done +#12 exporting manifest sha256:a785cb79caaef0fef7cca9b0146d7da32e0cf9ae6d8f1e36daea31e8ab3a7371 0.0s done +#12 exporting config sha256:f246aa0b7a8b0d2bacf1b9a4e87dea59eebf5b12f954a3c78d91fcf4279c8141 0.0s done +#12 exporting attestation manifest sha256:ed2877f8f3dc8c0a02043941f9c7fb229d60a1d1bc1cf0306e17638c5b3b4866 0.0s done +#12 exporting manifest list sha256:190c5a8e35b3f6fcc7aea5b52c094c6555a4f7612d333734a0b14d9c2bf36e11 +#12 exporting manifest list sha256:190c5a8e35b3f6fcc7aea5b52c094c6555a4f7612d333734a0b14d9c2bf36e11 0.0s done +#12 naming to docker.io/library/mtusi-lab6-api:grade3 done +#12 unpacking to docker.io/library/mtusi-lab6-api:grade3 +#12 unpacking to docker.io/library/mtusi-lab6-api:grade3 3.1s done +#12 DONE 8.6s + +#13 resolving provenance for metadata file +#13 DONE 0.1s + Image mtusi-lab6-api:grade3 Built diff --git a/evidence/grade3/build-tools.txt b/evidence/grade3/build-tools.txt new file mode 100644 index 0000000..410b375 --- /dev/null +++ b/evidence/grade3/build-tools.txt @@ -0,0 +1,223 @@ + Image mtusi-lab6-tools:grade3 Building +#1 [internal] load local bake definitions +#1 reading from stdin 522B 0.1s done +#1 DONE 0.1s + +#2 [internal] load build definition from Dockerfile.tools +#2 transferring dockerfile: 753B 0.0s done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/docker:27.5.1-cli +#3 DONE 0.1s + +#4 [internal] load .dockerignore +#4 transferring context: 216B 0.0s done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 391.26kB 0.2s done +#5 DONE 0.3s + +#6 [ 1/12] FROM docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c +#6 resolve docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c 0.2s done +#6 DONE 0.2s + +#7 [ 7/12] COPY contracts/ ./contracts/ +#7 CACHED + +#8 [ 8/12] COPY tests/ ./tests/ +#8 CACHED + +#9 [ 9/12] COPY scripts/check_contract.py scripts/validate_openapi.py ./scripts/ +#9 CACHED + +#10 [ 5/12] RUN pip install --no-cache-dir -r /tmp/requirements-tools.lock +#10 CACHED + +#11 [ 4/12] COPY infra/requirements-tools.lock /tmp/requirements-tools.lock +#11 CACHED + +#12 [ 3/12] WORKDIR /workspace +#12 CACHED + +#13 [10/12] COPY --chmod=0755 scripts/restart.sh scripts/stop-one.sh scripts/lab-containers.sh ./scripts/ +#13 CACHED + +#14 [ 2/12] RUN apk add --no-cache python3 py3-pip make && python3 -m venv /opt/venv +#14 CACHED + +#15 [ 6/12] COPY Makefile lab.json README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +#15 CACHED + +#16 [11/12] RUN mkdir -p /workspace/evidence +#16 CACHED + +#17 [12/12] COPY student-tests/ ./student-tests/ +#17 DONE 0.2s + +#18 exporting to image +#18 exporting layers +#18 exporting layers 1.3s done +#18 exporting manifest sha256:e45ce7fc9dac99e15e9e2a19d8de34430de0269d3cf893a57130b544c2482c18 +#18 exporting manifest sha256:e45ce7fc9dac99e15e9e2a19d8de34430de0269d3cf893a57130b544c2482c18 0.2s done +#18 exporting config sha256:d29d864544c087fa6e6a9eff946b957aae1dbeebd6054468ee9f319e9228a089 +#18 exporting config sha256:d29d864544c087fa6e6a9eff946b957aae1dbeebd6054468ee9f319e9228a089 0.1s done +#18 exporting attestation manifest sha256:5067a5564160de4348125a26bbaa29a874fb9c7eb63dd9855dd834a951924197 0.1s done +#18 exporting manifest list sha256:68280311adeb3e1b4719ad243e5766c4e81497671d3eb098244fef74b039ba5a +#18 exporting manifest list sha256:68280311adeb3e1b4719ad243e5766c4e81497671d3eb098244fef74b039ba5a 0.0s done +#18 naming to docker.io/library/mtusi-lab6-tools:grade3 0.0s done +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 0.2s done +#18 DONE 2.3s + +#19 resolving provenance for metadata file +#19 DONE 0.1s + Image mtusi-lab6-tools:grade3 Built + Image mtusi-lab6-tools:grade3 Building +#1 [internal] load local bake definitions +#1 reading from stdin 522B 0.1s done +#1 DONE 0.1s + +#2 [internal] load build definition from Dockerfile.tools +#2 transferring dockerfile: 753B 0.0s done +#2 DONE 0.1s + +#3 [internal] load metadata for docker.io/library/docker:27.5.1-cli +#3 DONE 0.2s + +#4 [internal] load .dockerignore +#4 transferring context: 216B 0.0s done +#4 DONE 0.1s + +#5 [internal] load build context +#5 transferring context: 56.32kB 0.1s done +#5 DONE 0.1s + +#6 [ 1/12] FROM docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c +#6 resolve docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c +#6 resolve docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c 0.2s done +#6 DONE 0.2s + +#7 [ 2/12] RUN apk add --no-cache python3 py3-pip make && python3 -m venv /opt/venv +#7 CACHED + +#8 [ 3/12] WORKDIR /workspace +#8 CACHED + +#9 [ 4/12] COPY infra/requirements-tools.lock /tmp/requirements-tools.lock +#9 CACHED + +#10 [ 5/12] RUN pip install --no-cache-dir -r /tmp/requirements-tools.lock +#10 CACHED + +#11 [ 6/12] COPY Makefile lab.json README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +#11 DONE 0.2s + +#12 [ 7/12] COPY contracts/ ./contracts/ +#12 DONE 0.1s + +#13 [ 8/12] COPY tests/ ./tests/ +#13 DONE 0.1s + +#14 [ 9/12] COPY scripts/check_contract.py scripts/validate_openapi.py ./scripts/ +#14 DONE 0.1s + +#15 [10/12] COPY --chmod=0755 scripts/restart.sh scripts/stop-one.sh scripts/lab-containers.sh ./scripts/ +#15 DONE 0.2s + +#16 [11/12] RUN mkdir -p /workspace/evidence +#16 DONE 0.4s + +#17 [12/12] COPY student-tests/ ./student-tests/ +#17 DONE 0.1s + +#18 exporting to image +#18 exporting layers +#18 exporting layers 0.6s done +#18 exporting manifest sha256:f91ffb90ab64cf8b57de173d51e7d7e158cce0ee6974c84b342d6a63e87e823d 0.0s done +#18 exporting config sha256:55f61aedb3fea20ff2c1a6ffa9296812fdcf315d828de4ca5221d7708f965131 0.0s done +#18 exporting attestation manifest sha256:a68a361964800e267e31ac1fde3933345ac7a90ec37b944a0435dcbf14fc713d 0.0s done +#18 exporting manifest list sha256:5bbdcbd60f6d5bcbb8fe842b37451066fb33d48d04b25b1c7dacecb7611156d2 +#18 exporting manifest list sha256:5bbdcbd60f6d5bcbb8fe842b37451066fb33d48d04b25b1c7dacecb7611156d2 0.0s done +#18 naming to docker.io/library/mtusi-lab6-tools:grade3 done +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 0.2s done +#18 DONE 1.1s + +#19 resolving provenance for metadata file +#19 DONE 0.1s + Image mtusi-lab6-tools:grade3 Built + Image mtusi-lab6-tools:grade3 Building +#1 [internal] load local bake definitions +#1 reading from stdin 522B 0.1s done +#1 DONE 0.1s + +#2 [internal] load build definition from Dockerfile.tools +#2 transferring dockerfile: 753B 0.0s done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/docker:27.5.1-cli +#3 DONE 0.1s + +#4 [internal] load .dockerignore +#4 transferring context: 216B 0.0s done +#4 DONE 0.1s + +#5 [internal] load build context +#5 transferring context: 22.01kB 0.1s done +#5 DONE 0.2s + +#6 [ 1/12] FROM docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c +#6 resolve docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c +#6 resolve docker.io/library/docker:27.5.1-cli@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c 0.2s done +#6 DONE 0.3s + +#7 [ 2/12] RUN apk add --no-cache python3 py3-pip make && python3 -m venv /opt/venv +#7 CACHED + +#8 [ 3/12] WORKDIR /workspace +#8 CACHED + +#9 [ 4/12] COPY infra/requirements-tools.lock /tmp/requirements-tools.lock +#9 CACHED + +#10 [ 6/12] COPY Makefile lab.json README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +#10 CACHED + +#11 [ 7/12] COPY contracts/ ./contracts/ +#11 CACHED + +#12 [ 5/12] RUN pip install --no-cache-dir -r /tmp/requirements-tools.lock +#12 CACHED + +#13 [ 8/12] COPY tests/ ./tests/ +#13 CACHED + +#14 [ 9/12] COPY scripts/check_contract.py scripts/validate_openapi.py ./scripts/ +#14 CACHED + +#15 [10/12] COPY --chmod=0755 scripts/restart.sh scripts/stop-one.sh scripts/lab-containers.sh ./scripts/ +#15 CACHED + +#16 [11/12] RUN mkdir -p /workspace/evidence +#16 CACHED + +#17 [12/12] COPY student-tests/ ./student-tests/ +#17 DONE 0.2s + +#18 exporting to image +#18 exporting layers +#18 exporting layers 0.6s done +#18 exporting manifest sha256:7ad98e41a2209ab13dd1a2a2b026595024f5b8dbd64ce3b09cd4cf04c632c49f 0.1s done +#18 exporting config sha256:d956f57aa9d6e90063aaa834bc24b759b792e3ffa44b3ced4d64ed2462149390 0.0s done +#18 exporting attestation manifest sha256:7de2aa04a16f3de01a91e00e596ddb31718b22499043e679169e63bca26280ba 0.0s done +#18 exporting manifest list sha256:ec8366828f34de355623e822df1a05663920ad239263dfece72c92e6e6e163b1 0.0s done +#18 naming to docker.io/library/mtusi-lab6-tools:grade3 +#18 naming to docker.io/library/mtusi-lab6-tools:grade3 0.0s done +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 +#18 unpacking to docker.io/library/mtusi-lab6-tools:grade3 3.6s done +#18 DONE 4.6s + +#19 resolving provenance for metadata file +#19 DONE 0.1s + Image mtusi-lab6-tools:grade3 Built diff --git a/evidence/grade3/compose-config.txt b/evidence/grade3/compose-config.txt new file mode 100644 index 0000000..bfda4a4 --- /dev/null +++ b/evidence/grade3/compose-config.txt @@ -0,0 +1 @@ +PASS: docker compose config --quiet (exit 0) diff --git a/evidence/grade3/compose-ps.txt b/evidence/grade3/compose-ps.txt new file mode 100644 index 0000000..b05098e --- /dev/null +++ b/evidence/grade3/compose-ps.txt @@ -0,0 +1,15 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab6-grade3-api1-1 mtusi-lab6-api:grade3 "sh -c 'exec uvicorn…" api1 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab6-grade3-api2-1 mtusi-lab6-api:grade3 "sh -c 'exec uvicorn…" api2 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab6-grade3-bootstrap-1 mtusi-lab6-api:grade3 "python -m src.s3" bootstrap 2 minutes ago Exited (0) 2 minutes ago +mtusi-lab6-grade3-broker-1 rabbitmq:4.1.4-alpine "sh -ec 'export RABB…" broker 2 minutes ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab6-grade3-grafana-1 grafana/grafana:12.0.2 "/run.sh" grafana 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:3006->3000/tcp +mtusi-lab6-grade3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8086->8080/tcp +mtusi-lab6-grade3-loki-1 grafana/loki:3.5.1 "/usr/bin/loki -conf…" loki 2 minutes ago Up 2 minutes 3100/tcp +mtusi-lab6-grade3-migrate-1 mtusi-lab6-api:grade3 "python -m src.db" migrate 2 minutes ago Exited (0) 2 minutes ago +mtusi-lab6-grade3-otel-collector-1 otel/opentelemetry-collector-contrib:0.127.0 "/otelcol-contrib --…" otel-collector 2 minutes ago Up 2 minutes 4317-4318/tcp, 55678-55679/tcp +mtusi-lab6-grade3-postgres-1 postgres:16.8-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp +mtusi-lab6-grade3-prometheus-1 prom/prometheus:v3.4.1 "/bin/prometheus --c…" prometheus 2 minutes ago Up 2 minutes (healthy) 9090/tcp +mtusi-lab6-grade3-s3-1 rustfs/rustfs:1.0.0-alpha.85@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a "sh -ec 'unset RUSTF…" s3 2 minutes ago Up 2 minutes (healthy) 9000-9001/tcp +mtusi-lab6-grade3-tempo-1 grafana/tempo:2.8.0 "/tempo -config.file…" tempo 2 minutes ago Up 2 minutes +mtusi-lab6-grade3-worker-1 mtusi-lab6-api:grade3 "python -m src.worker" worker 2 minutes ago Up 2 minutes 8080/tcp diff --git a/evidence/grade3/container-inspect.txt b/evidence/grade3/container-inspect.txt new file mode 100644 index 0000000..4bf0eff --- /dev/null +++ b/evidence/grade3/container-inspect.txt @@ -0,0 +1,14 @@ +/mtusi-lab6-grade3-api1-1 image=mtusi-lab6-api:grade3 project=mtusi-lab6-grade3 created=2026-11-20T17:51:27.351569315Z started=2026-11-20T17:51:46.754219716Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-api2-1 image=mtusi-lab6-api:grade3 project=mtusi-lab6-grade3 created=2026-11-20T17:51:27.361523016Z started=2026-11-20T17:51:46.332861172Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-bootstrap-1 image=mtusi-lab6-api:grade3 project=mtusi-lab6-grade3 created=2026-11-20T17:51:26.493803431Z started=2026-11-20T17:51:38.178388762Z state=exited exit=0 health=n/a OOM=false +/mtusi-lab6-grade3-broker-1 image=rabbitmq:4.1.4-alpine project=mtusi-lab6-grade3 created=2026-11-20T17:51:25.073582891Z started=2026-11-20T17:51:28.283219185Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-grafana-1 image=grafana/grafana:12.0.2 project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.822482067Z started=2026-11-20T17:51:29.301540158Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-lb-1 image=haproxy:3.0.11-alpine project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.876890872Z started=2026-11-20T17:51:30.399081536Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-loki-1 image=grafana/loki:3.5.1 project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.947886079Z started=2026-11-20T17:51:31.621311624Z state=running exit=0 health=n/a OOM=false +/mtusi-lab6-grade3-migrate-1 image=mtusi-lab6-api:grade3 project=mtusi-lab6-grade3 created=2026-11-20T17:51:26.503381432Z started=2026-11-20T17:51:43.569465389Z state=exited exit=0 health=n/a OOM=false +/mtusi-lab6-grade3-otel-collector-1 image=otel/opentelemetry-collector-contrib:0.127.0 project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.671924852Z started=2026-11-20T17:51:32.312936473Z state=running exit=0 health=n/a OOM=false +/mtusi-lab6-grade3-postgres-1 image=postgres:16.8-bookworm project=mtusi-lab6-grade3 created=2026-11-20T17:51:25.072930091Z started=2026-11-20T17:51:32.883020114Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-prometheus-1 image=prom/prometheus:v3.4.1 project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.887082973Z started=2026-11-20T17:51:35.224231081Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-s3-1 image=rustfs/rustfs:1.0.0-alpha.85@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.85115897Z started=2026-11-20T17:51:34.35986042Z state=running exit=0 health=healthy OOM=false +/mtusi-lab6-grade3-tempo-1 image=grafana/tempo:2.8.0 project=mtusi-lab6-grade3 created=2026-11-20T17:51:24.887638473Z started=2026-11-20T17:51:33.607690566Z state=running exit=0 health=n/a OOM=false +/mtusi-lab6-grade3-worker-1 image=mtusi-lab6-api:grade3 project=mtusi-lab6-grade3 created=2026-11-20T17:51:27.379623517Z started=2026-11-20T17:51:55.088842893Z state=running exit=0 health=n/a OOM=false diff --git a/evidence/grade3/course-sha256.txt b/evidence/grade3/course-sha256.txt new file mode 100644 index 0000000..b186522 --- /dev/null +++ b/evidence/grade3/course-sha256.txt @@ -0,0 +1,19 @@ +279BC276C5D873BBCEF73ECAA6862EB3F83DEFFE504B6DB3CD3052E1CB060C93 .github/pull_request_template.md +08F247085938BD1EFC4B8F7F48E3D520B1DC11CE569F4259491E8F01B9ABD02C CONTRIBUTING.md +1E3C65E345B7F89D08ED86A3B50061935EE8157907CB4043CFA40689781F54E0 COURSE.md +A2F517A3F90EE35D57F4FF6221FE65EC0F4DA3988B1B128748EC2CB0463EF281 README.md +4C04FAEDE4A2FFA35183455B51DA21764B7FA3997C0A4B3963614B82586EED28 contracts/README.md +D7C4C39E5393FC40D2AA5065A8B874274110EB8553F47C05D31DF17C0FA41D21 contracts/attachment-states.md +2816DA2E63058D55BDCBECEB92B231EA2509C86DF8C5A0AA908BE83BD0599469 contracts/examples.json +2B92256EE5E4646E3B16145A6227C1D65D656139207070790810F07E5C198606 contracts/infrastructure.json +060362D0877C07A5229556AC5AA012E164931E78DF72761F6A84771C6AC2A9F9 contracts/jobs.schema.json +B11EBF3AD60D6E39ED83D9BC8C7F3B0C038DEC9DAEC8A5560BCA3CEFA506DAC1 contracts/observability.md +03A97804D28C587CE4DE80591DF0706508E7072767B79FAA79D64E14598155A1 contracts/openapi.json +89AFB3063EC0CCD7CB68FDAC1DE57513F9CC81B77BAD96F202DAACA2B66A5E82 tests/README.md +B318084E2A4C380DB91A55D082190BF33979FA556A4C4CA87E01F46AE275775C tests/client.py +0EAA4E8837032D1786FC0C4B1576656FF3F4D1931FDD51B80E5C053F88AB3872 tests/fixtures/README.md +99EA7533C441F79CD630D15CE4D58D1EC4E06F6F0EFB3B77A5C088305CD1BD75 tests/fixtures/sample.png +EF48D8882AB138B9699A173E7419E5814C22A10B01B6B0DDBB50EA49481A5BAA tests/image_probe.py +9A9AFF05E109630E243C3B092D7BD564344E4872B29E38B6618647A08460A72E tests/load.py +4BA30B9C81A0664127DDAB2B0BEA5E76AA2087BE8BB25AA13CD5A8E69BE23FDD tests/resilience.py +354E168E1E644145C3FF50BBBE189FD217CCE5659AA125F588098025FFAB0FDF tests/smoke.py diff --git a/evidence/grade3/docker-stats.txt b/evidence/grade3/docker-stats.txt new file mode 100644 index 0000000..83f6efd --- /dev/null +++ b/evidence/grade3/docker-stats.txt @@ -0,0 +1,15 @@ +Snapshot UTC 2026-11-20T18:06:07.1639179Z +Command: docker stats --no-stream (running containers of mtusi-lab6-grade3) +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS +6ecf4f039efd mtusi-lab6-grade3-api1-1 30.01% 76.84MiB / 7.676GiB 0.98% 290kB / 367kB 0B / 0B 10 +255f4f7f0579 mtusi-lab6-grade3-api2-1 6.63% 74.52MiB / 7.676GiB 0.95% 13.9kB / 31.9kB 0B / 0B 9 +3f09ee76622f mtusi-lab6-grade3-broker-1 0.64% 77.36MiB / 7.676GiB 0.98% 8.41kB / 6.64kB 0B / 86kB 30 +2c03a3094afe mtusi-lab6-grade3-grafana-1 0.48% 76.96MiB / 384MiB 20.04% 65.1kB / 6.69kB 0B / 209kB 14 +6cde871f4dbc mtusi-lab6-grade3-lb-1 0.27% 73.6MiB / 7.676GiB 0.94% 15.7MB / 14.6MB 0B / 0B 5 +7ee4dcfdd2ee mtusi-lab6-grade3-loki-1 2.45% 73.74MiB / 448MiB 16.46% 203kB / 46.7kB 0B / 475kB 9 +94da2240abad mtusi-lab6-grade3-otel-collector-1 1.52% 54.77MiB / 192MiB 28.52% 1.24MB / 672kB 0B / 0B 10 +27797c076e8c mtusi-lab6-grade3-postgres-1 14.15% 26.39MiB / 7.676GiB 0.34% 2.04MB / 2.14MB 16.4kB / 647kB 8 +0d91c35f7d66 mtusi-lab6-grade3-prometheus-1 0.03% 22.01MiB / 384MiB 5.73% 137kB / 19.2kB 0B / 160kB 10 +28a42a667d04 mtusi-lab6-grade3-s3-1 9.58% 90.71MiB / 7.676GiB 1.15% 1.23MB / 859kB 0B / 1.25MB 14 +413ec4b05acc mtusi-lab6-grade3-tempo-1 0.99% 272.8MiB / 448MiB 60.89% 205kB / 64.6kB 0B / 2.99MB 9 +ed7c1dede058 mtusi-lab6-grade3-worker-1 0.02% 70.1MiB / 7.676GiB 0.89% 386kB / 326kB 0B / 0B 4 diff --git a/evidence/grade3/environment.txt b/evidence/grade3/environment.txt new file mode 100644 index 0000000..81c2ccc --- /dev/null +++ b/evidence/grade3/environment.txt @@ -0,0 +1,71 @@ +Measured UTC: 2026-11-20T17:53:40.0056858Z +docker version +Client: + Version: 29.9.3 + API version: 1.56 + Go version: go1.26.10 + Git commit: 5457199 + Built: Tue Nov 3 21:46:10 2026 + OS/Arch: windows/amd64 + Context: desktop-linux + +Server: Docker Desktop 4.99.0 (245822) + Engine: + Version: 29.9.3 + API version: 1.56 (minimum version 1.40) + Go version: go1.26.10 + Git commit: 70efc0e + Built: Tue Nov 3 21:41:42 2026 + OS/Arch: linux/amd64 + Experimental: false + containerd: + Version: v2.4.2 + GitCommit: b7a25261e029d68978bbf290df0a62fc819b3c51 + runc: + Version: 1.5.2 + GitCommit: v1.5.2-0-g29dd3dc2 + docker-init: + Version: 0.19.0 + GitCommit: de40ad0 +docker compose version +Docker Compose version v5.7.0 +docker info (selected fields) +CPU=4 RAM_bytes=8242376704 Engine=29.9.3 OS=linux +docker system df +TYPE TOTAL ACTIVE SIZE RECLAIMABLE +Images 35 19 5.569GB 936.4MB (16%) +Containers 71 12 16.28MB 16.12MB (99%) +Local Volumes 39 27 886.9MB 213.7MB (24%) +Build Cache 34 0 752.3MB 288.4MB +Volumes for Lab 6 +DRIVER VOLUME NAME +local mtusi-lab6-grade3_broker_data +local mtusi-lab6-grade3_grafana_data +local mtusi-lab6-grade3_loki_data +local mtusi-lab6-grade3_postgres_data +local mtusi-lab6-grade3_prometheus_data +local mtusi-lab6-grade3_s3_data +local mtusi-lab6-grade3_tempo_data +docker compose images +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +mtusi-lab6-grade3-api1-1 mtusi-lab6-api grade3 linux/amd64 190c5a8e35b3 323MB 2 minutes ago +mtusi-lab6-grade3-api2-1 mtusi-lab6-api grade3 linux/amd64 190c5a8e35b3 323MB 2 minutes ago +mtusi-lab6-grade3-bootstrap-1 mtusi-lab6-api grade3 linux/amd64 190c5a8e35b3 323MB 2 minutes ago +mtusi-lab6-grade3-broker-1 rabbitmq 4.1.4-alpine linux/amd64 b736d649308e 251MB 13 months ago +mtusi-lab6-grade3-grafana-1 grafana/grafana 12.0.2 linux/amd64 b5b59bfc7561 897MB 17 months ago +mtusi-lab6-grade3-lb-1 haproxy 3.0.11-alpine linux/amd64 e49b7beb96f4 44.4MB 17 months ago +mtusi-lab6-grade3-loki-1 grafana/loki 3.5.1 linux/amd64 a74594532eec 165MB 18 months ago +mtusi-lab6-grade3-migrate-1 mtusi-lab6-api grade3 linux/amd64 190c5a8e35b3 323MB 2 minutes ago +mtusi-lab6-grade3-otel-collector-1 otel/opentelemetry-collector-contrib 0.127.0 linux/amd64 e94cfd92357a 427MB 18 months ago +mtusi-lab6-grade3-postgres-1 postgres 16.8-bookworm linux/amd64 301bcb60b8a3 617MB 21 months ago +mtusi-lab6-grade3-prometheus-1 prom/prometheus v3.4.1 linux/amd64 9abc6cf6aea7 427MB 17 months ago +mtusi-lab6-grade3-s3-1 rustfs/rustfs linux/amd64 23ff2af1ec2d 313MB 8 months ago +mtusi-lab6-grade3-tempo-1 grafana/tempo 2.8.0 linux/amd64 3cbb1b4eae79 156MB 17 months ago +mtusi-lab6-grade3-worker-1 mtusi-lab6-api grade3 linux/amd64 190c5a8e35b3 323MB 2 minutes ago +Host readiness and Grafana +{"status":"ok"} +{ + "database": "ok", + "version": "12.0.2", + "commit": "5bda17e7c1cb313eb96266f2fdda73a6b35c3977" +} diff --git a/evidence/grade3/failover.txt b/evidence/grade3/failover.txt new file mode 100644 index 0000000..becc6f2 --- /dev/null +++ b/evidence/grade3/failover.txt @@ -0,0 +1,175 @@ +Public failover attempt 1 + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-tools-run-8df9718c7702 Creating + Container mtusi-lab6-grade3-tools-run-8df9718c7702 Created +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 0, "first_recovery_seconds": 2.068} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Verified stopped=api1 survivor=api2 + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-api1-1 Starting + Container mtusi-lab6-grade3-api1-1 Started + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy +Public failover attempt 2 + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-tools-run-75ef0a92c9da Creating + Container mtusi-lab6-grade3-tools-run-75ef0a92c9da Created +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 1.831} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Verified stopped=api1 survivor=api2 + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-api1-1 Starting + Container mtusi-lab6-grade3-api1-1 Started + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy +Public failover attempt 3 + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy + Container mtusi-lab6-grade3-tools-run-4d4a510144d4 Creating + Container mtusi-lab6-grade3-tools-run-4d4a510144d4 Created +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 1.36} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Verified stopped=api2 survivor=api1 + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-api2-1 Starting + Container mtusi-lab6-grade3-api2-1 Started + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-api2-1 Healthy +PASS: public failover for api1 and api2, both replicas recovered. diff --git a/evidence/grade3/git-status.txt b/evidence/grade3/git-status.txt new file mode 100644 index 0000000..a46a5e6 --- /dev/null +++ b/evidence/grade3/git-status.txt @@ -0,0 +1,26 @@ +## solution/bvt2301-gamayunov + M .env.example + M .gitignore + M REPORT.md +?? .gitattributes +?? Dockerfile +?? compose.yaml +?? evidence/grade3/ +?? infra/Dockerfile.tools +?? infra/grafana/ +?? infra/haproxy.cfg +?? infra/otel/collector.yaml +?? infra/otel/loki.yaml +?? infra/otel/prometheus.yaml +?? infra/otel/tempo.yaml +?? infra/requirements-tools.lock +?? migrations/ +?? requirements.lock +?? requirements.txt +?? scripts/init-env.ps1 +?? scripts/lab-containers.sh +?? scripts/restart.sh +?? scripts/stop-one.sh +?? scripts/test-failover-both.ps1 +?? src/ +?? student-tests/ diff --git a/evidence/grade3/image-tags.txt b/evidence/grade3/image-tags.txt new file mode 100644 index 0000000..641500e --- /dev/null +++ b/evidence/grade3/image-tags.txt @@ -0,0 +1,14 @@ +grafana/tempo:2.8.0 +grafana/loki:3.5.1 +mtusi-lab6-api:grade3 +mtusi-lab6-api:grade3 +mtusi-lab6-api:grade3 +haproxy:3.0.11-alpine +rustfs/rustfs:1.0.0-alpha.85@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a +otel/opentelemetry-collector-contrib:0.127.0 +grafana/grafana:12.0.2 +prom/prometheus:v3.4.1 +mtusi-lab6-api:grade3 +mtusi-lab6-api:grade3 +rabbitmq:4.1.4-alpine +postgres:16.8-bookworm diff --git a/evidence/grade3/integrity.txt b/evidence/grade3/integrity.txt new file mode 100644 index 0000000..9e883ec --- /dev/null +++ b/evidence/grade3/integrity.txt @@ -0,0 +1,7 @@ +Protected teacher files rehashed: 19; mismatches: 0 +git diff --cached --name-only: empty +git diff --check: PASS +PowerShell bootstrap repeated: existing .env and secret hashes unchanged +Windows PowerShell 5.1 clean directory + repeat: seven local secret files, hashes unchanged +Evidence text files scanned: 25; sensitive matches: 0 +API and worker Docker logs scanned in memory; sensitive matches: 0 diff --git a/evidence/grade3/lab5-regression.txt b/evidence/grade3/lab5-regression.txt new file mode 100644 index 0000000..840daad --- /dev/null +++ b/evidence/grade3/lab5-regression.txt @@ -0,0 +1,72 @@ + Container mtusi-lab6-grade3-tools-run-cf48c4e93752 Creating + Container mtusi-lab6-grade3-tools-run-cf48c4e93752 Created +LB ready after 0.048s +PASS Argon2id verifier confirmed without reading hash +docker stop worker containers=ed7c1dede058 +RabbitMQ persistent carrier schema valid job=c6fafcfc-67b7-4759-a415-b36fcb0f6291 attempt=1 fields=attachment_id,attempt,created_at,job_id,pipeline_version,schema_version,type +private RustFS object unsigned GET HTTP 403 +PASS stopped worker: uploads=202 ids=1a85cbc6-9e7f-48ed-ae14-703a74aeb9fd,f65a3c3f-093a-497e-a9d8-61b58289ed4a; queued content=409; linked and ordinary POST=201; concurrent same-key response identical via api1/api2 +docker restart api1 api2 containers=6ecf4f039efd,255f4f7f0579 +LB ready after 2.780s +api1 started before=2026-11-20T17:58:31.864675791Z after=2026-11-20T18:03:37.351711283Z +api2 started before=2026-11-20T17:59:05.976165298Z after=2026-11-20T18:03:38.403695041Z +broker queue=attachment.process.v1 durable=true ready=2 unacked=0 +docker restart broker containers=3f09ee76622f +broker healthy after 26.971s +broker started before=2026-11-20T17:54:23.1266914Z after=2026-11-20T18:03:46.161816838Z volume=mtusi-lab6-grade3_broker_data +broker queue=attachment.process.v1 durable=true ready=2 unacked=0 +PASS both APIs + RabbitMQ restarted with queued work and unchanged RabbitMQ volume +docker start worker containers=ed7c1dede058 +HTTP processing state after 3.488s +PASS processing metadata visible, content=409, linking=201 before final SQL commit +ready id=1a85cbc6-9e7f-48ed-ae14-703a74aeb9fd wait=0.110s original_sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 jpeg=256x160 thumb_sha256=b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546 read=api2/api1 +ready id=f65a3c3f-093a-497e-a9d8-61b58289ed4a wait=0.058s original_sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 jpeg=256x160 thumb_sha256=b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546 read=api1/api2 +worker logged two ready duplicates in Loki after 0.771s +broker queue=attachment.process.v1 durable=true ready=0 unacked=0 +worker acknowledged all messages after 1.881s +PASS duplicate delivery: ready snapshot unchanged, one SQL row and one S3 variant +private RustFS object unsigned GET HTTP 403 +docker start worker containers=ed7c1dede058 +negative upload wrong-mime HTTP 415 +negative upload corrupt HTTP 415 +negative upload oversize HTTP 413 +negative upload private HTTP 404 +negative upload anonymous HTTP 401 +negative upload invalid%2Fname.png HTTP 400 +negative upload 40,005,000 decoded pixels HTTP 413 +negative truncated JPEG HTTP 415 +ready id=00b1649c-d3fe-4d2e-90d3-13e464d6945d wait=0.622s original_sha256=91dad4332c3984d2f019280575a82fac7a0d7f859cf57e860814a6f786622a71 jpeg=81x256 thumb_sha256=edd37f18bfa4ad50788f2ae8c8bf97473d433e326531ca92e942c357af90770f read=api1/api2 +ready id=763de91e-88d2-492b-a6d1-4147911db580 wait=0.452s original_sha256=b15af0b56245cde806601ecff6e8f7fa1df12e7b6cd55530d356fc01a08c0aaf jpeg=37x19 thumb_sha256=8db1b1571a2b9333f7fe23f2a825770cdf9a18e6166a3fa08937fd786d6c50d8 read=api2/api1 +PASS true JPEG, EXIF orientation, floor rounding, no upscale, alpha pixel=(255, 255, 255) +docker stop worker containers=ed7c1dede058 +docker stop s3 containers=28a42a667d04 +docker start worker containers=ed7c1dede058 +persistent retry after real S3 stop after 9.947s +storage outage SQL status|attempt|reason|max=queued|2|storage_unavailable|5 +docker start s3 worker containers=28a42a667d04,ed7c1dede058 +LB ready after 0.619s +ready id=dbb71d1e-5b7d-4736-8866-e62994b26cfd wait=2.868s original_sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 jpeg=256x160 thumb_sha256=b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546 read=api2/api1 +PASS temporary S3 outage recovered the same attachment +docker stop worker containers=ed7c1dede058 +docker start worker containers=ed7c1dede058 +failed 5bee0f73-81a1-4b02-989c-f7dc901055a8 after 3.985s +PASS safe failure code=invalid_image; status|attempt|reason|max=failed|1|invalid_image|5; content/link HTTP 409 +failed 69a6c62d-8cf0-4d00-925b-7f5730cc6640 after 11.281s +PASS safe failure code=retry_exhausted; status|attempt|reason|max=failed|5|storage_unavailable|5; content/link HTTP 409 +docker stop broker containers=3f09ee76622f +PASS unavailable broker upload=503 (no false 202), ordinary message=201 +docker start broker containers=3f09ee76622f +broker healthy after failure case after 10.435s +docker stop api1 containers=6ecf4f039efd +api2 handles existing Bearer with api1 stopped after 0.013s +ready id=ba4d0177-963c-4a59-9d96-62e0bd3a99c2 wait=0.583s original_sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 jpeg=256x160 thumb_sha256=b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546 read=api2/api2 +docker start api1 containers=6ecf4f039efd +api1 container healthy after 11.701s +docker stop api2 containers=255f4f7f0579 +api1 handles existing Bearer with api2 stopped after 0.034s +ready id=bf092545-2821-4994-a311-3867657e8025 wait=0.491s original_sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 jpeg=256x160 thumb_sha256=b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546 read=api1/api1 +docker start api2 containers=255f4f7f0579 +api2 container healthy after 10.865s +PASS authorized new uploads and thumbnail reads with either replica stopped +broker queue=attachment.process.v1 durable=true ready=0 unacked=0 +PASS Lab 5 grade 3 Docker scenarios diff --git a/evidence/grade3/lab5-transfer.txt b/evidence/grade3/lab5-transfer.txt new file mode 100644 index 0000000..010665a --- /dev/null +++ b/evidence/grade3/lab5-transfer.txt @@ -0,0 +1,32 @@ +src/__init__.py +src/attachments.py +src/config.py +src/db.py +src/errors.py +src/jobs.py +src/main.py +src/models.py +src/pagination.py +src/s3.py +src/security.py +src/store.py +src/thumbnail.py +src/worker.py +migrations/001_initial.sql +migrations/002_message_idempotency.sql +migrations/003_attachments.sql +migrations/004_attachment_jobs.sql +student-tests/README.md +student-tests/grade3.py +compose.yaml +Dockerfile +requirements.txt +requirements.lock +.gitattributes +infra/haproxy.cfg +infra/Dockerfile.tools +infra/requirements-tools.lock +scripts/init-env.ps1 +scripts/lab-containers.sh +scripts/restart.sh +scripts/stop-one.sh diff --git a/evidence/grade3/load.txt b/evidence/grade3/load.txt new file mode 100644 index 0000000..e3fa293 --- /dev/null +++ b/evidence/grade3/load.txt @@ -0,0 +1,19 @@ + Container mtusi-lab6-grade3-tools-run-b8e8d0b9882a Creating + Container mtusi-lab6-grade3-tools-run-b8e8d0b9882a Created +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 8.0, + "elapsed_seconds": 8.148, + "concurrency": 2, + "successful_requests": 783, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 96.095, + "successful_latency_ms": { + "p50": 14.284, + "p95": 46.57, + "p99": 87.944 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/grade3/observability-before-tempo-probe.txt b/evidence/grade3/observability-before-tempo-probe.txt new file mode 100644 index 0000000..3f2e333 --- /dev/null +++ b/evidence/grade3/observability-before-tempo-probe.txt @@ -0,0 +1,47 @@ + Container mtusi-lab6-grade3-tools-run-ee7c6f332fcd Creating + Container mtusi-lab6-grade3-tools-run-ee7c6f332fcd Created +http://prometheus:9090/-/ready after 0.008s +http://loki:3100/ready after 0.011s +http://tempo:3200/ready after 0.006s +http://grafana:3000/api/health after 0.009s +http://otel-collector:13133/ after 0.005s +{"error_status": 401, "evidence": "traffic", "get": 48, "post_status": 201, "seconds": 7.386} +docker stop worker containers=ed7c1dede058 +{"accepted": 202, "attachment_id": "ced44dad-1ff8-4bf4-8ad1-59a9caff4710", "evidence": "worker.stop", "status": "queued"} +docker start worker containers=ed7c1dede058 +ready id=ced44dad-1ff8-4bf4-8ad1-59a9caff4710 wait=2.545s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api1 +{"attachment_id": "ced44dad-1ff8-4bf4-8ad1-59a9caff4710", "evidence": "worker.recovered", "seconds": 5.555, "status": "ready"} +docker stop worker containers=ed7c1dede058 +docker stop s3 containers=28a42a667d04 +docker start worker containers=ed7c1dede058 +terminal S3 outage after 21.291s +{"attachment_id": "ab4198ec-b981-4954-849f-fecdaae59f20", "error_code": "retry_exhausted", "evidence": "s3.outage", "http_status": 503, "request_id": "38b5b8fe-cbf9-4605-b143-adaa45276b92", "seconds": 33.442, "status": "failed"} +docker start s3 worker containers=28a42a667d04,ed7c1dede058 +LB ready after 0.213s +ready id=ced44dad-1ff8-4bf4-8ad1-59a9caff4710 wait=0.102s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api1/api1 +ready id=6833dc26-3e67-4451-884f-22f5b9888dd1 wait=0.351s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api1/api1 +{"evidence": "s3.recovered", "existing": "ced44dad-1ff8-4bf4-8ad1-59a9caff4710", "new": "6833dc26-3e67-4451-884f-22f5b9888dd1", "seconds": 34.617} +api.success after 0.078s +{"evidence": "api.success", "log": ["1795197559213180160", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "http.completed", "flags": "1", "method": "POST", "observed_timestamp": "1795197559213226645", "request_id": "bf1bb2a5-c2fb-4e38-8000-b5f04dd35ba5", "route": "/api/v1/chats/{chat_id}/messages", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "f1f4bbd168aa678c", "status_code": "201", "trace_id": "76a2c5927dfb49115c164d05d03a0717"}], "query": "{service_name=\"messenger-api\"} | request_id=\"bf1bb2a5-c2fb-4e38-8000-b5f04dd35ba5\" | event=\"http.completed\"", "tempo": {"duration_ns": 8295101, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "POST"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/chats/{chat_id}/messages"}}, {"key": "http.response.status_code", "value": {"intValue": "201"}}, {"key": "request.id", "value": {"stringValue": "bf1bb2a5-c2fb-4e38-8000-b5f04dd35ba5"}}, {"key": "outcome", "value": {"stringValue": "2xx"}}], "endTimeUnixNano": "1795197559213287145", "kind": "SPAN_KIND_SERVER", "name": "POST /api/v1/chats/{chat_id}/messages", "spanId": "8fS70WiqZ4w=", "startTimeUnixNano": "1795197559204992044", "status": {"code": "STATUS_CODE_OK"}, "traceId": "dqLFkn37SRFcFk0F0DoHFw=="}, "span_id": "f1f4bbd168aa678c", "trace_id": "76a2c5927dfb49115c164d05d03a0717"}} +api.401 after 0.030s +{"evidence": "api.401", "log": ["1795197559219530240", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "error_code": "unauthorized", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197559220888246", "request_id": "eefa634f-a3d9-43f8-a443-8c8e7446bd1f", "route": "/api/v1/users/me", "scope_name": "lab6.events", "service_instance_id": "api2", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "71c851c42aad541d", "status_code": "401", "trace_id": "ef85b2ad2fe979122a3eb46d6f3d4f3a"}], "query": "{service_name=\"messenger-api\"} | request_id=\"eefa634f-a3d9-43f8-a443-8c8e7446bd1f\" | event=\"http.completed\"", "tempo": {"duration_ns": 3564100, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api2"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/users/me"}}, {"key": "http.response.status_code", "value": {"intValue": "401"}}, {"key": "request.id", "value": {"stringValue": "eefa634f-a3d9-43f8-a443-8c8e7446bd1f"}}, {"key": "outcome", "value": {"stringValue": "4xx"}}], "endTimeUnixNano": "1795197559221038246", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/users/me", "spanId": "cchRxCqtVB0=", "startTimeUnixNano": "1795197559217474146", "status": {"code": "STATUS_CODE_OK"}, "traceId": "74WyrS/peRIqPrRtbz1POg=="}, "span_id": "71c851c42aad541d", "trace_id": "ef85b2ad2fe979122a3eb46d6f3d4f3a"}} +api.503 after 0.035s +{"evidence": "api.503", "log": ["1795197580195567872", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "ERROR", "error_code": "unavailable", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197580195670846", "request_id": "38b5b8fe-cbf9-4605-b143-adaa45276b92", "route": "/api/v1/attachments/{attachment_id}/content", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "17", "severity_text": "ERROR", "span_id": "72399d344e7678c1", "status_code": "503", "trace_id": "d6c51658d577d16464b35833972d216a"}], "query": "{service_name=\"messenger-api\"} | request_id=\"38b5b8fe-cbf9-4605-b143-adaa45276b92\" | event=\"http.completed\"", "tempo": {"duration_ns": 3321821646, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/attachments/{attachment_id}/content"}}, {"key": "http.response.status_code", "value": {"intValue": "503"}}, {"key": "request.id", "value": {"stringValue": "38b5b8fe-cbf9-4605-b143-adaa45276b92"}}, {"key": "outcome", "value": {"stringValue": "5xx"}}], "endTimeUnixNano": "1795197580195830346", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/attachments/{attachment_id}/content", "spanId": "cjmdNE52eME=", "startTimeUnixNano": "1795197576874008700", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "1sUWWNV30WRks1gzly0hag=="}, "span_id": "72399d344e7678c1", "trace_id": "d6c51658d577d16464b35833972d216a"}} +worker.ready after 0.040s +{"evidence": "worker.ready", "log": ["1795197572189066496", "job.outcome", {"attachment_id": "ced44dad-1ff8-4bf4-8ad1-59a9caff4710", "attempt": "1", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "job.outcome", "flags": "1", "job_id": "5cd192aa-36ff-4d8b-a18c-b8492838fd20", "job_type": "attachment.process.v1", "observed_timestamp": "1795197572189174414", "outcome": "ready", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "766a065a284e4a0c", "trace_id": "f2a03559daeac165ac3967bc0e509ae5"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"ced44dad-1ff8-4bf4-8ad1-59a9caff4710\" | event=\"job.outcome\" | outcome=\"ready\"", "tempo": {"duration_ns": 91833510, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "5cd192aa-36ff-4d8b-a18c-b8492838fd20"}}, {"key": "attachment_id", "value": {"stringValue": "ced44dad-1ff8-4bf4-8ad1-59a9caff4710"}}, {"key": "attempt", "value": {"intValue": "1"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "outcome", "value": {"stringValue": "ready"}}], "endTimeUnixNano": "1795197572190564314", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "dmoGWihOSgw=", "startTimeUnixNano": "1795197572098730804", "status": {"code": "STATUS_CODE_OK"}, "traceId": "8qA1WdrqwWWsOWe8DlCa5Q=="}, "span_id": "766a065a284e4a0c", "trace_id": "f2a03559daeac165ac3967bc0e509ae5"}} +worker.retry after 0.033s +{"evidence": "worker.retry", "log": ["1795197601572238336", "job.outcome", {"attachment_id": "ab4198ec-b981-4954-849f-fecdaae59f20", "attempt": "4", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "85b18864-7346-4464-9ef2-bc73f7a13e60", "job_type": "attachment.process.v1", "observed_timestamp": "1795197601572347160", "outcome": "retry", "reason": "storage_unavailable", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "36532994c4179f32", "trace_id": "084578dfdd1555f358fa8818e15ddfc7"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"ab4198ec-b981-4954-849f-fecdaae59f20\" | event=\"job.outcome\" | outcome=\"retry\"", "tempo": {"duration_ns": 6299745286, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "85b18864-7346-4464-9ef2-bc73f7a13e60"}}, {"key": "attachment_id", "value": {"stringValue": "ab4198ec-b981-4954-849f-fecdaae59f20"}}, {"key": "attempt", "value": {"intValue": "4"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "retry"}}], "endTimeUnixNano": "1795197601576487960", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "NlMplMQXnzI=", "startTimeUnixNano": "1795197595276742674", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "CEV4390VVfNY+ogY4V3fxw=="}, "span_id": "36532994c4179f32", "trace_id": "084578dfdd1555f358fa8818e15ddfc7"}} +worker.failed after 0.023s +{"evidence": "worker.failed", "log": ["1795197607873423616", "job.outcome", {"attachment_id": "ab4198ec-b981-4954-849f-fecdaae59f20", "attempt": "5", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "85b18864-7346-4464-9ef2-bc73f7a13e60", "job_type": "attachment.process.v1", "observed_timestamp": "1795197607873483475", "outcome": "failed", "reason": "retry_exhausted", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "db5650f89e4b986a", "trace_id": "50a2e92614ce781d910f877da044ff59"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"ab4198ec-b981-4954-849f-fecdaae59f20\" | event=\"job.outcome\" | outcome=\"failed\"", "tempo": {"duration_ns": 6296151014, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "85b18864-7346-4464-9ef2-bc73f7a13e60"}}, {"key": "attachment_id", "value": {"stringValue": "ab4198ec-b981-4954-849f-fecdaae59f20"}}, {"key": "attempt", "value": {"intValue": "5"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "failed"}}], "endTimeUnixNano": "1795197607874650775", "flags": 256, "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "21ZQ+J5LmGo=", "startTimeUnixNano": "1795197601578499761", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "UKLpJhTOeB2RD4d9oET/WQ=="}, "span_id": "db5650f89e4b986a", "trace_id": "50a2e92614ce781d910f877da044ff59"}} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_requests_total", "samples": [{"metric": {"__name__": "messenger_http_requests_total", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.509, "2"]}, {"metric": {"__name__": "messenger_http_requests_total", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.509, "1"]}, {"metric": {"__name__": "messenger_http_requests_total", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197609.509, "3"]}]} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_duration_seconds_sum", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.513, "0.029170603000011397"]}, {"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.513, "0.013631301000032181"]}, {"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197609.513, "9.908517125999992"]}]} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_duration_seconds_count", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.518, "2"]}, {"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.518, "1"]}, {"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197609.518, "3"]}]} +{"checked_series": 378, "evidence": "prometheus", "query": "messenger_http_duration_seconds_bucket", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.521, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.521, "2"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api2", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.521, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api2", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197609.521, "1"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197609.521, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197609.521, "3"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_outcomes_total", "samples": [{"metric": {"__name__": "messenger_job_outcomes_total", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197609.528, "4"]}, {"metric": {"__name__": "messenger_job_outcomes_total", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197609.528, "1"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_duration_seconds_sum", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_sum", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197609.531, "0.33556383699999515"]}, {"metric": {"__name__": "messenger_job_duration_seconds_sum", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197609.531, "3.2915629650000255"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_duration_seconds_count", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_count", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197609.533, "4"]}, {"metric": {"__name__": "messenger_job_duration_seconds_count", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197609.533, "1"]}]} +{"checked_series": 28, "evidence": "prometheus", "query": "messenger_job_duration_seconds_bucket", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "0.005", "outcome": "success"}, "value": [1795197609.535, "0"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "+Inf", "outcome": "success"}, "value": [1795197609.535, "4"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "0.005", "outcome": "fail"}, "value": [1795197609.535, "0"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "+Inf", "outcome": "fail"}, "value": [1795197609.535, "1"]}]} +all real counter and histogram series after 0.040s +{"evidence": "grafana.datasource", "health": {"details": {"application": "Prometheus", "features": {"rulerApiEnabled": false}}, "message": "Successfully queried the Prometheus API.", "status": "OK"}, "type": "prometheus", "uid": "lab6-prometheus", "url": "http://prometheus:9090"} +{"evidence": "grafana.datasource", "health": {"message": "Data source successfully connected.", "status": "OK"}, "type": "loki", "uid": "lab6-loki", "url": "http://loki:3100"} +FAIL: GET http://grafana:3000/api/datasources/uid/lab6-tempo/health: HTTP 404 diff --git a/evidence/grade3/observability-initial.txt b/evidence/grade3/observability-initial.txt new file mode 100644 index 0000000..4c04451 --- /dev/null +++ b/evidence/grade3/observability-initial.txt @@ -0,0 +1,36 @@ + Container mtusi-lab6-grade3-tools-run-12471b1a8945 Creating + Container mtusi-lab6-grade3-tools-run-12471b1a8945 Created +http://prometheus:9090/-/ready after 0.010s +http://loki:3100/ready after 15.042s +http://tempo:3200/ready after 15.039s +http://grafana:3000/api/health after 0.003s +http://otel-collector:13133/ after 0.004s +{"error_status": 401, "evidence": "traffic", "get": 48, "post_status": 201, "seconds": 7.125} +docker stop worker containers=ed7c1dede058 +{"accepted": 202, "attachment_id": "4bc1d9fe-b5aa-4740-a03e-2d037777bbb0", "evidence": "worker.stop", "status": "queued"} +docker start worker containers=ed7c1dede058 +ready id=4bc1d9fe-b5aa-4740-a03e-2d037777bbb0 wait=2.528s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api1 +{"attachment_id": "4bc1d9fe-b5aa-4740-a03e-2d037777bbb0", "evidence": "worker.recovered", "seconds": 5.45, "status": "ready"} +docker stop worker containers=ed7c1dede058 +docker stop s3 containers=28a42a667d04 +docker start worker containers=ed7c1dede058 +terminal S3 outage after 27.702s +{"attachment_id": "f02a02a6-2882-427f-8d4f-2e3dd9ffea9c", "error_code": "retry_exhausted", "evidence": "s3.outage", "http_status": 503, "request_id": "c01ed0eb-5bec-47a7-948d-9ce4d3e3ccb0", "seconds": 33.394, "status": "failed"} +docker start s3 worker containers=28a42a667d04,ed7c1dede058 +LB ready after 1.020s +ready id=4bc1d9fe-b5aa-4740-a03e-2d037777bbb0 wait=0.044s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api2 +ready id=e8a3d047-9580-4321-a2d3-c544591d3173 wait=0.354s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api2 +{"evidence": "s3.recovered", "existing": "4bc1d9fe-b5aa-4740-a03e-2d037777bbb0", "new": "e8a3d047-9580-4321-a2d3-c544591d3173", "seconds": 35.408} +api.success after 0.093s +{"evidence": "api.success", "log": ["1795197317795979520", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "http.completed", "flags": "1", "method": "POST", "observed_timestamp": "1795197317796039376", "request_id": "56903a14-1b3d-4411-af2b-45234650114d", "route": "/api/v1/chats/{chat_id}/messages", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "035bf5cfacec61e6", "status_code": "201", "trace_id": "6c4dba5b356a0aeee2f211dfada6e2e5"}], "query": "{service_name=\"messenger-api\"} | request_id=\"56903a14-1b3d-4411-af2b-45234650114d\" | event=\"http.completed\"", "tempo": {"duration_ns": 12094301, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "POST"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/chats/{chat_id}/messages"}}, {"key": "http.response.status_code", "value": {"intValue": "201"}}, {"key": "request.id", "value": {"stringValue": "56903a14-1b3d-4411-af2b-45234650114d"}}, {"key": "outcome", "value": {"stringValue": "2xx"}}], "endTimeUnixNano": "1795197317796122776", "kind": "SPAN_KIND_SERVER", "name": "POST /api/v1/chats/{chat_id}/messages", "spanId": "A1v1z6zsYeY=", "startTimeUnixNano": "1795197317784028475", "status": {"code": "STATUS_CODE_OK"}, "traceId": "bE26WzVqCu7i8hHfrabi5Q=="}, "span_id": "035bf5cfacec61e6", "trace_id": "6c4dba5b356a0aeee2f211dfada6e2e5"}} +api.401 after 0.038s +{"evidence": "api.401", "log": ["1795197317801392640", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "error_code": "unauthorized", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197317801440076", "request_id": "16534745-11e2-4344-b6b3-4ba973b666a7", "route": "/api/v1/users/me", "scope_name": "lab6.events", "service_instance_id": "api2", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "65ffae0eeaeb18fb", "status_code": "401", "trace_id": "9feec1d2fedc9012ac529709de5305c1"}], "query": "{service_name=\"messenger-api\"} | request_id=\"16534745-11e2-4344-b6b3-4ba973b666a7\" | event=\"http.completed\"", "tempo": {"duration_ns": 1788800, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api2"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/users/me"}}, {"key": "http.response.status_code", "value": {"intValue": "401"}}, {"key": "request.id", "value": {"stringValue": "16534745-11e2-4344-b6b3-4ba973b666a7"}}, {"key": "outcome", "value": {"stringValue": "4xx"}}], "endTimeUnixNano": "1795197317801500976", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/users/me", "spanId": "Zf+uDurrGPs=", "startTimeUnixNano": "1795197317799712176", "status": {"code": "STATUS_CODE_OK"}, "traceId": "n+7B0v7ckBKsUpcJ3lMFwQ=="}, "span_id": "65ffae0eeaeb18fb", "trace_id": "9feec1d2fedc9012ac529709de5305c1"}} +api.503 after 0.045s +{"evidence": "api.503", "log": ["1795197338434219008", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "ERROR", "error_code": "unavailable", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197338434271995", "request_id": "c01ed0eb-5bec-47a7-948d-9ce4d3e3ccb0", "route": "/api/v1/attachments/{attachment_id}/content", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "17", "severity_text": "ERROR", "span_id": "607e0517b2d2aeb5", "status_code": "503", "trace_id": "20efb114d4e8a3ba7b7e96ec33bc1d37"}], "query": "{service_name=\"messenger-api\"} | request_id=\"c01ed0eb-5bec-47a7-948d-9ce4d3e3ccb0\" | event=\"http.completed\"", "tempo": {"duration_ns": 3351038840, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/attachments/{attachment_id}/content"}}, {"key": "http.response.status_code", "value": {"intValue": "503"}}, {"key": "request.id", "value": {"stringValue": "c01ed0eb-5bec-47a7-948d-9ce4d3e3ccb0"}}, {"key": "outcome", "value": {"stringValue": "5xx"}}], "endTimeUnixNano": "1795197338434334095", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/attachments/{attachment_id}/content", "spanId": "YH4FF7LSrrU=", "startTimeUnixNano": "1795197335083295255", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "IO+xFNToo7p7fpbsM7wdNw=="}, "span_id": "607e0517b2d2aeb5", "trace_id": "20efb114d4e8a3ba7b7e96ec33bc1d37"}} +worker.ready after 0.027s +{"evidence": "worker.ready", "log": ["1795197330196337664", "job.outcome", {"attachment_id": "4bc1d9fe-b5aa-4740-a03e-2d037777bbb0", "attempt": "1", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "job.outcome", "flags": "1", "job_id": "3133e0f2-266a-4a9c-8807-de2c1517b041", "job_type": "attachment.process.v1", "observed_timestamp": "1795197330196400251", "outcome": "ready", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "399c054c2ac9431a", "trace_id": "30ff96028fa9123ee3180d3f10aec326"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"4bc1d9fe-b5aa-4740-a03e-2d037777bbb0\" | event=\"job.outcome\" | outcome=\"ready\"", "tempo": {"duration_ns": 105546911, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "3133e0f2-266a-4a9c-8807-de2c1517b041"}}, {"key": "attachment_id", "value": {"stringValue": "4bc1d9fe-b5aa-4740-a03e-2d037777bbb0"}}, {"key": "attempt", "value": {"intValue": "1"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "outcome", "value": {"stringValue": "ready"}}], "endTimeUnixNano": "1795197330197091151", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "OZwFTCrJQxo=", "startTimeUnixNano": "1795197330091544240", "status": {"code": "STATUS_CODE_OK"}, "traceId": "MP+WAo+pEj7jGA0/EK7DJg=="}, "span_id": "399c054c2ac9431a", "trace_id": "30ff96028fa9123ee3180d3f10aec326"}} +worker.retry after 0.026s +{"evidence": "worker.retry", "log": ["1795197340727251200", "job.outcome", {"attachment_id": "f02a02a6-2882-427f-8d4f-2e3dd9ffea9c", "attempt": "1", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "26103726-9852-4ed6-beee-277c4eb66a1b", "job_type": "attachment.process.v1", "observed_timestamp": "1795197340727327025", "outcome": "retry", "reason": "storage_unavailable", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "868e214de898da57", "trace_id": "b97f5f32edd0d27053270ee9abfec37c"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"f02a02a6-2882-427f-8d4f-2e3dd9ffea9c\" | event=\"job.outcome\" | outcome=\"retry\"", "tempo": {"duration_ns": 3341947036, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "26103726-9852-4ed6-beee-277c4eb66a1b"}}, {"key": "attachment_id", "value": {"stringValue": "f02a02a6-2882-427f-8d4f-2e3dd9ffea9c"}}, {"key": "attempt", "value": {"intValue": "1"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "retry"}}], "endTimeUnixNano": "1795197340730601526", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "ho4hTeiY2lc=", "startTimeUnixNano": "1795197337388654490", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "uX9fMu3Q0nBTJw7pq/7DfA=="}, "span_id": "868e214de898da57", "trace_id": "b97f5f32edd0d27053270ee9abfec37c"}} +worker.failed after 0.024s +{"evidence": "worker.failed", "log": ["1795197365946145792", "job.outcome", {"attachment_id": "f02a02a6-2882-427f-8d4f-2e3dd9ffea9c", "attempt": "5", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "26103726-9852-4ed6-beee-277c4eb66a1b", "job_type": "attachment.process.v1", "observed_timestamp": "1795197365946193890", "outcome": "failed", "reason": "retry_exhausted", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "fa3e394d53169297", "trace_id": "794bb1e719d0d35719e7d1f912dc080a"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"f02a02a6-2882-427f-8d4f-2e3dd9ffea9c\" | event=\"job.outcome\" | outcome=\"failed\"", "tempo": {"duration_ns": 6289493045, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "26103726-9852-4ed6-beee-277c4eb66a1b"}}, {"key": "attachment_id", "value": {"stringValue": "f02a02a6-2882-427f-8d4f-2e3dd9ffea9c"}}, {"key": "attempt", "value": {"intValue": "5"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "failed"}}], "endTimeUnixNano": "1795197365946743290", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "+j45TVMWkpc=", "startTimeUnixNano": "1795197359657250245", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "eUux5xnQ01cZ59H5EtwICg=="}, "span_id": "fa3e394d53169297", "trace_id": "794bb1e719d0d35719e7d1f912dc080a"}} +FAIL: timeout waiting for all real counter and histogram series: missing histogram buckets diff --git a/evidence/grade3/observability.txt b/evidence/grade3/observability.txt new file mode 100644 index 0000000..a108646 --- /dev/null +++ b/evidence/grade3/observability.txt @@ -0,0 +1,64 @@ + Container mtusi-lab6-grade3-tools-run-cd12cf604bd2 Creating + Container mtusi-lab6-grade3-tools-run-cd12cf604bd2 Created +http://prometheus:9090/-/ready after 0.022s +http://loki:3100/ready after 0.007s +http://tempo:3200/ready after 0.011s +http://grafana:3000/api/health after 0.013s +http://otel-collector:13133/ after 0.005s +{"error_status": 401, "evidence": "traffic", "get": 48, "post_status": 201, "seconds": 7.167} +docker stop worker containers=ed7c1dede058 +{"accepted": 202, "attachment_id": "13b56006-56c2-4178-a79d-576b7678cb73", "evidence": "worker.stop", "status": "queued"} +docker start worker containers=ed7c1dede058 +ready id=13b56006-56c2-4178-a79d-576b7678cb73 wait=2.556s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api1/api2 +{"attachment_id": "13b56006-56c2-4178-a79d-576b7678cb73", "evidence": "worker.recovered", "seconds": 5.45, "status": "ready"} +docker stop worker containers=ed7c1dede058 +docker stop s3 containers=28a42a667d04 +docker start worker containers=ed7c1dede058 +terminal S3 outage after 21.352s +{"attachment_id": "eafac0b1-8919-483e-95a7-da1cf59d0b01", "error_code": "retry_exhausted", "evidence": "s3.outage", "http_status": 503, "request_id": "328fab9a-a6ea-4466-93ef-79933a570099", "seconds": 33.642, "status": "failed"} +docker start s3 worker containers=28a42a667d04,ed7c1dede058 +LB ready after 1.831s +ready id=13b56006-56c2-4178-a79d-576b7678cb73 wait=0.048s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api2 +ready id=1c0079e4-a6b3-43e0-8768-dfc30b794965 wait=0.351s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api2/api1 +{"evidence": "s3.recovered", "existing": "13b56006-56c2-4178-a79d-576b7678cb73", "new": "1c0079e4-a6b3-43e0-8768-dfc30b794965", "seconds": 36.433} +api.success after 0.075s +{"evidence": "api.success", "log": ["1795197723160978944", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "http.completed", "flags": "1", "method": "POST", "observed_timestamp": "1795197723161052144", "request_id": "39059ac3-538a-4e91-966b-9bed2a28d56c", "route": "/api/v1/chats/{chat_id}/messages", "scope_name": "lab6.events", "service_instance_id": "api2", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "12e525cf65db522c", "status_code": "201", "trace_id": "6da5fd7eed430a7e1312e9c747d67a61"}], "query": "{service_name=\"messenger-api\"} | request_id=\"39059ac3-538a-4e91-966b-9bed2a28d56c\" | event=\"http.completed\"", "tempo": {"duration_ns": 13678001, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api2"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "POST"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/chats/{chat_id}/messages"}}, {"key": "http.response.status_code", "value": {"intValue": "201"}}, {"key": "request.id", "value": {"stringValue": "39059ac3-538a-4e91-966b-9bed2a28d56c"}}, {"key": "outcome", "value": {"stringValue": "2xx"}}], "endTimeUnixNano": "1795197723161136844", "kind": "SPAN_KIND_SERVER", "name": "POST /api/v1/chats/{chat_id}/messages", "spanId": "EuUlz2XbUiw=", "startTimeUnixNano": "1795197723147458843", "status": {"code": "STATUS_CODE_OK"}, "traceId": "baX9fu1DCn4TEunHR9Z6YQ=="}, "span_id": "12e525cf65db522c", "trace_id": "6da5fd7eed430a7e1312e9c747d67a61"}} +api.401 after 0.039s +{"evidence": "api.401", "log": ["1795197723167533824", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "error_code": "unauthorized", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197723167619645", "request_id": "a3ee0ed5-501b-494b-9073-aac6d50cb8b7", "route": "/api/v1/users/me", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "be75aa1d2881f3ac", "status_code": "401", "trace_id": "a9a876c33005c28f2417665ab8b04f8b"}], "query": "{service_name=\"messenger-api\"} | request_id=\"a3ee0ed5-501b-494b-9073-aac6d50cb8b7\" | event=\"http.completed\"", "tempo": {"duration_ns": 2127100, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/users/me"}}, {"key": "http.response.status_code", "value": {"intValue": "401"}}, {"key": "request.id", "value": {"stringValue": "a3ee0ed5-501b-494b-9073-aac6d50cb8b7"}}, {"key": "outcome", "value": {"stringValue": "4xx"}}], "endTimeUnixNano": "1795197723167737845", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/users/me", "spanId": "vnWqHSiB86w=", "startTimeUnixNano": "1795197723165610745", "status": {"code": "STATUS_CODE_OK"}, "traceId": "qah2wzAFwo8kF2ZauLBPiw=="}, "span_id": "be75aa1d2881f3ac", "trace_id": "a9a876c33005c28f2417665ab8b04f8b"}} +api.503 after 0.026s +{"evidence": "api.503", "log": ["1795197743892084736", "http.completed", {"code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "ERROR", "error_code": "unavailable", "event": "http.completed", "flags": "1", "method": "GET", "observed_timestamp": "1795197743892168357", "request_id": "328fab9a-a6ea-4466-93ef-79933a570099", "route": "/api/v1/attachments/{attachment_id}/content", "scope_name": "lab6.events", "service_instance_id": "api1", "service_name": "messenger-api", "service_version": "1.6.0", "severity_number": "17", "severity_text": "ERROR", "span_id": "f08fdec4a995781c", "status_code": "503", "trace_id": "5a31c6bf418fe31369f700b887b07060"}], "query": "{service_name=\"messenger-api\"} | request_id=\"328fab9a-a6ea-4466-93ef-79933a570099\" | event=\"http.completed\"", "tempo": {"duration_ns": 3325219817, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "api1"}, "service.name": {"stringValue": "messenger-api"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-api", "span": {"attributes": [{"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "http.request.method", "value": {"stringValue": "GET"}}, {"key": "http.route", "value": {"stringValue": "/api/v1/attachments/{attachment_id}/content"}}, {"key": "http.response.status_code", "value": {"intValue": "503"}}, {"key": "request.id", "value": {"stringValue": "328fab9a-a6ea-4466-93ef-79933a570099"}}, {"key": "outcome", "value": {"stringValue": "5xx"}}], "endTimeUnixNano": "1795197743892264857", "kind": "SPAN_KIND_SERVER", "name": "GET /api/v1/attachments/{attachment_id}/content", "spanId": "8I/exKmVeBw=", "startTimeUnixNano": "1795197740567045040", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "WjHGv0GP4xNp9wC4h7BwYA=="}, "span_id": "f08fdec4a995781c", "trace_id": "5a31c6bf418fe31369f700b887b07060"}} +worker.ready after 0.032s +{"evidence": "worker.ready", "log": ["1795197735769973760", "job.outcome", {"attachment_id": "13b56006-56c2-4178-a79d-576b7678cb73", "attempt": "1", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "INFO", "event": "job.outcome", "flags": "1", "job_id": "a95438c9-c463-4bb4-8229-90db62bc70b2", "job_type": "attachment.process.v1", "observed_timestamp": "1795197735770037179", "outcome": "ready", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "9", "severity_text": "INFO", "span_id": "28237be6723a2529", "trace_id": "9e838857cbb81f9149e42b5d85f5aa00"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"13b56006-56c2-4178-a79d-576b7678cb73\" | event=\"job.outcome\" | outcome=\"ready\"", "tempo": {"duration_ns": 108465110, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "a95438c9-c463-4bb4-8229-90db62bc70b2"}}, {"key": "attachment_id", "value": {"stringValue": "13b56006-56c2-4178-a79d-576b7678cb73"}}, {"key": "attempt", "value": {"intValue": "1"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "outcome", "value": {"stringValue": "ready"}}], "endTimeUnixNano": "1795197735770679979", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "KCN75nI6JSk=", "startTimeUnixNano": "1795197735662214869", "status": {"code": "STATUS_CODE_OK"}, "traceId": "noOIV8u4H5FJ5CtdhfWqAA=="}, "span_id": "28237be6723a2529", "trace_id": "9e838857cbb81f9149e42b5d85f5aa00"}} +worker.retry after 0.022s +{"evidence": "worker.retry", "log": ["1795197746182868480", "job.outcome", {"attachment_id": "eafac0b1-8919-483e-95a7-da1cf59d0b01", "attempt": "1", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "f24949ee-95bc-46da-bfcc-60d194800d81", "job_type": "attachment.process.v1", "observed_timestamp": "1795197746182933273", "outcome": "retry", "reason": "storage_unavailable", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "52a381adbc68d29c", "trace_id": "4fb07dc941710c12c60f3366b8024973"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"eafac0b1-8919-483e-95a7-da1cf59d0b01\" | event=\"job.outcome\" | outcome=\"retry\"", "tempo": {"duration_ns": 3326689516, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "f24949ee-95bc-46da-bfcc-60d194800d81"}}, {"key": "attachment_id", "value": {"stringValue": "eafac0b1-8919-483e-95a7-da1cf59d0b01"}}, {"key": "attempt", "value": {"intValue": "1"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "retry"}}], "endTimeUnixNano": "1795197746185644174", "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "UqOBrbxo0pw=", "startTimeUnixNano": "1795197742858954658", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "T7B9yUFxDBLGDzNmuAJJcw=="}, "span_id": "52a381adbc68d29c", "trace_id": "4fb07dc941710c12c60f3366b8024973"}} +worker.failed after 0.022s +{"evidence": "worker.failed", "log": ["1795197771373774848", "job.outcome", {"attachment_id": "eafac0b1-8919-483e-95a7-da1cf59d0b01", "attempt": "5", "code_file_path": "/app/src/telemetry.py", "code_function_name": "event", "code_line_number": "81", "deployment_environment_name": "lab", "detected_level": "WARN", "event": "job.outcome", "flags": "1", "job_id": "f24949ee-95bc-46da-bfcc-60d194800d81", "job_type": "attachment.process.v1", "observed_timestamp": "1795197771373954823", "outcome": "failed", "reason": "retry_exhausted", "scope_name": "lab6.events", "service_instance_id": "worker", "service_name": "messenger-worker", "service_version": "1.6.0", "severity_number": "13", "severity_text": "WARN", "span_id": "38be4efed3e6cbd5", "trace_id": "33febc64757b31ee367caaadf8b840a5"}], "query": "{service_name=\"messenger-worker\"} | attachment_id=\"eafac0b1-8919-483e-95a7-da1cf59d0b01\" | event=\"job.outcome\" | outcome=\"failed\"", "tempo": {"duration_ns": 6282504900, "resource": {"deployment.environment.name": {"stringValue": "lab"}, "service.instance.id": {"stringValue": "worker"}, "service.name": {"stringValue": "messenger-worker"}, "service.version": {"stringValue": "1.6.0"}}, "service": "messenger-worker", "span": {"attributes": [{"key": "messaging.operation.type", "value": {"stringValue": "process"}}, {"key": "messaging.system", "value": {"stringValue": "rabbitmq"}}, {"key": "job_id", "value": {"stringValue": "f24949ee-95bc-46da-bfcc-60d194800d81"}}, {"key": "attachment_id", "value": {"stringValue": "eafac0b1-8919-483e-95a7-da1cf59d0b01"}}, {"key": "attempt", "value": {"intValue": "5"}}, {"key": "job_type", "value": {"stringValue": "attachment.process.v1"}}, {"key": "dependency.name", "value": {"stringValue": "s3"}}, {"key": "outcome", "value": {"stringValue": "failed"}}], "endTimeUnixNano": "1795197771374566823", "flags": 256, "kind": "SPAN_KIND_CONSUMER", "name": "attachment.process.v1", "spanId": "OL5O/tPmy9U=", "startTimeUnixNano": "1795197765092061923", "status": {"code": "STATUS_CODE_ERROR"}, "traceId": "M/68ZHV7Me42fKqt+LhApQ=="}, "span_id": "38be4efed3e6cbd5", "trace_id": "33febc64757b31ee367caaadf8b840a5"}} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_requests_total", "samples": [{"metric": {"__name__": "messenger_http_requests_total", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.865, "7"]}, {"metric": {"__name__": "messenger_http_requests_total", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.865, "2"]}, {"metric": {"__name__": "messenger_http_requests_total", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197774.865, "6"]}]} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_duration_seconds_sum", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.869, "0.1421190100001013"]}, {"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.869, "0.02769800300006864"]}, {"metric": {"__name__": "messenger_http_duration_seconds_sum", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197774.869, "19.829921962999947"]}]} +{"checked_series": 27, "evidence": "prometheus", "query": "messenger_http_duration_seconds_count", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.872, "7"]}, {"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api2", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.872, "2"]}, {"metric": {"__name__": "messenger_http_duration_seconds_count", "instance": "api1", "job": "messenger-api", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197774.872, "6"]}]} +{"checked_series": 378, "evidence": "prometheus", "query": "messenger_http_duration_seconds_bucket", "samples": [{"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.876, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.876, "7"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api2", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.876, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api2", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "2xx"}, "value": [1795197774.876, "2"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "0.005", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197774.876, "0"]}, {"metric": {"__name__": "messenger_http_duration_seconds_bucket", "instance": "api1", "job": "messenger-api", "le": "+Inf", "method": "GET", "route": "/api/v1/attachments/{attachment_id}/content", "status_class": "5xx"}, "value": [1795197774.876, "6"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_outcomes_total", "samples": [{"metric": {"__name__": "messenger_job_outcomes_total", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197774.885, "1"]}, {"metric": {"__name__": "messenger_job_outcomes_total", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197774.885, "1"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_duration_seconds_sum", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_sum", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197774.888, "0.05200310299994726"]}, {"metric": {"__name__": "messenger_job_duration_seconds_sum", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197774.888, "3.298284464999938"]}]} +{"checked_series": 2, "evidence": "prometheus", "query": "messenger_job_duration_seconds_count", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_count", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "success"}, "value": [1795197774.891, "1"]}, {"metric": {"__name__": "messenger_job_duration_seconds_count", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "outcome": "fail"}, "value": [1795197774.891, "1"]}]} +{"checked_series": 28, "evidence": "prometheus", "query": "messenger_job_duration_seconds_bucket", "samples": [{"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "0.005", "outcome": "success"}, "value": [1795197774.897, "0"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "+Inf", "outcome": "success"}, "value": [1795197774.897, "1"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "0.005", "outcome": "fail"}, "value": [1795197774.897, "0"]}, {"metric": {"__name__": "messenger_job_duration_seconds_bucket", "instance": "worker", "job": "messenger-worker", "job_type": "attachment.process.v1", "le": "+Inf", "outcome": "fail"}, "value": [1795197774.897, "1"]}]} +all real counter and histogram series after 0.040s +{"evidence": "grafana.datasource", "health": {"details": {"application": "Prometheus", "features": {"rulerApiEnabled": false}}, "message": "Successfully queried the Prometheus API.", "status": "OK"}, "type": "prometheus", "uid": "lab6-prometheus", "url": "http://prometheus:9090"} +{"evidence": "grafana.datasource", "health": {"message": "Data source successfully connected.", "status": "OK"}, "type": "loki", "uid": "lab6-loki", "url": "http://loki:3100"} +{"evidence": "grafana.datasource", "health": {"body": "ready", "http_status": 200}, "type": "tempo", "uid": "lab6-tempo", "url": "http://tempo:3200"} +{"evidence": "grafana.panel", "frames": 9, "id": 1, "max_value": 1, "numeric_samples": 162, "query": "sum by (method, route) (rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", "title": "API · запросы/с"} +{"evidence": "grafana.panel", "frames": 2, "id": 2, "max_value": 0.12727272727272726, "numeric_samples": 36, "query": "sum by (status_class) (rate(messenger_http_requests_total{status_class=~\"4xx|5xx\",route!~\"/health/.*\"}[1m]))", "title": "API · 4xx и 5xx/с"} +{"evidence": "grafana.panel", "frames": 1, "id": 3, "max_value": 0.12404683022988666, "numeric_samples": 10, "query": "sum(rate(messenger_http_duration_seconds_sum{route!~\"/health/.*\"}[1m])) / sum(rate(messenger_http_duration_seconds_count{route!~\"/health/.*\"}[1m]))", "title": "API · средняя длительность"} +{"evidence": "grafana.panel", "frames": 2, "id": 4, "max_value": 1, "numeric_samples": 36, "query": "sum by (outcome) (messenger_job_outcomes_total)", "title": "Worker · успешные и неуспешные задания"} +{"evidence": "grafana.panel", "frames": 2, "id": 5, "max_value": 3.298284464999938, "numeric_samples": 36, "query": "sum by (outcome) (messenger_job_duration_seconds_sum) / sum by (outcome) (messenger_job_duration_seconds_count)", "title": "Worker · средняя длительность"} +{"evidence": "grafana.panel", "id": 6, "query": "{service_name=~\"messenger-api|messenger-worker\"} | event != \"\"", "rows": 286, "title": "События API и worker"} +{"evidence": "grafana.trace", "span_id": "12e525cf65db522c", "trace_id": "6da5fd7eed430a7e1312e9c747d67a61"} +{"evidence": "grafana.trace", "span_id": "28237be6723a2529", "trace_id": "9e838857cbb81f9149e42b5d85f5aa00"} +{"evidence": "telemetry.scan", "loki_rows": 286, "matches": 0, "metric_samples": 493, "spans_queries": 8} +docker restart prometheus loki tempo grafana containers=0d91c35f7d66,7ee4dcfdd2ee,413ec4b05acc,2c03a3094afe +same samples, logs, spans and dashboard after restart after 7.291s +{"after": {"grafana": "2026-11-20T18:03:01.470866436Z", "loki": "2026-11-20T18:02:58.136864274Z", "prometheus": "2026-11-20T18:02:57.024260103Z", "tempo": "2026-11-20T18:03:00.718016519Z"}, "before": {"grafana": "2026-11-20T17:54:38.151708123Z", "loki": "2026-11-20T17:54:36.557683052Z", "prometheus": "2026-11-20T17:54:35.32262272Z", "tempo": "2026-11-20T17:54:37.383377841Z"}, "dashboard": "lab6-messenger", "evidence": "storage.restart", "historical_query": "sum(messenger_job_outcomes_total{outcome=\"success\"})", "historical_samples": [{"metric": {}, "value": [1795197776.4, "1"]}], "trace_ids": ["6da5fd7eed430a7e1312e9c747d67a61", "9e838857cbb81f9149e42b5d85f5aa00"], "unix_time": 1795197776.399914, "volumes": {"grafana": "mtusi-lab6-grade3_grafana_data", "loki": "mtusi-lab6-grade3_loki_data", "prometheus": "mtusi-lab6-grade3_prometheus_data", "tempo": "mtusi-lab6-grade3_tempo_data"}} +docker stop otel-collector grafana containers=94da2240abad,2c03a3094afe +ready id=dab091ea-16d7-4854-83d2-2bfd2771007e wait=0.365s original_sha256=a8c48237481dcfa2b7374064586559f3e7bcdf9c99c5a8f570a72971facf19a8 jpeg=256x256 thumb_sha256=c33e884cadb4a5ae24d1069d30e18f72ee9c4cee084c618b5ce1afc8a76b838f read=api1/api2 +{"authenticated_get": 200, "disabled": ["otel-collector", "grafana"], "evidence": "telemetry.outage", "health": 200, "state": "ready", "uploaded": 202} +docker start otel-collector grafana containers=94da2240abad,2c03a3094afe +PASS: real metrics/logs/traces, provisioned Grafana, incidents and persistent telemetry diff --git a/evidence/grade3/offline.txt b/evidence/grade3/offline.txt new file mode 100644 index 0000000..23ae9c0 --- /dev/null +++ b/evidence/grade3/offline.txt @@ -0,0 +1,10 @@ + Network mtusi-lab6-grade3_default Creating + Network mtusi-lab6-grade3_default Creating + Network mtusi-lab6-grade3_default Created + Network mtusi-lab6-grade3_default Created + Container mtusi-lab6-grade3-tools-run-d03addc0156b Creating + Container mtusi-lab6-grade3-tools-run-d03addc0156b Created +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/grade3/persistence-after.txt b/evidence/grade3/persistence-after.txt new file mode 100644 index 0000000..19986f7 --- /dev/null +++ b/evidence/grade3/persistence-after.txt @@ -0,0 +1,12 @@ +/mtusi-lab6-grade3-api1-1 6ecf4f039efd02c3342529e18bf27cc1a2042aed5adf58213894b61698f3ad24 2026-11-20T17:54:24.352271034Z +/mtusi-lab6-grade3-api2-1 255f4f7f057924dee07f6b98496295fb3e402b215723555b532ef0755ba7aea2 2026-11-20T17:54:25.530271862Z +/mtusi-lab6-grade3-broker-1 3f09ee76622f70227b431759aba022da0d870ba31462a8113fd716da7d802213 2026-11-20T17:54:23.1266914Z +/mtusi-lab6-grade3-grafana-1 2c03a3094afe3eae9fd60ed749d73968769c8221b834b950428d2c5bc36cc094 2026-11-20T17:54:38.151708123Z +/mtusi-lab6-grade3-lb-1 6cde871f4dbc48b5b140158773d13ea389d5ec695133d796fac3635658e5c4b2 2026-11-20T17:54:29.143745354Z +/mtusi-lab6-grade3-loki-1 7ee4dcfdd2ee7b8053aeb75923d60e20b525fefcd59e696ecb159784cb883f02 2026-11-20T17:54:36.557683052Z +/mtusi-lab6-grade3-otel-collector-1 94da2240abad0a234cb209d5eabcc36a9d0af94226e1739d4fa2776661727bd0 2026-11-20T17:54:41.619530495Z +/mtusi-lab6-grade3-postgres-1 27797c076e8ca5dcc1c534ad8e094d8eaaa59c28caa78c1ea35cf5b1b005d5e3 2026-11-20T17:54:19.171591366Z +/mtusi-lab6-grade3-prometheus-1 0d91c35f7d66fdbdb94f8ef9f9ff050700b973c15bcb03ab2d05861e017671ca 2026-11-20T17:54:35.32262272Z +/mtusi-lab6-grade3-s3-1 28a42a667d04cb14b80e3c9ab0b81c8535cd48354129e5c6c9e1ae2bbbb97678 2026-11-20T17:54:21.243563294Z +/mtusi-lab6-grade3-tempo-1 413ec4b05acc23db26eabc8b51e972c5d8a04259ef408961a27f0c444fef1892 2026-11-20T17:54:37.383377841Z +/mtusi-lab6-grade3-worker-1 ed7c1dede058c1fd08d0c032e29fde927927efc854489ab3caf1f9807d18198e 2026-11-20T17:54:33.888642366Z diff --git a/evidence/grade3/persistence-before.txt b/evidence/grade3/persistence-before.txt new file mode 100644 index 0000000..6723215 --- /dev/null +++ b/evidence/grade3/persistence-before.txt @@ -0,0 +1,12 @@ +/mtusi-lab6-grade3-api1-1 6ecf4f039efd02c3342529e18bf27cc1a2042aed5adf58213894b61698f3ad24 2026-11-20T17:51:46.754219716Z +/mtusi-lab6-grade3-api2-1 255f4f7f057924dee07f6b98496295fb3e402b215723555b532ef0755ba7aea2 2026-11-20T17:51:46.332861172Z +/mtusi-lab6-grade3-broker-1 3f09ee76622f70227b431759aba022da0d870ba31462a8113fd716da7d802213 2026-11-20T17:51:28.283219185Z +/mtusi-lab6-grade3-grafana-1 2c03a3094afe3eae9fd60ed749d73968769c8221b834b950428d2c5bc36cc094 2026-11-20T17:51:29.301540158Z +/mtusi-lab6-grade3-lb-1 6cde871f4dbc48b5b140158773d13ea389d5ec695133d796fac3635658e5c4b2 2026-11-20T17:51:30.399081536Z +/mtusi-lab6-grade3-loki-1 7ee4dcfdd2ee7b8053aeb75923d60e20b525fefcd59e696ecb159784cb883f02 2026-11-20T17:51:31.621311624Z +/mtusi-lab6-grade3-otel-collector-1 94da2240abad0a234cb209d5eabcc36a9d0af94226e1739d4fa2776661727bd0 2026-11-20T17:51:32.312936473Z +/mtusi-lab6-grade3-postgres-1 27797c076e8ca5dcc1c534ad8e094d8eaaa59c28caa78c1ea35cf5b1b005d5e3 2026-11-20T17:51:32.883020114Z +/mtusi-lab6-grade3-prometheus-1 0d91c35f7d66fdbdb94f8ef9f9ff050700b973c15bcb03ab2d05861e017671ca 2026-11-20T17:51:35.224231081Z +/mtusi-lab6-grade3-s3-1 28a42a667d04cb14b80e3c9ab0b81c8535cd48354129e5c6c9e1ae2bbbb97678 2026-11-20T17:51:34.35986042Z +/mtusi-lab6-grade3-tempo-1 413ec4b05acc23db26eabc8b51e972c5d8a04259ef408961a27f0c444fef1892 2026-11-20T17:51:33.607690566Z +/mtusi-lab6-grade3-worker-1 ed7c1dede058c1fd08d0c032e29fde927927efc854489ab3caf1f9807d18198e 2026-11-20T17:51:55.088842893Z diff --git a/evidence/grade3/persistence.txt b/evidence/grade3/persistence.txt new file mode 100644 index 0000000..64824dc --- /dev/null +++ b/evidence/grade3/persistence.txt @@ -0,0 +1,5 @@ + Container mtusi-lab6-grade3-tools-run-5df095d3848b Creating + Container mtusi-lab6-grade3-tools-run-5df095d3848b Created +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/grade3/runtime-validation.txt b/evidence/grade3/runtime-validation.txt new file mode 100644 index 0000000..0bdc57b --- /dev/null +++ b/evidence/grade3/runtime-validation.txt @@ -0,0 +1,10 @@ +docker compose exec -T otel-collector /otelcol-contrib validate --config=/etc/otelcol-contrib/config.yaml +PASS: Collector validate exit 0 +docker compose exec -T api1 python -m pip check +No broken requirements found. +Tools permissions and versions +755 scripts/restart.sh +755 scripts/stop-one.sh +755 scripts/lab-containers.sh +Python 3.12.16 +GNU Make 4.4.1 diff --git a/evidence/grade3/smoke.txt b/evidence/grade3/smoke.txt new file mode 100644 index 0000000..f09007d --- /dev/null +++ b/evidence/grade3/smoke.txt @@ -0,0 +1,4 @@ + Container mtusi-lab6-grade3-tools-run-399492cb8e9a Creating + Container mtusi-lab6-grade3-tools-run-399492cb8e9a Created +python3 tests/smoke.py --base-url "http://lb:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/grade3/startup.txt b/evidence/grade3/startup.txt new file mode 100644 index 0000000..c2ee193 --- /dev/null +++ b/evidence/grade3/startup.txt @@ -0,0 +1,122 @@ + Volume mtusi-lab6-grade3_s3_data Creating + Volume mtusi-lab6-grade3_s3_data Creating + Volume mtusi-lab6-grade3_grafana_data Creating + Volume mtusi-lab6-grade3_grafana_data Creating + Volume mtusi-lab6-grade3_tempo_data Creating + Volume mtusi-lab6-grade3_tempo_data Creating + Container mtusi-lab6-grade3-lb-1 Creating + Volume mtusi-lab6-grade3_broker_data Creating + Volume mtusi-lab6-grade3_broker_data Creating + Volume mtusi-lab6-grade3_loki_data Creating + Volume mtusi-lab6-grade3_loki_data Creating + Container mtusi-lab6-grade3-otel-collector-1 Creating + Volume mtusi-lab6-grade3_prometheus_data Creating + Volume mtusi-lab6-grade3_prometheus_data Creating + Volume mtusi-lab6-grade3_postgres_data Creating + Volume mtusi-lab6-grade3_postgres_data Creating + Volume mtusi-lab6-grade3_grafana_data Created + Volume mtusi-lab6-grade3_grafana_data Created + Container mtusi-lab6-grade3-grafana-1 Creating + Volume mtusi-lab6-grade3_tempo_data Created + Volume mtusi-lab6-grade3_tempo_data Created + Container mtusi-lab6-grade3-tempo-1 Creating + Volume mtusi-lab6-grade3_postgres_data Created + Volume mtusi-lab6-grade3_postgres_data Created + Container mtusi-lab6-grade3-postgres-1 Creating + Volume mtusi-lab6-grade3_broker_data Created + Volume mtusi-lab6-grade3_broker_data Created + Container mtusi-lab6-grade3-broker-1 Creating + Volume mtusi-lab6-grade3_s3_data Created + Volume mtusi-lab6-grade3_s3_data Created + Container mtusi-lab6-grade3-s3-1 Creating + Volume mtusi-lab6-grade3_prometheus_data Created + Volume mtusi-lab6-grade3_prometheus_data Created + Container mtusi-lab6-grade3-prometheus-1 Creating + Volume mtusi-lab6-grade3_loki_data Created + Volume mtusi-lab6-grade3_loki_data Created + Container mtusi-lab6-grade3-loki-1 Creating + Container mtusi-lab6-grade3-otel-collector-1 Created + Container mtusi-lab6-grade3-tempo-1 Created + Container mtusi-lab6-grade3-grafana-1 Created + Container mtusi-lab6-grade3-prometheus-1 Created + Container mtusi-lab6-grade3-loki-1 Created + Container mtusi-lab6-grade3-lb-1 Created + Container mtusi-lab6-grade3-s3-1 Created + Container mtusi-lab6-grade3-bootstrap-1 Creating + Container mtusi-lab6-grade3-broker-1 Created + Container mtusi-lab6-grade3-postgres-1 Created + Container mtusi-lab6-grade3-migrate-1 Creating + Container mtusi-lab6-grade3-migrate-1 Created + Container mtusi-lab6-grade3-bootstrap-1 Created + Container mtusi-lab6-grade3-worker-1 Creating + Container mtusi-lab6-grade3-api1-1 Creating + Container mtusi-lab6-grade3-api2-1 Creating + Container mtusi-lab6-grade3-api2-1 Created + Container mtusi-lab6-grade3-api1-1 Created + Container mtusi-lab6-grade3-worker-1 Created + Container mtusi-lab6-grade3-broker-1 Starting + Container mtusi-lab6-grade3-grafana-1 Starting + Container mtusi-lab6-grade3-lb-1 Starting + Container mtusi-lab6-grade3-loki-1 Starting + Container mtusi-lab6-grade3-otel-collector-1 Starting + Container mtusi-lab6-grade3-postgres-1 Starting + Container mtusi-lab6-grade3-tempo-1 Starting + Container mtusi-lab6-grade3-s3-1 Starting + Container mtusi-lab6-grade3-prometheus-1 Starting + Container mtusi-lab6-grade3-broker-1 Started + Container mtusi-lab6-grade3-broker-1 Waiting + Container mtusi-lab6-grade3-grafana-1 Started + Container mtusi-lab6-grade3-lb-1 Started + Container mtusi-lab6-grade3-loki-1 Started + Container mtusi-lab6-grade3-otel-collector-1 Started + Container mtusi-lab6-grade3-postgres-1 Started + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-tempo-1 Started + Container mtusi-lab6-grade3-s3-1 Started + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-prometheus-1 Started + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-bootstrap-1 Starting + Container mtusi-lab6-grade3-bootstrap-1 Started + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-migrate-1 Starting + Container mtusi-lab6-grade3-migrate-1 Started + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-api2-1 Starting + Container mtusi-lab6-grade3-api1-1 Starting + Container mtusi-lab6-grade3-api2-1 Started + Container mtusi-lab6-grade3-api1-1 Started + Container mtusi-lab6-grade3-broker-1 Healthy + Container mtusi-lab6-grade3-worker-1 Starting + Container mtusi-lab6-grade3-worker-1 Started + Container mtusi-lab6-grade3-api1-1 Waiting + Container mtusi-lab6-grade3-grafana-1 Waiting + Container mtusi-lab6-grade3-broker-1 Waiting + Container mtusi-lab6-grade3-api2-1 Waiting + Container mtusi-lab6-grade3-prometheus-1 Waiting + Container mtusi-lab6-grade3-loki-1 Waiting + Container mtusi-lab6-grade3-bootstrap-1 Waiting + Container mtusi-lab6-grade3-tempo-1 Waiting + Container mtusi-lab6-grade3-postgres-1 Waiting + Container mtusi-lab6-grade3-otel-collector-1 Waiting + Container mtusi-lab6-grade3-s3-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Waiting + Container mtusi-lab6-grade3-worker-1 Waiting + Container mtusi-lab6-grade3-lb-1 Waiting + Container mtusi-lab6-grade3-migrate-1 Exited + Container mtusi-lab6-grade3-bootstrap-1 Exited + Container mtusi-lab6-grade3-prometheus-1 Healthy + Container mtusi-lab6-grade3-api1-1 Healthy + Container mtusi-lab6-grade3-api2-1 Healthy + Container mtusi-lab6-grade3-postgres-1 Healthy + Container mtusi-lab6-grade3-lb-1 Healthy + Container mtusi-lab6-grade3-s3-1 Healthy + Container mtusi-lab6-grade3-worker-1 Healthy + Container mtusi-lab6-grade3-tempo-1 Healthy + Container mtusi-lab6-grade3-loki-1 Healthy + Container mtusi-lab6-grade3-otel-collector-1 Healthy + Container mtusi-lab6-grade3-broker-1 Healthy + Container mtusi-lab6-grade3-grafana-1 Healthy diff --git a/infra/Dockerfile.tools b/infra/Dockerfile.tools new file mode 100644 index 0000000..a14a0b4 --- /dev/null +++ b/infra/Dockerfile.tools @@ -0,0 +1,15 @@ +FROM docker:27.5.1-cli +RUN apk add --no-cache python3 py3-pip make && python3 -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /workspace +COPY infra/requirements-tools.lock /tmp/requirements-tools.lock +RUN pip install --no-cache-dir -r /tmp/requirements-tools.lock +COPY Makefile lab.json README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +COPY contracts/ ./contracts/ +COPY tests/ ./tests/ +COPY scripts/check_contract.py scripts/validate_openapi.py ./scripts/ +COPY --chmod=0755 scripts/restart.sh scripts/stop-one.sh scripts/lab-containers.sh ./scripts/ +RUN mkdir -p /workspace/evidence +COPY student-tests/ ./student-tests/ +ENTRYPOINT [] +CMD ["make", "check"] diff --git a/infra/grafana/dashboards/lab6.json b/infra/grafana/dashboards/lab6.json new file mode 100644 index 0000000..4bf0add --- /dev/null +++ b/infra/grafana/dashboards/lab6.json @@ -0,0 +1,176 @@ +{ + "title": "ЛР6 · Messenger", + "uid": "lab6-messenger", + "schemaVersion": 39, + "version": 1, + "editable": false, + "time": { + "from": "now-15m", + "to": "now" + }, + "refresh": "10s", + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "API · запросы/с", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method, route) (rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))" + } + ] + }, + { + "id": 2, + "title": "API · 4xx и 5xx/с", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status_class) (rate(messenger_http_requests_total{status_class=~\"4xx|5xx\",route!~\"/health/.*\"}[1m]))" + } + ] + }, + { + "id": 3, + "title": "API · средняя длительность", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_duration_seconds_sum{route!~\"/health/.*\"}[1m])) / sum(rate(messenger_http_duration_seconds_count{route!~\"/health/.*\"}[1m]))" + } + ] + }, + { + "id": 4, + "title": "Worker · успешные и неуспешные задания", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (messenger_job_outcomes_total)" + } + ] + }, + { + "id": 5, + "title": "Worker · средняя длительность", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "lab6-prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (messenger_job_duration_seconds_sum) / sum by (outcome) (messenger_job_duration_seconds_count)" + } + ] + }, + { + "id": 6, + "title": "События API и worker", + "type": "logs", + "gridPos": { + "x": 0, + "y": 24, + "w": 24, + "h": 10 + }, + "datasource": { + "type": "loki", + "uid": "lab6-loki" + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=~\"messenger-api|messenger-worker\"} | event != \"\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "enableLogDetails": true + } + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/lab6.yaml b/infra/grafana/provisioning/dashboards/lab6.yaml new file mode 100644 index 0000000..73c369a --- /dev/null +++ b/infra/grafana/provisioning/dashboards/lab6.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: lab6 + type: file + disableDeletion: true + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/lab6.yaml b/infra/grafana/provisioning/datasources/lab6.yaml new file mode 100644 index 0000000..4e3a022 --- /dev/null +++ b/infra/grafana/provisioning/datasources/lab6.yaml @@ -0,0 +1,30 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: lab6-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 5s + - name: Loki + uid: lab6-loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: Trace + matcherType: label + matcherRegex: trace_id + datasourceUid: lab6-tempo + url: '$${__value.raw}' + - name: Tempo + uid: lab6-tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..d078402 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,29 @@ +defaults + mode http + timeout http-request 5s + timeout connect 500ms + timeout client 15s + timeout server 10s + timeout check 1s + retries 1 + retry-on conn-failure + option redispatch + +resolvers docker + nameserver embedded 127.0.0.11:53 + timeout resolve 500ms + timeout retry 500ms + hold valid 1s + hold nx 1s + +frontend public_http + bind :8080 + default_backend messenger + +backend messenger + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 1 resolvers docker init-addr none resolve-prefer ipv4 + server api1 api1:8080 + server api2 api2:8080 diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e2db750 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,51 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 256 + send_batch_max_size: 512 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 5m + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 3s + sending_queue: {queue_size: 32, num_consumers: 2} + retry_on_failure: {initial_interval: 1s, max_interval: 5s, max_elapsed_time: 20s} + otlp/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + timeout: 3s + sending_queue: {queue_size: 32, num_consumers: 2} + retry_on_failure: {initial_interval: 1s, max_interval: 5s, max_elapsed_time: 20s} + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..2afd104 --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,39 @@ +auth_enabled: false +analytics: + reporting_enabled: false +server: + http_listen_port: 3100 + log_level: warn +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: {store: inmemory} + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: {prefix: index_, period: 24h} +ingester: + wal: + dir: /loki/wal + flush_on_shutdown: true +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +limits_config: + allow_structured_metadata: true + retention_period: 48h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name] diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..ce88177 --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,8 @@ +global: + scrape_interval: 5s + scrape_timeout: 3s +scrape_configs: + - job_name: otel-collector + honor_labels: true + static_configs: + - targets: ['otel-collector:8889'] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..ef5f0c9 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 3200 + log_level: warn +usage_report: + reporting_enabled: false +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +ingester: + max_block_duration: 30s + flush_all_on_shutdown: true +compactor: + compaction: + block_retention: 48h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/infra/requirements-tools.lock b/infra/requirements-tools.lock new file mode 100644 index 0000000..a35a4e5 --- /dev/null +++ b/infra/requirements-tools.lock @@ -0,0 +1,20 @@ +attrs==26.1.0 +certifi==2026.10.23 +charset-normalizer==3.5.2 +idna==3.20 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +pathable==0.4.4 +PyYAML==6.0.3 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +typing_extensions==4.16.0 +urllib3==2.8.0 +Pillow==12.2.0 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..5e51fed --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> ''), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user ON chat_members(user_id, chat_id); +CREATE INDEX chats_order ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK (expires_at > created_at) +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/migrations/002_message_idempotency.sql b/migrations/002_message_idempotency.sql new file mode 100644 index 0000000..f077adb --- /dev/null +++ b/migrations/002_message_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + method text NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'), + message_id uuid NOT NULL REFERENCES messages(id) DEFERRABLE INITIALLY DEFERRED, + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, key) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..47cf204 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,35 @@ +-- Preserve existing message rows and stored idempotency keys. +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD UNIQUE (chat_id, id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255 + AND strpos(filename, '/') = 0 AND strpos(filename, chr(92)) = 0 + AND filename !~ '[[:cntrl:]]'), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 COLLATE "C" ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT transaction_timestamp(), + updated_at timestamptz NOT NULL DEFAULT transaction_timestamp(), + CHECK ((status = 'ready') = (error_code IS NULL)), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id), + UNIQUE (chat_id, id) +); + +CREATE TABLE message_attachments ( + chat_id uuid NOT NULL, + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position integer NOT NULL CHECK (position BETWEEN 1 AND 10), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + FOREIGN KEY (chat_id, message_id) REFERENCES messages(chat_id, id) ON DELETE CASCADE, + FOREIGN KEY (chat_id, attachment_id) REFERENCES attachments(chat_id, id) +); diff --git a/migrations/004_attachment_jobs.sql b/migrations/004_attachment_jobs.sql new file mode 100644 index 0000000..8dd3009 --- /dev/null +++ b/migrations/004_attachment_jobs.sql @@ -0,0 +1,32 @@ +-- Existing Lab 4 rows keep their original keys, metadata and message links. +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments DROP CONSTRAINT attachments_check; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +ALTER TABLE attachments ADD CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD CHECK ((status = 'failed') = (error_code IS NOT NULL)); + +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK (pipeline_version = 1), + staging_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + attempt integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 20), + retry_at timestamptz NOT NULL DEFAULT now(), + last_error text, + CHECK (attempt BETWEEN 1 AND max_attempts) +); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK (pipeline_version = 1), + name text NOT NULL CHECK (name = 'thumbnail'), + content_type text NOT NULL CHECK (content_type = 'image/jpeg'), + size_bytes integer NOT NULL CHECK (size_bytes > 0), + sha256 text NOT NULL CHECK (sha256 COLLATE "C" ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width BETWEEN 1 AND 256), + height integer NOT NULL CHECK (height BETWEEN 1 AND 256), + object_key text NOT NULL UNIQUE, + PRIMARY KEY (attachment_id, pipeline_version, name) +); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..48d28bf --- /dev/null +++ b/requirements.lock @@ -0,0 +1,41 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +idna==3.20 +psycopg==3.2.6 +psycopg-binary==3.2.6 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.3 +pydantic_core==2.33.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.34.2 +boto3==1.40.45 +botocore==1.40.45 +jmespath==1.0.1 +Pillow==12.2.0 +python-dateutil==2.9.0.post0 +s3transfer==0.14.0 +six==1.17.0 +urllib3==2.6.3 +pika==1.3.2 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +opentelemetry-exporter-otlp-proto-http==1.34.1 +opentelemetry-exporter-otlp-proto-common==1.34.1 +opentelemetry-proto==1.34.1 +protobuf==5.29.6 +googleapis-common-protos==1.75.0 +importlib-metadata==8.7.1 +zipp==4.1.2 +requests==2.34.2 +certifi==2026.10.23 +charset-normalizer==3.5.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1728dc5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.12 +starlette==0.46.2 +pydantic==2.11.3 +uvicorn==0.34.2 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +argon2-cffi==23.1.0 +boto3==1.40.45 +Pillow==12.2.0 +pika==1.3.2 +opentelemetry-sdk==1.34.1 +opentelemetry-exporter-otlp-proto-http==1.34.1 diff --git a/scripts/init-env.ps1 b/scripts/init-env.ps1 new file mode 100644 index 0000000..2018e4d --- /dev/null +++ b/scripts/init-env.ps1 @@ -0,0 +1,79 @@ +$ErrorActionPreference = 'Stop' +$root = Split-Path -Parent $PSScriptRoot +$utf8 = [Text.UTF8Encoding]::new($false) +$secretDir = Join-Path $root 'secrets' +[void][IO.Directory]::CreateDirectory($secretDir) + +# Compose bind-mounts these files via Docker Desktop. Keep the host ACL private. +$sid = [Security.Principal.WindowsIdentity]::GetCurrent().User +if (Test-Path -LiteralPath $secretDir -PathType Container) { + $security = [Security.AccessControl.DirectorySecurity]::new() + $security.SetOwner($sid) + $security.SetAccessRuleProtection($true, $false) + $inheritance = [Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit' + foreach ($principal in @($sid, [Security.Principal.SecurityIdentifier]::new('S-1-5-18'))) { + $rule = [Security.AccessControl.FileSystemAccessRule]::new( + $principal, 'FullControl', $inheritance, 'None', 'Allow' + ) + $security.AddAccessRule($rule) + } + $current = Get-Acl -LiteralPath $secretDir + $identityType = [Security.Principal.SecurityIdentifier] + $principals = @($sid.Value, 'S-1-5-18') | Select-Object -Unique + $rules = @($current.GetAccessRules($true, $true, $identityType)) + $matching = @($rules | Where-Object { + $_.IdentityReference.Value -in $principals -and + $_.FileSystemRights -eq 'FullControl' -and $_.AccessControlType -eq 'Allow' -and + $_.InheritanceFlags -eq $inheritance -and $_.PropagationFlags -eq 'None' -and + -not $_.IsInherited + }) + $owner = $current.GetOwner($identityType) + if (-not $current.AreAccessRulesProtected -or $owner.Value -ne $sid.Value -or + $rules.Count -ne $principals.Count -or $matching.Count -ne $principals.Count -or + @($matching.IdentityReference.Value | Select-Object -Unique).Count -ne $principals.Count) { + Set-Acl -LiteralPath $secretDir -AclObject $security + } +} + +function New-HexSecret([string]$name, [int]$length) { + $file = Join-Path $secretDir $name + if (-not (Test-Path -LiteralPath $file)) { + $bytes = New-Object byte[] $length + $rng = [Security.Cryptography.RandomNumberGenerator]::Create() + try { $rng.GetBytes($bytes) } finally { $rng.Dispose() } + $value = [BitConverter]::ToString($bytes).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllText($file, $value, $utf8) + } +} +New-HexSecret 'broker_password' 32 +$brokerPassword = [IO.File]::ReadAllText((Join-Path $secretDir 'broker_password')).Trim() +if ($brokerPassword -cnotmatch '^[a-f0-9]{64}$') { throw 'Invalid local RabbitMQ secret format.' } +$brokerUrl = Join-Path $secretDir 'broker_url' +$brokerExpected = "amqp://messenger:$brokerPassword@broker:5672/%2F" +if (-not (Test-Path -LiteralPath $brokerUrl)) { + [IO.File]::WriteAllText($brokerUrl, $brokerExpected, $utf8) +} +if ([IO.File]::ReadAllText($brokerUrl).Trim() -cne $brokerExpected) { + throw 'Existing broker_url and broker_password disagree.' +} +New-HexSecret 'grafana_admin_password' 24 +New-HexSecret 'postgres_password' 32 +New-HexSecret 's3_access_key_id' 10 +New-HexSecret 's3_secret_access_key' 24 + +$password = [IO.File]::ReadAllText((Join-Path $secretDir 'postgres_password')).Trim() +if ($password -cnotmatch '^[a-f0-9]{64}$') { throw 'Invalid local PostgreSQL secret format.' } +$url = Join-Path $secretDir 'database_url' +$expected = "postgresql://messenger:${password}@postgres:5432/messenger" +if (-not (Test-Path -LiteralPath $url)) { + [IO.File]::WriteAllText($url, $expected, $utf8) +} +if ([IO.File]::ReadAllText($url).Trim() -cne $expected) { + throw 'Existing database_url and postgres_password disagree.' +} + +$envFile = Join-Path $root '.env' +if (-not (Test-Path -LiteralPath $envFile)) { + Copy-Item -LiteralPath (Join-Path $root '.env.example') -Destination $envFile +} +Write-Output 'Lab 6 local configuration ready.' diff --git a/scripts/lab-containers.sh b/scripts/lab-containers.sh new file mode 100755 index 0000000..c777a7e --- /dev/null +++ b/scripts/lab-containers.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Docker socket actions are scoped to the explicitly named course stack. +case "${LAB6_PROJECT:-}" in + mtusi-lab6-grade3|mtusi-lab6-grade3-*) ;; + *) echo 'LAB6_PROJECT must identify the Lab 6 test stack' >&2; exit 2 ;; +esac + +container() { + ids=$(docker ps -aq --filter "label=com.docker.compose.project=$LAB6_PROJECT" \ + --filter "label=com.docker.compose.service=$1" \ + --filter 'label=com.docker.compose.oneoff=False') + set -- $ids + if [ "$#" -ne 1 ]; then + echo 'Expected one Lab 6 service container' >&2 + return 2 + fi + printf '%s\n' "$1" +} diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..34a2e65 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +. "$(dirname "$0")/lab-containers.sh" +broker=$(container broker) +docker stop --time 3 "$(container worker)" +docker restart --time 4 "$(container postgres)" "$(container s3)" "$broker" +docker restart --time 4 "$(container api1)" "$(container api2)" "$(container lb)" + +# API readiness covers PostgreSQL and S3. Let AMQP recover before returning. +remaining=55 +while [ "$(docker inspect --format '{{.State.Health.Status}}' "$broker")" != healthy ]; do + remaining=$((remaining - 1)) + if [ "$remaining" -le 0 ]; then + echo 'RabbitMQ did not recover after restart' >&2 + exit 1 + fi + sleep 1 +done +docker start "$(container worker)" +docker restart --time 10 "$(container prometheus)" "$(container loki)" "$(container tempo)" "$(container grafana)" "$(container otel-collector)" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..a0fd713 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +case "${TARGET_INSTANCE_ID:-}" in + api1|api2) ;; + *) echo 'TARGET_INSTANCE_ID must be api1 or api2' >&2; exit 2 ;; +esac +. "$(dirname "$0")/lab-containers.sh" +id=$(container "$TARGET_INSTANCE_ID") +docker stop --time 4 "$id" diff --git a/scripts/test-failover-both.ps1 b/scripts/test-failover-both.ps1 new file mode 100644 index 0000000..fd1b9fa --- /dev/null +++ b/scripts/test-failover-both.ps1 @@ -0,0 +1,39 @@ +$ErrorActionPreference = 'Stop' +Push-Location (Split-Path -Parent $PSScriptRoot) +try { + $project = 'mtusi-lab6-grade3' + $api = @('api1', 'api2') + $tested = @{} + for ($attempt = 1; $attempt -le 8 -and $tested.Count -lt 2; $attempt++) { + Write-Output "Public failover attempt $attempt" + docker compose -p $project start --wait --wait-timeout 60 api1 api2 + if ($LASTEXITCODE) { throw 'Could not restore both replicas.' } + # An odd number of GETs changes the initial round-robin position + # on even attempts. The unmodified public test still chooses the target. + if ($attempt % 2 -eq 0) { + docker compose -p $project exec -T lb wget -q -O /dev/null http://127.0.0.1:8080/health/live + if ($LASTEXITCODE) { throw 'HAProxy is not ready.' } + } + try { + docker compose -p $project run --rm --no-deps tools make test-failover BASE_URL=http://lb:8080 ACTION_SCRIPT=scripts/stop-one.sh + if ($LASTEXITCODE) { throw 'Public failover test failed.' } + $stopped = @(docker compose -p $project ps --status exited --services api1 api2) + if ($LASTEXITCODE -or $stopped.Count -ne 1 -or $stopped[0] -notin $api) { + throw 'Expected exactly one stopped API replica.' + } + $running = @(docker compose -p $project ps --status running --services api1 api2) + if ($LASTEXITCODE -or $running.Count -ne 1 -or $running[0] -notin $api) { + throw 'Expected exactly one surviving API replica.' + } + $tested[$stopped[0]] = $true + Write-Output "Verified stopped=$($stopped[0]) survivor=$($running[0])" + } finally { + docker compose -p $project start --wait --wait-timeout 60 api1 api2 + if ($LASTEXITCODE) { throw 'Could not restore both replicas after failover.' } + } + } + if ($tested.Count -ne 2) { throw 'The public test did not select both API replicas.' } + Write-Output 'PASS: public failover for api1 and api2, both replicas recovered.' +} finally { + Pop-Location +} diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..faf8ee7 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,165 @@ +import hashlib +import re +import unicodedata +from io import BytesIO +from uuid import uuid4 + +import pika +from PIL import Image + +from src import jobs, s3 +from src.db import connection +from src.errors import ApiError +from src.store import public + + +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +METADATA = """id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, error_code, created_at, updated_at""" + + +def oversized(): + return ApiError(413, "payload_too_large", "Превышен допустимый размер изображения") + + +def invalid_image(): + return ApiError(415, "unsupported_media_type", "Ожидается корректный PNG или JPEG") + + +def validate_filename(filename): + if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + + +async def read_upload(request): + lengths = request.headers.getlist("content-length") + if lengths: + if len(lengths) != 1 or not re.fullmatch(r"[0-9]+", lengths[0]): + raise ApiError(400, "invalid_request", "Некорректный Content-Length") + length = lengths[0].lstrip("0") or "0" + if len(length) > len(str(MAX_UPLOAD_BYTES)) or int(length) > MAX_UPLOAD_BYTES: + raise oversized() + + body = BytesIO() + async for chunk in request.stream(): + if len(chunk) > MAX_UPLOAD_BYTES - body.tell(): + raise oversized() + body.write(chunk) + body.seek(0) + return body + + +def validate_image(body, content_type): + try: + if content_type == "image/png": + # Pillow's verify stops at the IEND chunk before reading its CRC. + with body.getbuffer() as data: + if data[-12:] != b"\x00\x00\x00\x00IEND\xae\x42\x60\x82": + raise invalid_image() + with Image.open(body, formats=(FORMATS[content_type],)) as image: + if image.width * image.height > MAX_IMAGE_PIXELS: + raise oversized() + image.verify() + with Image.open(body, formats=(FORMATS[content_type],)) as image: + # Check each frame of a PNG while bounding the total decode work. + for frame in range(getattr(image, "n_frames", 1)): + if (frame + 1) * image.width * image.height > MAX_IMAGE_PIXELS: + raise oversized() + image.seek(frame) + image.load() + except Image.DecompressionBombError: + raise oversized() from None + except (OSError, ValueError, SyntaxError, IndexError, EOFError): + raise invalid_image() from None + finally: + body.seek(0) + + +def upload(chat_id, owner_id, filename, content_type, body): + validate_image(body, content_type) + with body.getbuffer() as data: + digest = hashlib.sha256(data).hexdigest() + size = len(data) + attachment_id = uuid4() + job_id = uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"originals/{attachment_id}" + s3.put(staging_key, body, size, content_type) + + # Both rows commit before publish, so a fast consumer can always find them. + with connection() as conn: + row = conn.execute( + f"""INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING {METADATA}""", + (attachment_id, chat_id, owner_id, filename, content_type, size, digest, object_key), + ).fetchone() + conn.execute( + """INSERT INTO attachment_jobs + (job_id,attachment_id,pipeline_version,staging_key,created_at,max_attempts) + VALUES (%s,%s,%s,%s,%s,%s)""", + (job_id, attachment_id, jobs.PIPELINE_VERSION, staging_key, row["created_at"], + jobs.MAX_ATTEMPTS), + ) + try: + with jobs.broker() as channel: + jobs.publish(channel, jobs.carrier(job_id, attachment_id, row["created_at"])) + except (pika.exceptions.AMQPError, OSError): + # The outcome can be ambiguous. Keep staging and metadata for a consumer + # that already has the message. Never return 202 without a confirm. + raise ApiError(503, "unavailable", "Очередь вложений временно недоступна") from None + + return public({**row, "variants": []}) + + +VARIANT_METADATA = "name, content_type, size_bytes, sha256, width, height" + + +def get(attachment_id, user_id): + with connection() as conn: + row = visible(conn, attachment_id, user_id) + variants = [] + if row["status"] == "ready": + variants = conn.execute( + f"""SELECT {VARIANT_METADATA} FROM attachment_variants + WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name""", + (attachment_id, jobs.PIPELINE_VERSION), + ).fetchall() + del row["object_key"] + return public({**row, "variants": variants}) + + +def visible(conn, attachment_id, user_id): + row = conn.execute( + f"""SELECT {METADATA}, object_key FROM attachments a + WHERE id=%s AND EXISTS ( + SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=%s + )""", + (attachment_id, user_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def content(attachment_id, user_id, variant): + with connection() as conn: + row = visible(conn, attachment_id, user_id) + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение недоступно") + if variant != "original": + row = conn.execute( + """SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants + WHERE (attachment_id,pipeline_version,name)=(%s,%s,%s)""", + (attachment_id, jobs.PIPELINE_VERSION, variant), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + + data = s3.download(row["object_key"], row["size_bytes"]) + if len(data) != row["size_bytes"] or hashlib.sha256(data).hexdigest() != row["sha256"]: + raise s3.unavailable() + return data, row["content_type"] diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..c516ba4 --- /dev/null +++ b/src/config.py @@ -0,0 +1,16 @@ +import os +from pathlib import Path + + +def secret(name): + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text().strip() + raise RuntimeError(f"{name}_FILE is required") + + +SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +if not 1 <= SESSION_TTL <= 604800: + raise RuntimeError("SESSION_TTL_SECONDS must be 1..604800") +if os.getenv("SESSION_STORE", "postgres") != "postgres": + raise RuntimeError("This implementation uses PostgreSQL sessions") diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..b0fef8f --- /dev/null +++ b/src/db.py @@ -0,0 +1,40 @@ +from contextlib import contextmanager +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from src.config import secret + +pool = ConnectionPool( + conninfo=secret("DATABASE_URL"), min_size=1, max_size=8, + timeout=2, max_waiting=32, reconnect_timeout=5, open=False, + kwargs={"connect_timeout": 2, "row_factory": dict_row, + "options": "-c statement_timeout=2000 -c lock_timeout=1500 -c idle_in_transaction_session_timeout=5000 -c timezone=UTC"}, +) + + +def migrate(): + # Serialize migration jobs if two operators start the stack concurrently. + with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as conn: + conn.execute("SET LOCAL lock_timeout = '10s'") + conn.execute("SET LOCAL statement_timeout = '15s'") + conn.execute("SELECT pg_advisory_xact_lock(202602)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())") + for path in sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")): + if not conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,)) + + +@contextmanager +def connection(): + with pool.connection() as conn: + # psycopg context commits on success, rolls back on EVERY exception. + yield conn + + +if __name__ == "__main__": + migrate() + print("PASS: versioned migrations applied (already applied versions skipped)") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..2e7670f --- /dev/null +++ b/src/errors.py @@ -0,0 +1,34 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + def __init__( + self, + status_code: int, + code: str, + message: str, + ): + self.status_code = status_code + self.code = code + self.message = message + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, +) -> JSONResponse: + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..667d468 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,81 @@ +"""AMQP carrier and durable publishing, shared by API and worker.""" + +import json +import logging +import os +from contextlib import contextmanager +from datetime import datetime +from typing import Literal +from uuid import UUID + +import pika +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + +from src.config import secret + +QUEUE = "attachment.process.v1" +PIPELINE_VERSION = 1 +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +if not 1 <= MAX_ATTEMPTS <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS must be 1..20") + +# Report dependency failures at the call site without logging connection details. +logging.getLogger("pika").setLevel(logging.CRITICAL) + + +class Job(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + schema_version: Literal[1] + type: Literal["attachment.process.v1"] + job_id: UUID + attachment_id: UUID + pipeline_version: int = Field(ge=1) + attempt: int = Field(ge=1) + created_at: AwareDatetime + traceparent: str = Field(default="", pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + tracestate: str = Field(default="", max_length=512) + + +def carrier(job_id: UUID, attachment_id: UUID, created_at: datetime, attempt: int = 1): + return Job( + schema_version=1, type=QUEUE, job_id=job_id, attachment_id=attachment_id, + pipeline_version=PIPELINE_VERSION, attempt=attempt, created_at=created_at, + ) + + +def parse(body): + # Validate JSON primitive types before Pydantic's UUID/time parsing. + values = json.loads(body) + if not isinstance(values, dict) or any( + type(values.get(k)) is not int for k in ("schema_version", "pipeline_version", "attempt") + ) or any(type(values.get(k)) is not str for k in ("job_id", "attachment_id", "created_at")): + raise ValueError("Bad job carrier") + return Job.model_validate_json(body) + + +@contextmanager +def broker(): + params = pika.URLParameters(secret("BROKER_URL")) + params.socket_timeout = 2 + params.stack_timeout = 4 + params.blocked_connection_timeout = 4 + params.heartbeat = 30 + conn = pika.BlockingConnection(params) + try: + channel = conn.channel() + channel.queue_declare(queue=QUEUE, durable=True, arguments={"x-queue-type": "classic"}) + channel.confirm_delivery() + yield channel + finally: + if conn.is_open: + conn.close() + + +def publish(channel, job): + channel.basic_publish( + exchange="", routing_key=QUEUE, body=job.model_dump_json(exclude_unset=True).encode(), + properties=pika.BasicProperties(delivery_mode=2, content_type="application/json", + message_id=str(job.job_id)), + mandatory=True, + ) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..ff80bbc --- /dev/null +++ b/src/main.py @@ -0,0 +1,247 @@ +import hashlib +import logging +import os +import re +import secrets +import time +from contextlib import asynccontextmanager +from uuid import UUID, uuid4 + +import anyio +import psycopg +from opentelemetry.trace import SpanKind +from fastapi import Depends, FastAPI, Query, Request, Response +from starlette.requests import ClientDisconnect +from fastapi.exceptions import RequestValidationError +from psycopg_pool import PoolTimeout, TooManyRequests +from starlette.exceptions import HTTPException + +from src import attachments, s3, store, telemetry +from src.config import SESSION_TTL +from src.db import connection, pool +from src.errors import ApiError, error_response +from src.models import Attachment, Chat, ChatPage, CreateChat, CreateMessage, CreateUser, Message, MessagePage, User +from src.security import basic_credentials, hasher, token_hash, unauthorized, verify + + +INSTANCE_ID = os.getenv("INSTANCE_ID") or uuid4().hex +IDEMPOTENCY_KEY = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +@asynccontextmanager +async def lifespan(app): + anyio.to_thread.current_default_thread_limiter().total_tokens = 16 + app.state.upload_slots = anyio.Semaphore(2) + app.state.telemetry = telemetry.Telemetry("messenger-api", INSTANCE_ID) + pool.open() + try: + pool.wait(timeout=10) + yield + finally: + pool.close() + app.state.telemetry.shutdown() + + +app = FastAPI(title="MTUSI Messenger", version=telemetry.VERSION, lifespan=lifespan, + docs_url=None, redoc_url=None, openapi_url=None) + + +def failure(request, status, code, message): + request.state.error_code = code + response = error_response(request, status, code, message) + if status == 401: + basic = request.url.path == "/api/v1/auth/sessions" + response.headers['WWW-Authenticate'] = ('Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"') + return response + + +@app.middleware("http") +async def request_context(request, call_next): + signals = request.app.state.telemetry + started = time.perf_counter() + method = request.method if request.method in telemetry.METHODS else "OTHER" + with signals.tracer.start_as_current_span( + method, kind=SpanKind.SERVER, record_exception=False, set_status_on_exception=False, + ) as span: + response = await dispatch_request(request, call_next) + route = getattr(request.scope.get("route"), "path", getattr(request.state, "route_template", "unmatched")) + status = response.status_code + status_class = f"{status // 100}xx" + labels = {"method": method, "route": route, "status_class": status_class} + signals.http_requests.add(1, labels) + signals.http_duration.record(time.perf_counter() - started, labels) + + request_id = request.state.request_id + error_code = getattr(request.state, "error_code", None) + span.update_name(f"{method} {route}") + span.set_attributes({ + "http.request.method": method, "http.route": route, + "http.response.status_code": status, "request.id": request_id, + }) + telemetry.complete_span(span, status_class, failed=status >= 500) + severity = logging.ERROR if status >= 500 else logging.WARNING if status >= 400 else logging.INFO + details = {"error_code": error_code} if error_code else {} + telemetry.event("http.completed", severity, method=method, route=route, + status_code=status, request_id=request_id, **details) + return response + + +async def dispatch_request(request, call_next): + request.state.request_id = str(uuid4()) + try: + if request.method == "POST" and request.url.path == "/api/v1/auth/sessions" and await request.body(): + request.state.route_template = "/api/v1/auth/sessions" + raise ApiError(400, "invalid_request", "Вход не принимает тело запроса") + response = await call_next(request) + except ApiError as exc: + response = failure(request, exc.status_code, exc.code, exc.message) + except (psycopg.Error, PoolTimeout, TooManyRequests): + response = failure(request, 503, "unavailable", "Хранилище временно недоступно") + except Exception as exc: + logging.error("Unhandled request error: %s", type(exc).__name__) + response = failure(request, 500, "internal_error", "Внутренняя ошибка сервера") + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error(request, exc): + return failure(request, exc.status_code, exc.code, exc.message) + + +@app.exception_handler(RequestValidationError) +async def validation_error(request, exc): + return failure(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(HTTPException) +async def http_error(request, exc): + return failure(request, exc.status_code, "not_found" if exc.status_code == 404 else "invalid_request", "Недопустимый HTTP-запрос") + + +def require_user(request: Request): + digest = token_hash(request.headers.get('Authorization')) + with connection() as conn: + user = conn.execute( + """SELECT u.id,u.username,u.display_name,u.created_at + FROM sessions s JOIN users u ON u.id=s.user_id + WHERE s.token_hash=%s AND s.expires_at > clock_timestamp()""", + (digest,), + ).fetchone() + if user is None: + unauthorized() + request.state.token_hash = digest + return User(**store.public(user)) + + +@app.get('/health/live') +def live(): + return {"status": "ok"} + + +@app.get('/health/ready') +def ready(): + with connection() as conn: + conn.execute("SELECT 1").fetchone() + s3.ready() + return {"status": "ok"} + + +@app.post('/api/v1/users', status_code=201, response_model=User) +def register(data: CreateUser): + return store.create_user(data, hasher.hash(data.password)) + + +@app.post('/api/v1/auth/sessions', status_code=201) +def login(request: Request): + username, password = basic_credentials(request.headers.get('Authorization')) + with connection() as conn: + row = conn.execute("SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", (username,)).fetchone() + valid = verify(row['password_hash'] if row else None, password) + if not row or not valid: + unauthorized() + token = secrets.token_urlsafe(32) + digest = hashlib.sha256(token.encode('ascii')).digest() + with connection() as conn: + result = conn.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at", (digest, row['id'], SESSION_TTL)).fetchone() + del row['password_hash'] + return {"token": token, "token_type": "Bearer", "expires_at": store.public(result['expires_at']), "user": store.public(row)} + + +@app.delete('/api/v1/auth/sessions/current', status_code=204) +def logout(request: Request, user: User = Depends(require_user)): + with connection() as conn: + conn.execute("DELETE FROM sessions WHERE token_hash=%s", (request.state.token_hash,)) + return Response(status_code=204) + + +@app.get('/api/v1/users/me', response_model=User) +def me(user: User = Depends(require_user)): + return user + + +@app.post('/api/v1/chats', status_code=201, response_model=Chat) +def create_chat(data: CreateChat, user: User = Depends(require_user)): + return store.create_chat(data, user.id) + + +@app.get('/api/v1/chats', response_model=ChatPage) +def chats(user: User = Depends(require_user), limit: int = Query(50, ge=1, le=100), cursor: str | None = Query(None, min_length=1, max_length=2048)): + return store.list_chats(user.id, limit, cursor) + + +@app.get('/api/v1/chats/{chat_id}', response_model=Chat) +def chat(chat_id: UUID, user: User = Depends(require_user)): + return store.get_chat(str(chat_id), user.id) + + +@app.post('/api/v1/chats/{chat_id}/messages', status_code=201, response_model=Message) +def send(request: Request, chat_id: UUID, data: CreateMessage, user: User = Depends(require_user)): + keys = request.headers.getlist("Idempotency-Key") + if len(keys) != 1 or not IDEMPOTENCY_KEY.fullmatch(keys[0]): + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + return store.create_message( + str(chat_id), user.id, request.method, keys[0], data.text, data.attachment_ids, + ) + + +@app.get('/api/v1/chats/{chat_id}/messages', response_model=MessagePage) +def history(chat_id: UUID, user: User = Depends(require_user), limit: int = Query(50, ge=1, le=100), cursor: str | None = Query(None, min_length=1, max_length=2048)): + return store.list_messages(str(chat_id), user.id, limit, cursor) + +@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202, response_model=Attachment) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str = Query(..., min_length=1, max_length=255), + user: User = Depends(require_user), +): + await anyio.to_thread.run_sync(store.get_chat, str(chat_id), user.id) + attachments.validate_filename(filename) + types = request.headers.getlist("content-type") + if len(types) != 1 or types[0] not in attachments.FORMATS: + raise attachments.invalid_image() + async with request.app.state.upload_slots: + try: + body = await attachments.read_upload(request) + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Загрузка прервана") from None + with body: + return await anyio.to_thread.run_sync( + attachments.upload, chat_id, user.id, filename, types[0], body, + ) + + +@app.get('/api/v1/attachments/{attachment_id}', response_model=Attachment) +def attachment_metadata(attachment_id: UUID, user: User = Depends(require_user)): + return attachments.get(attachment_id, user.id) + + +@app.get('/api/v1/attachments/{attachment_id}/content') +def attachment_content( + attachment_id: UUID, user: User = Depends(require_user), variant: str = Query("original"), +): + data, content_type = attachments.content(attachment_id, user.id, variant) + return Response(data, media_type=content_type, + headers={"Cache-Control": "private, no-store"}) diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..dce1b6e --- /dev/null +++ b/src/models.py @@ -0,0 +1,171 @@ +from typing import Literal +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from uuid import UUID + + +class CreateUser(BaseModel): + model_config = ConfigDict( + extra="forbid", + strict=True, + ) + + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-z0-9_]{3,32}$", + ) + + display_name: str = Field( + min_length=1, + max_length=100, + ) + + @field_validator("display_name") + @classmethod + def display_name_must_not_be_blank(cls, value: str) -> str: + if not value.strip() or "\x00" in value: + raise ValueError("display_name must not be blank") + + return value + + + password: str = Field(min_length=12, max_length=128) + + @field_validator("password") + @classmethod + def password_must_not_be_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("password must not be blank") + return value + + +class User(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + username: str + display_name: str + created_at: str + + +class CreateChat(BaseModel): + model_config = ConfigDict( + extra="forbid", + strict=True, + ) + + title: str = Field( + min_length=1, + max_length=100, + ) + + member_ids: list[str] = Field( + min_length=1, + max_length=100, + ) + + @field_validator("title") + @classmethod + def title_must_not_be_blank(cls, value: str) -> str: + if not value.strip() or "\x00" in value: + raise ValueError("title must not be blank") + + return value + + @field_validator("member_ids") + @classmethod + def validate_member_ids(cls, value: list[str]) -> list[str]: + normalized = [] + for member_id in value: + try: + normalized.append(str(UUID(member_id))) + except (ValueError, TypeError): + raise ValueError("member_ids must contain valid UUIDs") from None + + if len(normalized) != len(set(normalized)): + raise ValueError("member_ids must be unique") + + return normalized + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + title: str + member_ids: list[str] + created_at: str + + +class ChatPage(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict( + extra="forbid", + strict=True, + ) + + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def validate_attachment_ids(cls, value: list[str]) -> list[str]: + normalized = [str(UUID(attachment_id)) for attachment_id in value] + if len(normalized) != len(set(normalized)): + raise ValueError("attachment_ids must be unique") + return normalized + + @model_validator(mode="after") + def validate_content(self): + if "\x00" in self.text or (not self.text.strip() and not self.attachment_ids): + raise ValueError("Message requires text or attachments") + return self + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: list[str] + + +class MessagePage(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[Message] + next_cursor: str | None + +class AttachmentVariant(BaseModel): + name: Literal["thumbnail"] + content_type: Literal["image/jpeg"] + size_bytes: int + sha256: str + width: int + height: int + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + owner_id: str + filename: str + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[AttachmentVariant] + error_code: str | None + created_at: str + updated_at: str diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 0000000..563a8e5 --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,74 @@ +import base64 +import json +import re +from datetime import datetime +from uuid import UUID + +from src.errors import ApiError + + +def encode_cursor( + resource: str, + owner_id: str, + created_at: str, + object_id: str, +) -> str: + payload = { + "resource": resource, + "owner_id": owner_id, + "created_at": created_at, + "id": object_id, + } + + raw = json.dumps( + payload, + separators=(",", ":"), + ).encode("utf-8") + + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor( + cursor: str, + expected_resource: str, + expected_owner_id: str, +) -> tuple[str, str]: + try: + if not re.fullmatch(r"[A-Za-z0-9_-]+", cursor): + raise ValueError + padding = "=" * (-len(cursor) % 4) + + raw = base64.b64decode( + (cursor + padding).encode("ascii"), altchars=b"-_", validate=True + ) + + payload = json.loads(raw.decode("utf-8")) + + if payload["resource"] != expected_resource: + raise ValueError + + if payload["owner_id"] != expected_owner_id: + raise ValueError + + created_at = payload["created_at"] + object_id = payload["id"] + + # The cursor must carry the same UTC timestamp format produced + # by the server. Invalid timestamps must return HTTP 400 before a SQL cast. + if not isinstance(created_at, str) or not re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z", + created_at, + ): + raise ValueError + + datetime.fromisoformat(created_at.replace("Z", "+00:00")) + object_id = str(UUID(object_id)) + + return created_at, object_id + + except Exception: + raise ApiError( + status_code=400, + code="invalid_request", + message="Некорректный cursor", + ) diff --git a/src/s3.py b/src/s3.py new file mode 100644 index 0000000..713cb71 --- /dev/null +++ b/src/s3.py @@ -0,0 +1,92 @@ +"""Synchronous access to the single private object bucket.""" + +import os +from contextlib import closing + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from src import telemetry +from src.config import secret +from src.errors import ApiError + + +BUCKET = os.environ["S3_BUCKET"] +client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + region_name=os.environ["S3_REGION"], + aws_access_key_id=secret("S3_ACCESS_KEY_ID"), + aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"), + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=0.5, + read_timeout=2, + retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + proxies={}, + ), +) + + +def unavailable(): + telemetry.storage_unavailable() + return ApiError(503, "unavailable", "Хранилище вложений временно недоступно") + + +def ready(): + try: + client.head_bucket(Bucket=BUCKET) + except (ClientError, BotoCoreError): + raise unavailable() from None + + +def put(key, content, size, content_type): + try: + client.put_object( + Bucket=BUCKET, Key=key, Body=content, ContentLength=size, + ContentType=content_type, + ) + except (ClientError, BotoCoreError): + raise unavailable() from None + + +def download(key, size): + try: + result = client.get_object(Bucket=BUCKET, Key=key) + with closing(result["Body"]) as body: + if result["ContentLength"] != size: + raise unavailable() + return body.read(size) + except (ClientError, BotoCoreError): + raise unavailable() from None + + +def delete(key): + try: + client.delete_object(Bucket=BUCKET, Key=key) + except (ClientError, BotoCoreError): + raise unavailable() from None + + +def bootstrap(): + try: + client.head_bucket(Bucket=BUCKET) + except ClientError as exc: + if exc.response["ResponseMetadata"]["HTTPStatusCode"] != 404: + raise + client.create_bucket(Bucket=BUCKET) + # New buckets are private by default. Remove any old public bucket policy. + client.delete_bucket_policy(Bucket=BUCKET) + client.head_bucket(Bucket=BUCKET) + + +if __name__ == "__main__": + try: + bootstrap() + except (ClientError, BotoCoreError): + raise SystemExit("S3 bootstrap failed") from None + print("PASS: private attachment bucket bootstrapped") diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..6d95ee3 --- /dev/null +++ b/src/security.py @@ -0,0 +1,47 @@ +import base64 +import hashlib +import re +import secrets + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError + +from src.errors import ApiError + +# Explicit bounded Argon2id parameters: 19 MiB, 2 passes, one lane. +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = hasher.hash(secrets.token_urlsafe(32)) + + +def unauthorized(): + raise ApiError(401, "unauthorized", "Требуется действительная сессия") + + +def basic_credentials(header): + try: + scheme, value = header.split(" ", 1) + if scheme.lower() != "basic": + raise ValueError + username, password = base64.b64decode(value, validate=True).decode("utf-8").split(":", 1) + if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError, AttributeError): + unauthorized() + + +def verify(encoded, password): + try: + return hasher.verify(encoded or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + + +def token_hash(header): + try: + scheme, token = header.split(" ", 1) + if scheme.lower() != "bearer" or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token): + raise ValueError + return hashlib.sha256(token.encode("ascii")).digest() + except (ValueError, AttributeError): + unauthorized() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..459b819 --- /dev/null +++ b/src/store.py @@ -0,0 +1,174 @@ +from datetime import datetime +from uuid import UUID, uuid4 + +from psycopg.errors import ForeignKeyViolation, UniqueViolation + +from src.db import connection +from src.errors import ApiError +from src.pagination import decode_cursor, encode_cursor + + +def public(value): + if isinstance(value, dict): + return {key: public(item) for key, item in value.items()} + if isinstance(value, list): + return [public(item) for item in value] + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + return value.isoformat().replace("+00:00", "Z") + return value + + +def create_user(data, password_hash): + try: + with connection() as conn: + row = conn.execute( + "INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at", + (uuid4(), data.username, data.display_name, password_hash), + ).fetchone() + return public(row) + except UniqueViolation: + raise ApiError(409, "conflict", "Username уже занят") from None + + +def create_chat(data, caller): + if caller not in data.member_ids: + raise ApiError(400, "invalid_request", "Создатель должен входить в member_ids") + try: + with connection() as conn: + chat = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), data.title)).fetchone() + # FK errors here MUST roll back the already inserted parent chat. + with conn.cursor() as cursor: + cursor.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", [(chat['id'], member) for member in data.member_ids]) + chat['member_ids'] = data.member_ids + return public(chat) + except ForeignKeyViolation: + raise ApiError(400, "invalid_request", "Указан неизвестный участник") from None + + +CHAT_SELECT = """SELECT c.id,c.title,c.created_at, + ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids + FROM chats c WHERE EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)""" + + +def get_chat(chat_id, user_id): + with connection() as conn: + row = conn.execute(CHAT_SELECT + " AND c.id=%s", (user_id, chat_id)).fetchone() + if not row: + raise ApiError(404, "not_found", "Объект не найден") + return public(row) + + +def page(rows, limit, resource, owner): + items = public(rows[:limit]) + cursor = None + if len(rows) > limit: + last = items[-1] + cursor = encode_cursor(resource, owner, last['created_at'], last['id']) + return {"items": items, "next_cursor": cursor} + + +def list_chats(user_id, limit, cursor): + params = [user_id] + sql = CHAT_SELECT + if cursor is not None: + timestamp, object_id = decode_cursor(cursor, "chats", user_id) + sql += " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" + params += [timestamp, object_id] + sql += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(sql, params).fetchall() + return page(rows, limit, "chats", user_id) + + +MESSAGE_SELECT = """SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id=m.id ORDER BY ma.position) AS attachment_ids + FROM messages m""" + + +def check_attachments(conn, chat_id, attachment_ids): + if not attachment_ids: + return + rows = conn.execute( + """SELECT id,status FROM attachments + WHERE chat_id=%s AND id=ANY(%s::uuid[]) ORDER BY id FOR SHARE""", + (chat_id, attachment_ids), + ).fetchall() + if len(rows) != len(attachment_ids): + raise ApiError(404, "not_found", "Объект не найден") + if any(row["status"] == "failed" for row in rows): + raise ApiError(409, "conflict", "Вложение недоступно") + + +def create_message(chat_id, user_id, method, key, text, attachment_ids): + scope = (user_id, method, chat_id, key) + message_id = uuid4() + with connection() as conn: + if not conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s FOR KEY SHARE", + (chat_id, user_id), + ).fetchone(): + raise ApiError(404, "not_found", "Объект не найден") + + # A concurrent ON CONFLICT waits for the first transaction. The deferred + # FK allows reserving the proposed message ID before inserting the row. + claimed = conn.execute( + """INSERT INTO message_idempotency (user_id,method,chat_id,key,message_id,expires_at) + VALUES (%s,%s,%s,%s,%s,clock_timestamp() + interval '24 hours') + ON CONFLICT (user_id,method,chat_id,key) DO UPDATE + SET message_id=EXCLUDED.message_id, expires_at=clock_timestamp() + interval '24 hours' + WHERE message_idempotency.expires_at <= clock_timestamp() + RETURNING message_id""", + (*scope, message_id), + ).fetchone() + + if claimed: + check_attachments(conn, chat_id, attachment_ids) + row = conn.execute( + """INSERT INTO messages (id,chat_id,sender_id,text) + VALUES (%s,%s,%s,%s) + RETURNING id,chat_id,sender_id,text,created_at""", + (message_id, chat_id, user_id, text), + ).fetchone() + if attachment_ids: + with conn.cursor() as cursor: + cursor.executemany( + """INSERT INTO message_attachments + (chat_id,message_id,attachment_id,position) VALUES (%s,%s,%s,%s)""", + [ + (chat_id, message_id, attachment_id, position) + for position, attachment_id in enumerate(attachment_ids, 1) + ], + ) + row["attachment_ids"] = attachment_ids + else: + # READ COMMITTED gives this statement a new snapshot. The UPSERT + # still holds the key row locked, including when its WHERE was false. + row = conn.execute( + MESSAGE_SELECT + """ JOIN message_idempotency i ON i.message_id=m.id + WHERE (i.user_id,i.method,i.chat_id,i.key)=(%s,%s,%s,%s)""", + scope, + ).fetchone() + if row["text"] != text or [str(value) for value in row["attachment_ids"]] != attachment_ids: + raise ApiError(409, "conflict", "Ключ уже использован с другим сообщением") + return public(row) + + +def list_messages(chat_id, user_id, limit, cursor): + with connection() as conn: + # Access precedes cursor decoding, including malformed foreign cursors. + if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone(): + raise ApiError(404, "not_found", "Объект не найден") + params = [chat_id] + sql = MESSAGE_SELECT + " WHERE m.chat_id=%s" + if cursor is not None: + timestamp, object_id = decode_cursor(cursor, "messages", chat_id) + sql += " AND (created_at,id) > (%s::timestamptz,%s::uuid)" + params += [timestamp, object_id] + sql += " ORDER BY created_at,id LIMIT %s" + params.append(limit + 1) + rows = conn.execute(sql, params).fetchall() + return page(rows, limit, "messages", chat_id) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..b00e632 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,96 @@ +"""Small allowlisted telemetry surface. No automatic HTTP or dependency capture.""" + +import logging +import os + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import Status, StatusCode + +VERSION = "1.6.0" +BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60) +METHODS = {"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "TRACE", "CONNECT"} +EVENTS = logging.getLogger("lab6.events") +EVENTS.propagate = False +EVENTS.addHandler(logging.NullHandler()) + + +class Telemetry: + def __init__(self, service_name, instance_id): + # Explicit resource and destinations. Do not read client HTTP headers. + resource = Resource({ + "service.name": os.getenv("OTEL_SERVICE_NAME", service_name), + "service.version": VERSION, + "service.instance.id": instance_id, + "deployment.environment.name": "lab", + }) + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + self.traces = TracerProvider(resource=resource) + self.traces.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces", timeout=2), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000, + )) + self.tracer = self.traces.get_tracer("messenger", VERSION) + self.logs = LoggerProvider(resource=resource) + self.logs.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f"{endpoint}/v1/logs", timeout=2), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000, + )) + self.log_handler = LoggingHandler(logger_provider=self.logs) + EVENTS.setLevel(logging.INFO) + EVENTS.addHandler(self.log_handler) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics", timeout=2), + export_interval_millis=5000, export_timeout_millis=2000, + ) + self.metrics = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[View( + instrument_type=None, instrument_name="messenger.*.duration", + aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS), + )], + ) + meter = self.metrics.get_meter("messenger", VERSION) + self.http_requests = meter.create_counter("messenger.http.requests") + self.http_duration = meter.create_histogram("messenger.http.duration", unit="s") + self.job_outcomes = meter.create_counter("messenger.job.outcomes") + self.job_duration = meter.create_histogram("messenger.job.duration", unit="s") + + def shutdown(self): + EVENTS.removeHandler(self.log_handler) + self.metrics.shutdown(timeout_millis=2500) + self.logs.shutdown() + self.traces.shutdown() + + +def event(name, severity=logging.INFO, **attributes): + # Only this logger has an OTLP handler. Callers pass fixed strings and + # explicitly selected technical IDs. Attach no exceptions or raw requests. + EVENTS.log(severity, name, extra={"event": name, **attributes}) + + +def storage_unavailable(): + span = trace.get_current_span() + if span.is_recording(): + span.set_attribute("dependency.name", "s3") + event("dependency.unavailable", logging.WARNING, dependency="s3") + + +def complete_span(span, outcome, failed=False): + span.set_attribute("outcome", outcome) + if failed: + span.set_status(Status(StatusCode.ERROR)) + else: + span.set_status(Status(StatusCode.OK)) diff --git a/src/thumbnail.py b/src/thumbnail.py new file mode 100644 index 0000000..6f6bdd6 --- /dev/null +++ b/src/thumbnail.py @@ -0,0 +1,25 @@ +"""Create the single promised grade 3 variant.""" + +from io import BytesIO + +from PIL import Image, ImageOps + + +def thumbnail(data): + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as image: + oriented = ImageOps.exif_transpose(image) + width, height = oriented.size + divisor = max(width, height, 256) + size = (max(1, width * 256 // divisor), max(1, height * 256 // divisor)) + if "A" in oriented.getbands() or "transparency" in oriented.info: + rgba = oriented.convert("RGBA") + white = Image.new("RGB", rgba.size, "white") + white.paste(rgba, mask=rgba.getchannel("A")) + rgb = white + else: + rgb = oriented.convert("RGB") + if rgb.size != size: + rgb = rgb.resize(size, Image.Resampling.LANCZOS) + result = BytesIO() + rgb.save(result, format="JPEG", quality=85) + return result.getvalue(), size diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..fb7bff5 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,199 @@ +"""One AMQP consumer with durable attempt accounting and idempotent effects.""" + +import hashlib +import logging +import time +import os +from functools import partial +from io import BytesIO + +import pika +import psycopg +from opentelemetry.trace import SpanKind +from psycopg.rows import dict_row + +from src import jobs, s3, telemetry +from src.config import secret +from src.errors import ApiError +from src.thumbnail import thumbnail + +RETRY_SECONDS = 3 + + +def select_job(conn, job): + return conn.execute( + """SELECT j.*, a.status, a.object_key, a.content_type, a.size_bytes, a.sha256, + EXISTS (SELECT 1 FROM attachment_variants v + WHERE (v.attachment_id, v.pipeline_version, v.name) = + (j.attachment_id, j.pipeline_version, 'thumbnail')) AS has_thumbnail + FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE (j.job_id,j.attachment_id,j.pipeline_version,j.created_at)=(%s,%s,%s,%s)""", + (job.job_id, job.attachment_id, job.pipeline_version, job.created_at), + ).fetchone() + + +def state(conn, row, status, error=None): + conn.execute( + """UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() + WHERE id=%s""", + (status, error, row["attachment_id"]), + ) + + +def retry_or_fail(conn, row, reason, permanent=False): + attempt = row["attempt"] + conn.execute("UPDATE attachment_jobs SET last_error=%s WHERE job_id=%s", (reason, row["job_id"])) + if permanent or attempt == row["max_attempts"]: + code = reason if permanent else "retry_exhausted" + state(conn, row, "failed", code) + return ("failed", code) + + conn.execute( + """UPDATE attachment_jobs SET attempt=attempt+1, + retry_at=clock_timestamp()+(%s * interval '1 second') WHERE job_id=%s""", + (RETRY_SECONDS, row["job_id"]), + ) + state(conn, row, "queued") + return ("retry", reason) + + +def objects(row): + original = s3.download(row["staging_key"], row["size_bytes"]) + if len(original) != row["size_bytes"] or hashlib.sha256(original).hexdigest() != row["sha256"]: + raise ValueError("Staging checksum mismatch") + + data, (width, height) = thumbnail(original) + key = f"variants/{row['attachment_id']}/v{row['pipeline_version']}/thumbnail.jpg" + s3.put(row["object_key"], BytesIO(original), len(original), row["content_type"]) + s3.put(key, BytesIO(data), len(data), "image/jpeg") + return key, len(data), hashlib.sha256(data).hexdigest(), width, height + + +def process(conn, row): + try: + variant = objects(row) + except ApiError: + with conn.transaction(): + return retry_or_fail(conn, row, "storage_unavailable") + except (ValueError, OSError, SyntaxError, IndexError): + with conn.transaction(): + return retry_or_fail(conn, row, "invalid_image", permanent=True) + + with conn.transaction(): + conn.execute( + """INSERT INTO attachment_variants + (attachment_id,pipeline_version,name,content_type,object_key,size_bytes,sha256,width,height) + VALUES (%s,%s,'thumbnail','image/jpeg',%s,%s,%s,%s,%s)""", + (row["attachment_id"], row["pipeline_version"], *variant), + ) + state(conn, row, "ready") + + try: + s3.delete(row["staging_key"]) + except ApiError: + telemetry.event("job.staging_retained", logging.WARNING) + return ("ready", None) + + +def deliver(signals, channel, method, _properties, body): + started = time.perf_counter() + with signals.tracer.start_as_current_span( + "attachment.process.v1", kind=SpanKind.CONSUMER, record_exception=False, + set_status_on_exception=False, attributes={"messaging.operation.type": "process", + "messaging.system": "rabbitmq"}, + ) as span: + try: + process_delivery(signals, span, started, channel, method, body) + except (pika.exceptions.AMQPError, psycopg.Error, OSError): + telemetry.complete_span(span, "delivery_interrupted", failed=True) + telemetry.event("job.delivery_interrupted", logging.WARNING) + raise + + +def process_delivery(signals, span, started, channel, method, body): + try: + job = jobs.parse(body) + except ValueError: + telemetry.complete_span(span, "rejected", failed=True) + telemetry.event("job.carrier_rejected", logging.WARNING) + channel.basic_reject(method.delivery_tag, requeue=False) + return + + details = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id), + "attempt": job.attempt, "job_type": job.type} + span.set_attributes(details) + + with psycopg.connect(secret("DATABASE_URL"), row_factory=dict_row, connect_timeout=3, + options="-c statement_timeout=5000 -c lock_timeout=3000", + autocommit=True) as conn: + # The session lock spans state commits and S3 writes. Closing the database + # session releases it, including on worker crash. Prefetch is one. + conn.execute("SELECT pg_advisory_lock(hashtextextended(%s, 0))", (str(job.job_id),)) + row = select_job(conn, job) + if row is None or job.attempt > row["attempt"]: + telemetry.complete_span(span, "rejected", failed=True) + telemetry.event("job.unregistered", logging.WARNING, **details) + channel.basic_reject(method.delivery_tag, requeue=False) + return + + status = row["status"] + if status == "ready": + if not row["has_thumbnail"]: + raise RuntimeError("Missing committed thumbnail") + outcome, reason = "duplicate_ready", None + elif status == "failed": + outcome, reason = "duplicate_failed", None + elif status == "processing": + # A prior consumer committed its claim but lost its session before + # finalizing. Its attempt counts against the same persistent budget. + with conn.transaction(): + outcome, reason = retry_or_fail(conn, row, "interrupted") + elif job.attempt < row["attempt"]: + # Covers interruption after committing retry but before publisher confirm. + outcome, reason = "republish_retry", None + else: + delay = conn.execute( + "SELECT GREATEST(0,EXTRACT(EPOCH FROM (%s-clock_timestamp()))) AS delay", + (row["retry_at"],), + ).fetchone()["delay"] + channel.connection.sleep(float(delay)) + with conn.transaction(): + state(conn, row, "processing") + started = time.perf_counter() + telemetry.event("job.processing", **details) + outcome, reason = process(conn, row) + + # process()/retry_or_fail() have committed the transition. A later + # redelivery of ready/failed must not increment terminal counters. + if outcome in {"ready", "failed"}: + labels = {"job_type": job.type, "outcome": "success" if outcome == "ready" else "fail"} + signals.job_outcomes.add(1, labels) + signals.job_duration.record(time.perf_counter() - started, labels) + + telemetry.complete_span(span, outcome, failed=outcome in {"failed", "retry"}) + severity = logging.WARNING if outcome in {"failed", "retry"} else logging.INFO + problem = {"reason": reason} if reason else {} + telemetry.event("job.outcome", severity, outcome=outcome, **details, **problem) + + if outcome in {"retry", "republish_retry"}: + attempt = row["attempt"] + (outcome == "retry") + jobs.publish(channel, jobs.carrier(job.job_id, job.attachment_id, job.created_at, attempt)) + channel.basic_ack(method.delivery_tag) + + +def main(): + signals = telemetry.Telemetry("messenger-worker", os.getenv("INSTANCE_ID", "worker")) + while True: + try: + with jobs.broker() as channel: + channel.basic_qos(prefetch_count=1) + channel.basic_consume(queue=jobs.QUEUE, on_message_callback=partial(deliver, signals), auto_ack=False) + telemetry.event("worker.consumer_ready") + channel.start_consuming() + except (pika.exceptions.AMQPError, psycopg.Error, OSError) as exc: + telemetry.event("worker.reconnecting", logging.WARNING, reason=type(exc).__name__) + time.sleep(2) + + +if __name__ == "__main__": + main() diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..c3a1d09 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,12 @@ +# Проверки ЛР5 + +В Windows PowerShell поднимите стенд по REPORT.md, затем выполните: + +```powershell +docker compose --profile test build tools +docker compose run --rm --no-deps tools python student-tests/grade3.py +``` + +Тест использует Docker socket из профиля tools и действует только на сервисы локального Compose-проекта `mtusi-lab6-grade3` или его именованного варианта с суффиксом. Он останавливает и запускает worker, API, S3 и RabbitMQ, временно блокирует запись одного типа данных в PostgreSQL для наблюдения processing и изменяет staging двух собственных вложений для инъекции ошибок. Не запускайте его на стенде с чужой нагрузкой или одновременно с публичными сценариями отказов. + +Сохраняются тестовые пользователи, чаты, файлы и журнал попыток. Токены используются в памяти. Сценарий сам восстанавливает затронутые сервисы при проверяемых сбоях. После прерывания процесса состояние стенда можно привести к норме командой `docker compose up -d --wait --wait-timeout 150`. diff --git a/student-tests/grade3.py b/student-tests/grade3.py new file mode 100644 index 0000000..0b235e8 --- /dev/null +++ b/student-tests/grade3.py @@ -0,0 +1,454 @@ +"""Reproduce Lab 5 acceptance scenarios using only local Compose services.""" +import hashlib +import json +import os +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from io import BytesIO +from pathlib import Path +import re +import subprocess +import sys +import time +from uuid import UUID, uuid4 + +import requests +from jsonschema import Draft202012Validator, FormatChecker +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +PROJECT = os.getenv("LAB6_PROJECT", "") +SERVICES = {"api1", "api2", "broker", "worker", "postgres", "s3", "lb"} +JOB_SCHEMA = json.loads((ROOT / "contracts/jobs.schema.json").read_text(encoding="utf-8")) +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def docker(*args): + completed = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=45) + check(completed.returncode == 0, f"docker {args[0]} returned {completed.returncode}") + return completed.stdout.strip() + + +def container(service): + check(service in SERVICES, "unexpected service") + ids = docker("ps", "-aq", "--filter", f"label=com.docker.compose.project={PROJECT}", + "--filter", f"label=com.docker.compose.service={service}", + "--filter", "label=com.docker.compose.oneoff=False").splitlines() + check(len(ids) == 1, f"expected one container for {service}") + return ids[0] + + +def action(verb, *services): + ids = [container(service) for service in services] + print(f"docker {verb} {' '.join(services)} containers={','.join(ids)}") + extra = ("--time", "2") if verb in {"stop", "restart"} else () + docker(verb, *extra, *ids) + + +def inspect(service, fmt): + return docker("inspect", "--format", fmt, container(service)) + + +def inside(service, *args): + return docker("exec", container(service), *args) + + +def psql_args(): + return ("psql", "-XAt", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger") + + +def sql(statement): + return inside("postgres", *psql_args(), "-c", statement) + + +@contextmanager +def block_variant_insert(): + # Compatible with metadata SELECT, blocks the worker's final INSERT only. + args = ["docker", "exec", "-i", container("postgres"), *psql_args()] + proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + try: + proc.stdin.write("BEGIN; LOCK attachment_variants IN SHARE MODE; SELECT 'locked';\n") + proc.stdin.flush() + for _ in range(3): + if proc.stdout.readline().strip() == "locked": + break + else: + raise Failure("could not lock variants") + yield + finally: + try: + proc.communicate("COMMIT;\n", timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + raise Failure("could not release variants lock") from None + check(proc.returncode == 0, "variant lock session failed") + + +def eventually(description, fn, seconds=45, interval=0.2): + start = time.monotonic() + last = "" + while time.monotonic() - start < seconds: + try: + value = fn() + if value: + print(f"{description} after {time.monotonic()-start:.3f}s") + return value + except Failure as exc: + last = str(exc) + time.sleep(interval) + raise Failure(f"timeout waiting for {description}: {last}") + + +def ready(client): + eventually("LB ready", lambda: client.call("GET", "/health/ready", schema="Health")) + + +def metadata(client, actor, attachment): + path = f"/api/v1/attachments/{attachment['id']}" + return client.call("GET", path, actor=actor, schema="Attachment")["json"] + + +def expect_unavailable(client, actor, attachment): + path = f"/api/v1/attachments/{attachment['id']}/content" + for variant in ("original", "thumbnail"): + client.call("GET", f"{path}?variant={variant}", 409, actor=actor) + + +def upload(client, actor, chat, data=SAMPLE, mime="image/png", expected=202, filename="fixture"): + response = client.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={filename}", + expected, actor=actor, raw=data, headers={"Content-Type": mime}, + schema="Attachment" if expected == 202 else None) + if expected == 202: + row = response["json"] + check(row["status"] == "queued" and row["variants"] == [] and row["error_code"] is None, + "expected queued snapshot") + check((row["size_bytes"], row["sha256"], row["content_type"]) == + (len(data), hashlib.sha256(data).hexdigest(), mime), "staging metadata mismatch") + return response + + +def assert_ready(client, actor, attachment, data, dimensions, mime="image/png"): + start = time.monotonic() + row = client.wait_ready(actor, attachment) + check(row["error_code"] is None and len(row["variants"]) == 1, "invalid ready result") + check((row["content_type"], row["size_bytes"], row["sha256"]) == + (mime, len(data), hashlib.sha256(data).hexdigest()), "original metadata changed") + content_path = f"/api/v1/attachments/{attachment['id']}/content" + original = client.call("GET", content_path, actor=actor) + check(original["bytes"] == data and original["headers"].get_content_type() == mime, + "original content changed") + thumb = client.call("GET", content_path + "?variant=thumbnail", actor=actor) + check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail HTTP MIME") + variant = row["variants"][0] + check(variant["name"] == "thumbnail" and variant["content_type"] == "image/jpeg" + and (variant["width"], variant["height"]) == dimensions, "thumbnail metadata") + check((variant["sha256"], variant["size_bytes"]) == + (hashlib.sha256(thumb["bytes"]).hexdigest(), len(thumb["bytes"])), "thumbnail checksum") + with Image.open(BytesIO(thumb["bytes"])) as im: + im.load() + check(im.format == "JPEG" and im.size == dimensions and not im.getexif(), + "invalid thumbnail format, dimensions or EXIF") + pixel = im.getpixel((0, 0)) + print(f"ready id={row['id']} wait={time.monotonic()-start:.3f}s original_sha256={row['sha256']} " + f"jpeg={dimensions[0]}x{dimensions[1]} thumb_sha256={variant['sha256']} " + f"read={original['headers']['X-Instance-Id']}/{thumb['headers']['X-Instance-Id']}") + return row, pixel + + +def job_state(attachment): + aid = UUID(attachment["id"]) + return sql(f"""SELECT a.status,j.attempt,COALESCE(j.last_error,''),j.max_attempts + FROM attachments a JOIN attachment_jobs j ON a.id=j.attachment_id + WHERE a.id='{aid}'""") + + +def private(service, key): + status = inside(service, "python", "-c", """ +import sys +from urllib.error import HTTPError +from urllib.request import urlopen +from src import s3 +try: + urlopen(f"http://s3:9000/{s3.BUCKET}/{sys.argv[1]}", timeout=3).close() +except HTTPError as error: + print(error.code) +else: + print(200) +""", key) + check(status == "403", "unsigned S3 request was not rejected") + print("private RustFS object unsigned GET HTTP 403") + + +def queued_carrier(): + # Observe the real persistent AMQP message and put it back without ack. + raw = inside("api1", "python", "-c", """ +import json +from src import jobs +with jobs.broker() as channel: + method, props, body = channel.basic_get(jobs.QUEUE, auto_ack=False) + if method: + channel.basic_nack(method.delivery_tag, requeue=True) + print(json.dumps({"delivery_mode":props.delivery_mode, "job":json.loads(body)})) +""") + check(bool(raw), "no queued job in RabbitMQ") + observed = json.loads(raw) + check(observed["delivery_mode"] == 2, "AMQP message is not persistent") + carrier = observed["job"] + Draft202012Validator(JOB_SCHEMA, format_checker=FormatChecker()).validate(carrier) + print(f"RabbitMQ persistent carrier schema valid job={carrier['job_id']} " + f"attempt={carrier['attempt']} fields={','.join(sorted(carrier))}") + return carrier + + +def queue_stats(): + text = inside("broker", "rabbitmqctl", "-q", "list_queues", + "name", "durable", "messages_ready", "messages_unacknowledged") + for line in text.splitlines(): + fields = line.split() + if fields and fields[0] == "attachment.process.v1": + check(fields[1] == "true", "queue durability is off") + print(f"broker queue={fields[0]} durable={fields[1]} ready={fields[2]} unacked={fields[3]}") + return tuple(map(int, fields[2:])) + raise Failure("RabbitMQ queue absent") + + +def duplicate(client, actor, attachment, carrier, ready_snapshot): + inside("api1", "python", "-c", """ +import sys +from src import jobs +job = jobs.parse(sys.argv[1]) +with jobs.broker() as channel: + jobs.publish(channel, job) + jobs.publish(channel, job) +""", json.dumps(carrier)) + job_id = carrier["job_id"] + query = ('{service_name="messenger-worker"} | event="job.outcome" ' + f'| job_id="{job_id}" | attempt="1" | outcome="duplicate_ready"') + + def duplicates_logged(): + response = requests.get("http://loki:3100/loki/api/v1/query_range", + params={"query": query, "limit": 10}, timeout=5) + check(response.ok, "Loki duplicate query failed") + rows = response.json()["data"]["result"] + return sum(len(row["values"]) for row in rows) >= 2 + + eventually("worker logged two ready duplicates in Loki", duplicates_logged) + eventually("worker acknowledged all messages", lambda: queue_stats()[:2] == (0, 0)) + check(metadata(client, actor, attachment) == ready_snapshot, "duplicate changed ready metadata") + aid = UUID(attachment["id"]) + check(sql(f"SELECT count(*) FROM attachment_variants WHERE attachment_id='{aid}'") == "1", + "duplicate created another variant row") + count = inside("api1", "python", "-c", """ +import sys +from src import s3 +result = s3.client.list_objects_v2(Bucket=s3.BUCKET, Prefix=f"variants/{sys.argv[1]}/") +print(result["KeyCount"]) +""", str(aid)) + check(count == "1", "duplicate created another variant object") + print("PASS duplicate delivery: ready snapshot unchanged, one SQL row and one S3 variant") + + +def restart_scenarios(c, alice, bob, eve, chat): + action("stop", "worker") + try: + a = upload(c, alice, chat)["json"] + carrier = queued_carrier() + check(carrier["attachment_id"] == a["id"] and carrier["attempt"] == 1, + "RabbitMQ job identity differs from upload") + b = upload(c, bob, chat)["json"] + check(a["id"] != b["id"], "distinct uploads reused ID") + key = uuid4().hex + with ThreadPoolExecutor(max_workers=8) as pool: + calls = list(pool.map(lambda _: c.message(bob, chat, "", key, [a["id"]]), range(8))) + check(len({x["json"]["id"] for x in calls}) == 1, "idempotency race produced duplicates") + observed = {x["headers"]["X-Instance-Id"] for x in calls} + check(observed == {"api1", "api2"}, "idempotency requests did not use both replicas") + check(calls[0]["json"]["attachment_ids"] == [a["id"]], "queued attachment link lost") + c.message(alice, chat, "ordinary text with worker stopped") + check(metadata(c, bob, a) == a, "queued metadata changed with worker stopped") + expect_unavailable(c, bob, a) + private("api1", f"staging/{a['id']}") + print(f"PASS stopped worker: uploads=202 ids={a['id']},{b['id']}; queued content=409; " + "linked and ordinary POST=201; concurrent same-key response identical via api1/api2") + + starts = {s: inspect(s, "{{.State.StartedAt}}") for s in ("api1", "api2")} + action("restart", "api1", "api2") + ready(c) + for service, old in starts.items(): + new = inspect(service, "{{.State.StartedAt}}") + check(new != old, f"{service} did not actually restart") + print(f"{service} started before={old} after={new}") + check(metadata(c, bob, a) == a and job_state(a).startswith("queued|1|"), + "accepted job lost after API restart") + + volume_format = "{{range .Mounts}}{{if eq .Destination \"/var/lib/rabbitmq\"}}{{.Name}}{{end}}{{end}}" + volume = inspect("broker", volume_format) + check(volume == f"{PROJECT}_broker_data", "unexpected RabbitMQ volume") + check(queue_stats() == (2, 0), "expected two pending jobs before broker restart") + broker_start = inspect("broker", "{{.State.StartedAt}}") + action("restart", "broker") + eventually("broker healthy", lambda: inspect("broker", "{{.State.Health.Status}}") == "healthy") + broker_after = inspect("broker", "{{.State.StartedAt}}") + check(broker_start != broker_after and inspect("broker", volume_format) == volume, + "broker restart or volume identity mismatch") + print(f"broker started before={broker_start} after={broker_after} volume={volume}") + check(queue_stats() == (2, 0), "broker restart lost accepted jobs") + check(metadata(c, bob, b) == b, "second job metadata lost") + print("PASS both APIs + RabbitMQ restarted with queued work and unchanged RabbitMQ volume") + + path = f"/api/v1/attachments/{a['id']}" + c.call("GET", path, 401) + c.call("GET", path + "/content?variant=thumbnail", 404, actor=eve) + with block_variant_insert(): + action("start", "worker") + processing = eventually( + "HTTP processing state", + lambda: next((row for row in (a, b) if metadata(c, bob, row)["status"] == "processing"), None), + 6, interval=0.05, + ) + expect_unavailable(c, bob, processing) + c.message(bob, chat, "linked during processing", attachments=[processing["id"]]) + print("PASS processing metadata visible, content=409, linking=201 before final SQL commit") + a_ready, _ = assert_ready(c, bob, a, SAMPLE, (256, 160)) + assert_ready(c, alice, b, SAMPLE, (256, 160)) + duplicate(c, bob, a, carrier, a_ready) + private("api1", f"originals/{a['id']}") + c.call("GET", path + "/content?variant=unknown", 404, actor=bob) + finally: + action("start", "worker") + + +def image_bytes(image, fmt, **kwargs): + data = BytesIO() + image.save(data, fmt, **kwargs) + return data.getvalue() + + +def input_and_image_cases(c, alice, eve, chat): + for data, mime, status, actor, filename in ( + (SAMPLE, "image/jpeg", 415, alice, "wrong-mime"), + (SAMPLE[:-8], "image/png", 415, alice, "corrupt"), + (b"x" * (10 * 1024 * 1024 + 1), "image/png", 413, alice, "oversize"), + (SAMPLE, "image/png", 404, eve, "private"), + (SAMPLE, "image/png", 401, None, "anonymous"), + (SAMPLE, "image/png", 400, alice, "invalid%2Fname.png"), + ): + upload(c, actor, chat, data, mime, status, filename) + print(f"negative upload {filename} HTTP {status}") + huge = image_bytes(Image.new("1", (8001, 5000)), "PNG") + upload(c, alice, chat, huge, expected=413) + print("negative upload 40,005,000 decoded pixels HTTP 413") + + exif = Image.Exif() + exif[274] = 6 + original = image_bytes(Image.new("RGB", (319, 101), "navy"), "JPEG", exif=exif) + broken = original[:100] + upload(c, alice, chat, broken, mime="image/jpeg", expected=415) + print("negative truncated JPEG HTTP 415") + oriented = upload(c, alice, chat, original, "image/jpeg")["json"] + assert_ready(c, alice, oriented, original, (81, 256), "image/jpeg") + + original = image_bytes(Image.new("RGBA", (37, 19), (255, 0, 0, 0)), "PNG") + alpha = upload(c, alice, chat, original)["json"] + _, pixel = assert_ready(c, alice, alpha, original, (37, 19)) + check(min(pixel) >= 250, "transparent pixel was not composited on white") + print(f"PASS true JPEG, EXIF orientation, floor rounding, no upscale, alpha pixel={pixel}") + + +def failure_cases(c, alice, bob, chat): + action("stop", "worker") + try: + a = upload(c, alice, chat)["json"] + action("stop", "s3") + action("start", "worker") + state = eventually("persistent retry after real S3 stop", lambda: + (s := job_state(a)) and "storage_unavailable" in s and s) + print(f"storage outage SQL status|attempt|reason|max={state}") + finally: + action("start", "s3", "worker") + ready(c) + assert_ready(c, bob, a, SAMPLE, (256, 160)) + check(int(job_state(a).split("|")[1]) >= 2, "no durable retries recorded") + print("PASS temporary S3 outage recovered the same attachment") + + action("stop", "worker") + try: + missing = upload(c, alice, chat)["json"] + corrupt = upload(c, alice, chat)["json"] + inside("api1", "python", "-c", """ +from io import BytesIO +import sys +from src import s3 +s3.delete(f"staging/{sys.argv[1]}") +damaged = BytesIO(b"X" * int(sys.argv[3])) +s3.put(f"staging/{sys.argv[2]}", damaged, damaged.getbuffer().nbytes, "image/png") +""", missing["id"], corrupt["id"], str(corrupt["size_bytes"])) + finally: + action("start", "worker") + for row, error in ((corrupt, "invalid_image"), (missing, "retry_exhausted")): + failed = eventually(f"failed {row['id']}", lambda: + (meta := metadata(c, bob, row))["status"] == "failed" and meta) + check(failed["error_code"] == error and failed["variants"] == [], "unsafe failed result") + state = job_state(row) + status, current, _, maximum = state.split("|") + check(status == "failed" and int(current) == (1 if error == "invalid_image" else int(maximum)), + "unexpected number of attempts") + expect_unavailable(c, bob, row) + c.call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, actor=bob, + body={"text": "", "attachment_ids": [row["id"]]}, + headers={"Idempotency-Key": uuid4().hex}) + print(f"PASS safe failure code={error}; status|attempt|reason|max={state}; content/link HTTP 409") + + action("stop", "broker") + try: + response = upload(c, alice, chat, expected=503) + check(response["json"]["error"]["code"] == "unavailable", "unexpected queue-down response") + c.message(alice, chat, "normal API works when broker is stopped") + print("PASS unavailable broker upload=503 (no false 202), ordinary message=201") + finally: + action("start", "broker") + eventually("broker healthy after failure case", lambda: + inspect("broker", "{{.State.Health.Status}}") == "healthy") + + +def replica_cases(c, alice, bob, chat): + for stopped, survivor in (("api1", "api2"), ("api2", "api1")): + action("stop", stopped) + try: + eventually(f"{survivor} handles existing Bearer with {stopped} stopped", lambda: + c.call("GET", "/api/v1/users/me", actor=bob, schema="User")["headers"] + ["X-Instance-Id"] == survivor) + response = upload(c, alice, chat) + check(response["headers"]["X-Instance-Id"] == survivor, "wrong upload replica") + assert_ready(c, bob, response["json"], SAMPLE, (256, 160)) + finally: + action("start", stopped) + eventually(f"{stopped} container healthy", lambda: + inspect(stopped, "{{.State.Health.Status}}") == "healthy") + print("PASS authorized new uploads and thumbnail reads with either replica stopped") + + +def main(): + check(re.fullmatch(r"mtusi-lab6-grade3(?:-[a-z0-9-]+)?", PROJECT), "unsafe project name") + c = Client("http://lb:8080") + ready(c) + alice, bob, eve, chat = c.bootstrap() + alice_id = UUID(alice["id"]) + check(sql(f"SELECT password_hash LIKE '$argon2id$%' FROM users WHERE id='{alice_id}'") == "t", + "stored verifier is not Argon2id") + print("PASS Argon2id verifier confirmed without reading hash") + restart_scenarios(c, alice, bob, eve, chat) + input_and_image_cases(c, alice, eve, chat) + failure_cases(c, alice, bob, chat) + replica_cases(c, alice, bob, chat) + check(queue_stats() == (0, 0), "queue did not drain") + print("PASS Lab 5 grade 3 Docker scenarios") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/observability.py b/student-tests/observability.py new file mode 100644 index 0000000..f4f5bae --- /dev/null +++ b/student-tests/observability.py @@ -0,0 +1,405 @@ +"""Exercise real OTLP storage, Grafana and two isolated Compose incidents.""" + +import base64 +import json +import math +import re +import time +from pathlib import Path +from uuid import uuid4 + +import requests +from PIL import Image + +from grade3 import ( + Client, Failure, action, assert_ready, check, docker, eventually, inspect, + image_bytes, metadata, ready, run, upload, +) + +PROM = "http://prometheus:9090" +LOKI = "http://loki:3100" +TEMPO = "http://tempo:3200" +GRAFANA = "http://grafana:3000" +SERVICES = r'{service_name=~"messenger-api|messenger-worker"}' +DASH = "lab6-messenger" +SQUARE = image_bytes(Image.new("RGB", (320, 320), "navy"), "PNG") +SESSION = requests.Session() +AUTH = None +START = 0 +TRACE_RECORDS = [] +METRIC_RECORDS = [] + + +def http(url, params=None, payload=None): + try: + method = "POST" if payload is not None else "GET" + response = SESSION.request(method, url, params=params, json=payload, timeout=8, + auth=AUTH if url.startswith(GRAFANA + "/") else None) + check(response.ok, f"{method} {url}: HTTP {response.status_code}") + return response + except requests.RequestException as exc: + raise Failure(f"backend connection: {url} ({type(exc).__name__})") from None + + +def data(url, params=None, payload=None): + return http(url, params, payload).json() + + +def report(event, **fields): + print(json.dumps({"evidence": event, **fields}, ensure_ascii=False, sort_keys=True)) + + +def prom(query, when=None, base=PROM): + params = {"query": query} + if when is not None: + params["time"] = when + response = data(base + "/api/v1/query", params) + check(response["status"] == "success", "Prometheus query failed") + return response["data"]["result"] + + +def loki(query, base=LOKI, end=None, limit=1000): + response = data(base + "/loki/api/v1/query_range", { + "query": query, "start": START, "end": end or time.time_ns(), "limit": limit, + }) + check(response["status"] == "success", "Loki query failed") + return [(ts, body, series["stream"] | (extra[0] if extra else {})) + for series in response["data"]["result"] for ts, body, *extra in series["values"]] + + +def value(item): + return item["value"][1] + + +def trace_hex(value, length): + if re.fullmatch(r"[0-9a-f]{" + str(length) + "}", value or ""): + return value + return base64.b64decode(value or "").hex() + + +def get_span(trace_id, span_id, service, kind, base=TEMPO): + payload = data(base + "/api/traces/" + trace_id) + for batch in payload.get("batches", payload.get("resourceSpans", [])): + attrs = {a["key"]: a["value"] for a in batch["resource"]["attributes"]} + if attrs["service.name"]["stringValue"] != service: + continue + check(attrs["service.version"]["stringValue"] == "1.6.0", "incorrect service version") + check(attrs["deployment.environment.name"]["stringValue"] == "lab", "incorrect environment") + check(attrs["service.instance.id"]["stringValue"] in {"api1", "api2", "worker"}, "bad instance") + for scope in batch.get("scopeSpans", []): + for span in scope["spans"]: + if trace_hex(span["spanId"], 16) != span_id: + continue + check(trace_hex(span["traceId"], 32) == trace_id, "trace ID mismatch") + check(span["kind"] in (kind, {2: "SPAN_KIND_SERVER", 5: "SPAN_KIND_CONSUMER"}[kind]), + "unexpected span kind") + duration = int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) + check(duration > 0, "non-positive span duration") + TRACE_RECORDS.append(payload) + return {"trace_id": trace_id, "span_id": span_id, "duration_ns": duration, + "service": service, "resource": attrs, "span": span} + raise Failure("Tempo did not return matching service/span") + + +def correlate(label, selector, service, kind, key, val, outcome, status): + query = '{service_name="' + service + '"}' + selector + + def linked(): + rows = loki(query) + if not rows: + return None + timestamp, body, fields = rows[0] + trace_id, span_id = fields["trace_id"], fields["span_id"] + check(re.fullmatch("[0-9a-f]{32}", trace_id) and int(trace_id, 16), "invalid Loki trace ID") + check(re.fullmatch("[0-9a-f]{16}", span_id) and int(span_id, 16), "invalid Loki span ID") + check(fields.get("severity_text") in {"INFO", "WARN", "WARNING", "ERROR"} + and fields["event"] == body, "missing structured log fields") + record = get_span(trace_id, span_id, service, kind) + span = record["span"] + attrs = {a["key"]: a["value"] for a in span["attributes"]} + check(attrs[key]["stringValue"] == val and attrs["outcome"]["stringValue"] == outcome, + "different event in Tempo") + check(span["status"]["code"] in (status, {1: "STATUS_CODE_OK", 2: "STATUS_CODE_ERROR"}[status]), + "incorrect trace status") + check(int(span["startTimeUnixNano"]) <= int(timestamp) <= int(span["endTimeUnixNano"]), + "log timestamp outside span") + return {"query": query, "log": [timestamp, body, fields], "tempo": record} + + result = eventually(label, linked, seconds=45, interval=1) + report(label, **result) + return result + + +def check_metrics(): + allowed = { + "messenger_http_requests_total": {"method", "route", "status_class"}, + "messenger_http_duration_seconds_sum": {"method", "route", "status_class"}, + "messenger_http_duration_seconds_count": {"method", "route", "status_class"}, + "messenger_http_duration_seconds_bucket": {"method", "route", "status_class", "le"}, + "messenger_job_outcomes_total": {"job_type", "outcome"}, + "messenger_job_duration_seconds_sum": {"job_type", "outcome"}, + "messenger_job_duration_seconds_count": {"job_type", "outcome"}, + "messenger_job_duration_seconds_bucket": {"job_type", "outcome", "le"}, + } + paths = json.loads(Path("contracts/openapi.json").read_text())["paths"] + metrics = {} + for name, labels in allowed.items(): + rows = prom(name) + check(bool(rows), f"no series for {name}") + expected_service = "messenger-api" if "http" in name else "messenger-worker" + for row in rows: + fields = row["metric"] + check(set(fields) == labels | {"__name__", "job", "instance"}, f"unexpected labels on {name}") + check(fields["job"] == expected_service, "unexpected Prometheus service label") + check(fields["instance"] in ({"api1", "api2"} if "http" in name else {"worker"}), + "unexpected instance label") + if "route" in fields: + check(fields["route"] in paths or fields["route"] == "unmatched", "raw route label") + else: + check(fields["job_type"] == "attachment.process.v1", "unknown job type") + if "http" in name: + required = [row for row in rows if row["metric"]["route"] == "/api/v1/attachments/{attachment_id}/content"] + check(all(any(r["metric"]["status_class"] == status and float(value(r)) > 0 + for r in required) for status in ("2xx", "5xx")), f"missing HTTP series for {name}") + else: + check(all(any(r["metric"]["outcome"] == outcome and float(value(r)) > 0 + for r in rows) for outcome in ("success", "fail")), f"missing job series for {name}") + if "bucket" in name: + check({0.005, 0.25, 60, math.inf} <= {float(r["metric"]["le"]) for r in rows}, "missing histogram buckets") + METRIC_RECORDS.extend(rows) + metrics[name] = rows + # Keep the verified dynamic routes and an actual bucket sample in review evidence. + for name, rows in metrics.items(): + samples = [r for r in rows if r["metric"].get("le", "+Inf") in ("0.005", "+Inf") + and r["metric"].get("route", "/api/v1/attachments/{attachment_id}/content") + == "/api/v1/attachments/{attachment_id}/content"] + report("prometheus", query=name, checked_series=len(rows), samples=samples) + return metrics + + +def check_grafana(api, worker): + sources = data(GRAFANA + "/api/datasources") + for kind in ("prometheus", "loki", "tempo"): + uid = "lab6-" + kind + source = next((s for s in sources if s["uid"] == uid and s["type"] == kind), None) + check(source is not None, "Grafana datasource not provisioned") + if kind == "tempo": + # The built-in Tempo plugin in this Grafana version has no + # plugin health handler. Prove readiness and real trace queries via proxy. + status = http(GRAFANA + f"/api/datasources/proxy/uid/{uid}/ready") + health = {"http_status": status.status_code, "body": status.text.strip()} + else: + health = data(GRAFANA + f"/api/datasources/uid/{uid}/health") + check(health["status"].lower() == "ok", "Grafana datasource unhealthy") + report("grafana.datasource", uid=uid, type=kind, url=source["url"], health=health) + dashboards = data(GRAFANA + "/api/dashboards/uid/" + DASH) + panels = dashboards["dashboard"]["panels"] + check(len(panels) == 6, "expected six Grafana panels") + start_ms = str(int(START / 1_000_000) - 30_000) + end_ms = str(time.time_ns() // 1_000_000) + for panel in panels: + target = panel["targets"][0] + ds = panel["datasource"] + if ds["type"] == "prometheus": + query = target | {"datasource": ds, "intervalMs": 5000, "maxDataPoints": 120, "range": True} + response = data(GRAFANA + "/api/ds/query", payload={ + "from": start_ms, "to": end_ms, "queries": [query], + })["results"]["A"] + check(not response.get("error"), "Grafana panel query failed") + frames = response.get("frames", []) + values = [v for f in frames for v in f["data"]["values"][-1] + if isinstance(v, (int, float)) and math.isfinite(v)] + check(values and any(v > 0 for v in values), f"no signal in Grafana panel {panel['id']}") + report("grafana.panel", id=panel["id"], title=panel["title"], query=target["expr"], + frames=len(frames), numeric_samples=len(values), max_value=max(values)) + else: + rows = loki(target["expr"], base=GRAFANA + "/api/datasources/proxy/uid/lab6-loki") + check({fields["service_name"] for _, _, fields in rows} >= {"messenger-api", "messenger-worker"}, + "Grafana Loki query missing a service") + report("grafana.panel", id=panel["id"], title=panel["title"], query=target["expr"], rows=len(rows)) + link = next(s for s in sources if s["uid"] == "lab6-loki")["jsonData"]["derivedFields"][0] + check(link["datasourceUid"] == "lab6-tempo" and link["matcherRegex"] == "trace_id" + and link["url"] == "${__value.raw}", "Grafana Loki-to-Tempo link not provisioned") + for linked in (api, worker): + trace = linked["tempo"] + result = get_span(trace["trace_id"], trace["span_id"], trace["service"], + 2 if trace["service"] == "messenger-api" else 5, + base=GRAFANA + "/api/datasources/proxy/uid/lab6-tempo") + report("grafana.trace", trace_id=result["trace_id"], span_id=result["span_id"]) + + + + +def incidents(client, alice, chat, filename): + from grade3 import SERVICES as compose_services + compose_services.update({"prometheus", "loki", "tempo", "grafana", "otel-collector"}) + direct = Client("http://api1:8080", timeout=5) + + started = time.monotonic() + action("stop", "worker") + try: + queued = upload(client, alice, chat, data=SQUARE, filename=filename)["json"] + check(metadata(client, alice, queued)["status"] == "queued", "worker was expected to be stopped") + check(inspect("worker", "{{.State.Running}}") == "false", "worker still running") + report("worker.stop", accepted=202, attachment_id=queued["id"], status="queued") + finally: + action("start", "worker") + assert_ready(client, alice, queued, SQUARE, (256, 256)) + report("worker.recovered", attachment_id=queued["id"], status="ready", + seconds=round(time.monotonic() - started, 3)) + + action("stop", "worker") + failure_started = time.monotonic() + try: + failing = upload(client, alice, chat, data=SQUARE, filename=filename)["json"] + action("stop", "s3") + action("start", "worker") + path = f"/api/v1/attachments/{queued['id']}/content" + error = direct.call("GET", path, 503, actor=alice) + request_id = error["headers"]["X-Request-Id"] + # More than one scrape separates some errors, so rate() has a delta. + direct.call("GET", path, 503, actor=alice) + direct.call("GET", path, 503, actor=alice) + + def failed(): + state = metadata(direct, alice, failing) + return state if state["status"] == "failed" else None + + failed_state = eventually("terminal S3 outage", failed, seconds=55, interval=0.7) + check(failed_state["error_code"] == "retry_exhausted", "expected retry budget exhaustion") + report("s3.outage", http_status=503, request_id=request_id, attachment_id=failing["id"], + status="failed", error_code=failed_state["error_code"], + seconds=round(time.monotonic() - failure_started, 3)) + finally: + action("start", "s3", "worker") + ready(client) + assert_ready(client, alice, queued, SQUARE, (256, 256)) + new_attachment = upload(client, alice, chat, data=SQUARE, filename=filename)["json"] + assert_ready(client, alice, new_attachment, SQUARE, (256, 256)) + report("s3.recovered", existing=queued["id"], new=new_attachment["id"], + seconds=round(time.monotonic() - failure_started, 3)) + return queued["id"], failing["id"], request_id + + +def verify_safe(actors, marker, filename, headers): + logs = loki(SERVICES + ' | event != ""', limit=4000) + literals = [marker, filename, b"\x89PNG".hex()] + for actor in actors: + registration = actor["registration"] + password = registration["password"] + literals.extend((actor["token"], password, + base64.b64encode(f"{registration['username']}:{password}".encode()).decode())) + literals.extend(headers) + # Inspect raw backend responses in memory and emit only counts. + blob = json.dumps([logs, METRIC_RECORDS, TRACE_RECORDS], ensure_ascii=False) + matches = sum(blob.count(secret) for secret in literals) + patterns = re.compile( + r'(?i)\b(?:authorization|secret_access_key|x-amz-signature|password|bearer)\b' + r'|-----BEGIN [A-Z ]*PRIVATE KEY-----|iVBORw0KGgo|/9j/' + r'|/api/v1/(?:chats|attachments)/[0-9a-f]{8}-' + ) + matches += len(patterns.findall(blob)) + check(matches == 0, "sensitive data found in backend responses") + report("telemetry.scan", loki_rows=len(logs), spans_queries=len(TRACE_RECORDS), + metric_samples=len(METRIC_RECORDS), matches=matches) + + +def restart_stores(api, worker): + services = ("prometheus", "loki", "tempo", "grafana") + + def starts(): + return {service: inspect(service, "{{.State.StartedAt}}") for service in services} + + def volumes(): + return {service: inspect(service, '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{end}}{{end}}') + for service in services} + + before = starts() + mounts = volumes() + check(all(mounts.values()), "missing named backend volume") + when = time.time() + end = time.time_ns() + query = 'sum(messenger_job_outcomes_total{outcome="success"})' + historical = prom(query, when) + check(historical and float(value(historical[0])) > 0, "missing historical sample") + action("restart", *services) + + def same_history(): + check(prom(query, when) == historical, "Prometheus historical sample changed") + for linked in (api, worker): + check(tuple(linked["log"]) in loki(linked["query"], end=end), "Loki lost control event") + trace = linked["tempo"] + repeat = get_span(trace["trace_id"], trace["span_id"], trace["service"], + 2 if trace["service"] == "messenger-api" else 5) + check(repeat["span"] == trace["span"], "Tempo lost control span") + return data(GRAFANA + "/api/dashboards/uid/" + DASH)["dashboard"]["uid"] == DASH + + eventually("same samples, logs, spans and dashboard after restart", same_history, seconds=80, interval=1) + after = starts() + check(all(before[s] != after[s] for s in services) and volumes() == mounts, + "unexpected process or volume after restart") + report("storage.restart", before=before, after=after, volumes=mounts, historical_query=query, + unix_time=when, historical_samples=historical, + trace_ids=[api["tempo"]["trace_id"], worker["tempo"]["trace_id"]], dashboard=DASH) + + +def main(): + global AUTH, START + AUTH = ("admin", Path("/run/secrets/grafana_admin_password").read_text().strip()) + START = time.time_ns() + for url in (PROM + "/-/ready", LOKI + "/ready", TEMPO + "/ready", GRAFANA + "/api/health", + "http://otel-collector:13133/"): + eventually(url, lambda: http(url), seconds=75, interval=1) + client = Client(timeout=10) + alice, bob, eve, chat = client.bootstrap() + marker = "message-canary-" + uuid4().hex + filename = "filename-canary-" + uuid4().hex + ".png" + header_canary = "header-canary-" + uuid4().hex + created = client.message(alice, chat, marker, key=header_canary) + api_id = created["headers"]["X-Request-Id"] + check(created["status"] == 201, "POST must succeed") + denied = client.call("GET", "/api/v1/users/me", 401) + request_401 = denied["headers"]["X-Request-Id"] + started = time.monotonic() + for index in range(48): + client.call("GET", "/api/v1/users/me", actor=alice) + if index % 8 == 0: + client.call("GET", "/api/v1/users/me", 401) + time.sleep(0.14) + report("traffic", get=48, post_status=201, error_status=401, seconds=round(time.monotonic()-started, 3)) + success_id, failure_id, request_503 = incidents(client, alice, chat, filename) + + api = correlate("api.success", f' | request_id="{api_id}" | event="http.completed"', + "messenger-api", 2, "request.id", api_id, "2xx", 1) + correlate("api.401", f' | request_id="{request_401}" | event="http.completed"', + "messenger-api", 2, "request.id", request_401, "4xx", 1) + error = correlate("api.503", f' | request_id="{request_503}" | event="http.completed"', + "messenger-api", 2, "request.id", request_503, "5xx", 2) + check(any(a["key"] == "dependency.name" and a["value"]["stringValue"] == "s3" + for a in error["tempo"]["span"]["attributes"]), "S3 not diagnosed by API trace") + worker = correlate("worker.ready", f' | attachment_id="{success_id}" | event="job.outcome" | outcome="ready"', + "messenger-worker", 5, "attachment_id", success_id, "ready", 1) + retry = correlate("worker.retry", f' | attachment_id="{failure_id}" | event="job.outcome" | outcome="retry"', + "messenger-worker", 5, "attachment_id", failure_id, "retry", 2) + check(retry["log"][2]["reason"] == "storage_unavailable", "S3 not diagnosed by retry log") + correlate("worker.failed", f' | attachment_id="{failure_id}" | event="job.outcome" | outcome="failed"', + "messenger-worker", 5, "attachment_id", failure_id, "failed", 2) + eventually("all real counter and histogram series", check_metrics, seconds=30, interval=2) + check_grafana(api, worker) + verify_safe((alice, bob, eve), marker, filename, [header_canary, chat["id"]]) + restart_stores(api, worker) + action("stop", "otel-collector", "grafana") + try: + client.call("GET", "/health/ready", schema="Health") + client.call("GET", "/api/v1/users/me", actor=alice) + attachment = upload(client, alice, chat, data=SQUARE)["json"] + assert_ready(client, alice, attachment, SQUARE, (256, 256)) + report("telemetry.outage", health=200, authenticated_get=200, uploaded=202, + state="ready", disabled=["otel-collector", "grafana"]) + finally: + action("start", "otel-collector", "grafana") + print("PASS: real metrics/logs/traces, provisioned Grafana, incidents and persistent telemetry") + + +if __name__ == "__main__": + run(main) -- 2.54.0