From 54869eeb8f8b7b29324f74ec82c23f49f6247704 Mon Sep 17 00:00:00 2001 From: Xabenskie Date: Tue, 10 Nov 2026 18:09:59 +0300 Subject: [PATCH] [+] lab6 --- .dockerignore | 3 + .env.example | 33 +- .github/workflows/application.yml | 25 + Dockerfile | 18 + Makefile | 39 + REPORT.md | 113 +- compose.yaml | 238 +++ evidence/check.txt | 2 + evidence/concurrent-idempotency.json | 11 + evidence/db-alert-firing.json | 15 + evidence/db-alert-resolved.json | 1 + evidence/dependency-failures.json | 20 + evidence/fresh-migrations.txt | 70 + evidence/graceful-shutdown.json | 16 + evidence/grafana-provisioning.json | 22 + evidence/infrastructure.json | 341 ++++ evidence/jobs-faults.json | 27 + evidence/last-stop.json | 11 + evidence/nginx.json | 11 + evidence/observability.json | 342 ++++ evidence/openapi.txt | 2 + evidence/persistence.json | 680 +++++++ evidence/persistence.txt | 2 + evidence/public-failover.json | 15 + evidence/public-failover.txt | 3 + evidence/queue-alert-firing.json | 15 + evidence/queue-alert-resolved.json | 1 + evidence/rolling-update.json | 171 ++ evidence/sequential-failover.json | 135 ++ evidence/smoke.txt | 2 + evidence/student-tests.txt | 15 + evidence/upload-logs.json | 244 +++ evidence/upload-trace.json | 1584 +++++++++++++++++ infra/minio/Dockerfile | 28 + infra/minio/api-policy.json | 45 + infra/minio/bootstrap.sh | 9 + infra/minio/server.sh | 5 + infra/nginx/nginx.conf | 62 + infra/observability/alerts.yaml | 17 + infra/observability/collector.yaml | 80 + .../grafana/dashboards/messenger.json | 436 +++++ .../provisioning/dashboards/dashboards.yaml | 7 + .../provisioning/datasources/datasources.yaml | 28 + infra/observability/loki.yaml | 28 + infra/observability/prometheus.yaml | 9 + infra/observability/tempo.yaml | 21 + migrations/001_initial.sql | 34 + migrations/002_idempotency.sql | 11 + migrations/003_attachments.sql | 23 + migrations/004_jobs.sql | 19 + requirements.lock | 61 + requirements.txt | 15 + scripts/bootstrap.sh | 33 + scripts/check_fresh_migrations.sh | 16 + scripts/cluster.py | 196 ++ scripts/restart.sh | 4 + scripts/stop-one.sh | 5 + scripts/verify_cluster.py | 447 +++++ scripts/verify_jobs.py | 123 ++ scripts/verify_observability.py | 144 ++ src/messenger/__init__.py | 0 src/messenger/app.py | 323 ++++ src/messenger/attachments.py | 175 ++ src/messenger/cleanup.py | 72 + src/messenger/db.py | 62 + src/messenger/errors.py | 19 + src/messenger/jobs.py | 231 +++ src/messenger/messages.py | 54 + src/messenger/migrate.py | 29 + src/messenger/models.py | 75 + src/messenger/object_store.py | 123 ++ src/messenger/pagination.py | 56 + src/messenger/pipeline.py | 49 + src/messenger/processor.py | 33 + src/messenger/security.py | 49 + src/messenger/telemetry.py | 154 ++ src/messenger/worker.py | 77 + student-tests/test_api.py | 141 ++ student-tests/test_attachments.py | 187 ++ student-tests/test_idempotency.py | 119 ++ student-tests/test_jobs.py | 63 + student-tests/test_pipeline.py | 72 + student-tests/test_telemetry.py | 51 + 83 files changed, 8300 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/check.txt create mode 100644 evidence/concurrent-idempotency.json create mode 100644 evidence/db-alert-firing.json create mode 100644 evidence/db-alert-resolved.json create mode 100644 evidence/dependency-failures.json create mode 100644 evidence/fresh-migrations.txt create mode 100644 evidence/graceful-shutdown.json create mode 100644 evidence/grafana-provisioning.json create mode 100644 evidence/infrastructure.json create mode 100644 evidence/jobs-faults.json create mode 100644 evidence/last-stop.json create mode 100644 evidence/nginx.json create mode 100644 evidence/observability.json create mode 100644 evidence/openapi.txt create mode 100644 evidence/persistence.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/public-failover.json create mode 100644 evidence/public-failover.txt create mode 100644 evidence/queue-alert-firing.json create mode 100644 evidence/queue-alert-resolved.json create mode 100644 evidence/rolling-update.json create mode 100644 evidence/sequential-failover.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/upload-logs.json create mode 100644 evidence/upload-trace.json create mode 100644 infra/minio/Dockerfile create mode 100644 infra/minio/api-policy.json create mode 100644 infra/minio/bootstrap.sh create mode 100644 infra/minio/server.sh create mode 100644 infra/nginx/nginx.conf create mode 100644 infra/observability/alerts.yaml create mode 100644 infra/observability/collector.yaml create mode 100644 infra/observability/grafana/dashboards/messenger.json create mode 100644 infra/observability/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/observability/loki.yaml create mode 100644 infra/observability/prometheus.yaml create mode 100644 infra/observability/tempo.yaml create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_jobs.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100755 scripts/bootstrap.sh create mode 100755 scripts/check_fresh_migrations.sh create mode 100644 scripts/cluster.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/verify_cluster.py create mode 100644 scripts/verify_jobs.py create mode 100644 scripts/verify_observability.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/app.py create mode 100644 src/messenger/attachments.py create mode 100644 src/messenger/cleanup.py create mode 100644 src/messenger/db.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/jobs.py create mode 100644 src/messenger/messages.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/models.py create mode 100644 src/messenger/object_store.py create mode 100644 src/messenger/pagination.py create mode 100644 src/messenger/pipeline.py create mode 100644 src/messenger/processor.py create mode 100644 src/messenger/security.py create mode 100644 src/messenger/telemetry.py create mode 100644 src/messenger/worker.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_jobs.py create mode 100644 student-tests/test_pipeline.py create mode 100644 student-tests/test_telemetry.py diff --git a/.dockerignore b/.dockerignore index bc16e55..9c4c0b6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.superpowers +.pytest_cache diff --git a/.env.example b/.env.example index 2b32655..c8675d2 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,5 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# bootstrap generates .env and local files under ignored secrets/. SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Compose enables OTEL_ENABLED=1 and sends all three signals to collector:4318. +# Grafana operator password is generated in secrets/grafana_password. +# Fixed profile: APIx2; worker concurrency1/prefetch1; processing45s/lease90s; max5 attempts. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..dd1aac0 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,25 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + observability: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: make up + - run: make check validate-container test test-student test-fresh + - run: make test-jobs + - run: make test-cluster + - run: make test-observability + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab6-evidence + path: evidence/ + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..db3f0d2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.13.5-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt requirements-dev.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger +COPY src/ src/ +COPY migrations/ migrations/ +COPY student-tests/ student-tests/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +ARG APP_RELEASE=lab6-v1 +ENV APP_RELEASE=${APP_RELEASE} +LABEL org.opencontainers.image.version=${APP_RELEASE} +USER messenger +EXPOSE 8080 +CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"] diff --git a/Makefile b/Makefile index 5d86d17..1c2212f 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,42 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +.PHONY: bootstrap up down test-student test-cluster rolling +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + docker compose build api minio minio-init + docker compose up -d --wait --wait-timeout 120 + $(PYTHON) scripts/cluster.py wait +down: + docker compose down +test-student: + $(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +test-cluster: + $(PYTHON) scripts/verify_cluster.py +rolling: + $(PYTHON) scripts/cluster.py rolling + +.PHONY: test-fresh load-evidence verify-persistence verify-failover +test-fresh: + ./scripts/check_fresh_migrations.sh +load-evidence: + $(PYTHON) scripts/capture_load.py +verify-persistence: + $(PYTHON) scripts/verify_cluster.py persistence +verify-failover: + $(PYTHON) scripts/verify_cluster.py public-failover + $(PYTHON) scripts/verify_cluster.py failover + +.PHONY: validate-container +validate-container: + $(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py + + +.PHONY: test-jobs +test-jobs: + $(PYTHON) scripts/verify_jobs.py + +.PHONY: test-observability +test-observability: + $(PYTHON) scripts/verify_observability.py diff --git a/REPORT.md b/REPORT.md index b97ae42..8619738 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,114 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Лукьянов К.Р. БПИ2303 +- Целевая оценка: **4**; трек5 не заявлен. +- Commit/PR: заполняется автором после самостоятельного commit/push. +- Источники, библиотеки и помощь ИИ: FastAPI, Pydantic, Uvicorn, Psycopg, redis-py, boto3, Pillow, OpenTelemetry Python SDK1.34.1 (semantic conventions0.55b1), pytest. Документация [Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel metrics SDK](https://opentelemetry-python.readthedocs.io/en/stable/sdk/metrics.html), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker/Compose v2, Python3.9+, Make. Docker VM желательно выделить не менее8GiB RAM; ресурсы сервисов ограничены compose.yaml. Все зависимости поднимаются из репозитория. Остановить предыдущий стенд, сохранив volumes: используются те же localhost-порты. + +```sh +git clone +cd +make up +curl -fsS http://localhost:8080/health/ready +make check +make validate-container +make test +make test-student +make test-fresh +make test-jobs +make test-cluster +make test-observability +``` + +Основной endpoint: http://localhost:8080. Grafana: http://localhost:3000; пользователь operator, локальный пароль в secrets/grafana_password. Prometheus9090, Loki3100, Tempo3200 доступны только через127.0.0.1 для диагностики. Значения secrets не печатать. Dashboard и три datasource импортируются автоматически. make down сохраняет volumes, down --volumes намеренно удаляет данные текущей лабораторной. + +Bootstrap создаёт локальные credentials; API/MinIO/mc собираются Dockerfile. MinIO/mc — закреплённые официальные исходники с commit guard, как в lab5; registry-образы/бинарный архив были недоступны. Python и транзитивные зависимости закреплены requirements.lock. Go на хосте не нужен. First build требует registry/GitHub/package repositories. + +Отказные проверки запускать последовательно. test-fresh использует отдельный случайный project и удаляет его временные volumes; в этом отдельном тесте схемы SDK выключен, поскольку telemetry backend туда не поднимается. На основном стенде smoke/31 тест и эксперименты выполняются с telemetry on. Синтетические записи тестов остаются в общей БД. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Nginx → API×2 → PG / profile cache / MinIO staging + job/outbox + relay → Streams broker → worker/processor → S3 variants + PG ready +API / relay / worker / processor → OTLP HTTP → Collector + ├─ Prometheus: metrics + ├─ Loki: logs + └─ Tempo: traces + Grafana → все три источника +``` + +Бизнес-API lab5 сохранён: 202queued, durable staging/outbox, thumbnail/crop/WebP, ACL, общий Bearer, идемпотентность, retry/failed/replay и cleanup. Nginx остаётся балансировщиком. Публичные contracts/tests/README/COURSE текущей лабы не изменялись. Служебные документы — lab6/docs. + +Версии: Collector contrib0.128.0, Prometheus3.4.1, Loki3.5.1, Tempo2.7.2, Grafana12.0.1; PG17.5, Redis7.4.4, Nginx1.28.0. Backend volumes сохраняют историю (retention7d), это не HA и не backup. Collector — перенос/обработка, не долговременная БД; Grafana — интерфейс запросов. + +**Трассы и контекст.** Server-span API берёт W3C context. Persist сохраняет carrier в PG job, relay создаёт producer-span и inject в payload, worker extract создаёт consumer-span. Retry связан с исходным upload и имеет job.outcome/attempt; retry/failed помечается ERROR. Дочерний processor получает carrier через stdin, экспортирует реальные S3 spans и завершает exporter перед выходом. postgres.query/commit измеряют фактические SQL/COMMIT, не аренду connection с изображениями/S3. Текст SQL, параметры, storage key и credentials не экспортируются. + +**Логи.** JSON: timestamp, severity, service_name, event, trace_id/span_id и request_id для HTTP. Event attributes проходят allowlist. Body, Authorization, passwords/tokens, filenames и private keys исключены. X-Trace-Id — дополнительный диагностический заголовок, X-Request-Id сохраняется. API и worker логи найдены в Loki по trace_id. + +**Ресурсы.** service.name=messenger-api/messenger-worker/messenger-relay, service.version=lab6-v1, случайный service.instance.id, deployment.environment.name=lab. Processor наследует worker instance, не создаёт отдельную серию metrics на каждую job. Processor экспортирует traces/logs, metrics обработки остаются у долгоживущего worker. + +| Сигнал / имя Prometheus | Смысл / единицы | +| --- | --- | +| http_requests_total | количество попыток бизнес-запросов API; method/route template/status_class | +| http_server_duration_seconds_bucket | HTTP duration, seconds; p50/p95/p99 из histogram buckets всех API | +| jobs_processed_total | попытки обработки по job_type/outcome ready/retry/failed | +| job_processing_duration_seconds_bucket | длительность попытки, seconds | +| attachment_ready_seconds_bucket | accepted→ready, seconds | +| queue_depth / queue_oldest_age_seconds | незавершённые queued/processing, count/seconds; собирает relay | +| db_pool_used / db_pool_waiting | занятые соединения / ожидающие запросы | +| worker_busy / worker_capacity | текущая занятость / ёмкость одного worker | +| process_memory_bytes / process_cpu_seconds_total | RSS / накопленное процессорное время | + +Технические UUID не являются labels. Неизвестный route — unmatched, UUID заменяются шаблонами. Histogram buckets0.005–120s, ready0.1–300s; units отображаются в dashboard. HTTP latency фиксируется до отправки тела response, а не до завершения полного скачивания клиентом. Queue age — backlog, не processing duration. Пример p95: + +```promql +histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m]))) +``` + +**SLI/SLO и alerts.** SLI=1−5xx/все бизнес-попытки API, health исключены; клиентские4xx не считаются серверным отказом. Повтор Nginx может образовать две попытки API — это SLI backend, не точный счётчик пользовательских действий. Цель>=99% за30d. Короткий учебный прогон не доказывает достижение30-дневного SLO. Alerts: error ratio>5% в окне1m for15s; oldest job>15s for15s. Правила и локальный runbook в infra/observability/alerts.yaml, доставка наружу не настраивалась. + +**Ограничение telemetry.** SDK batch queues256, export batch32, задержка500ms, timeout1s; metrics interval2s. Collector memory_limiter192MiB/spike48MiB, container256MiB; batch128/max256, очереди exporter256 batches,2consumers, retry horizon20s. Неограниченного буфера нет. Prometheus exporter metric_expiration10s удаляет gauges ушедших replicas, не удерживая старый queue age5min. Backend outage может терять telemetry; запросы приложения не ждут его восстановления. + +**Восстановление очереди.** Consumer reclaim неподтверждённой доставки —5s, независимо от PG processing lease90s. Живой процесс защищён advisory lock и общим deadline45s. Это устраняет задержку повторной доставки после краткого DB-сбоя, сохраняя защиту от параллельного эффекта. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Прогоны2026-11-09, Docker Desktop/macOS Apple Silicon. Ветка solution/bpi2303-lukyanov, коммитов/push агент не делает. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Проверка | Фактический результат / evidence | +| --- | --- | +| check/OpenAPI/public smoke | PASS: check.txt, openapi.txt, smoke.txt | +| Собственные регрессии |31 тест PASS: student-tests.txt | +| Пустые volumes/повтор миграций |31 тест PASS: fresh-migrations.txt | +| Сохранность pipeline lab5 | jobs-faults.json: COMMIT/publish gap, broker restart, kill послеPUT, lease, retry/replay, duplicate PASS | +| Restart/idempotency/failover/SIGTERM/rolling | соответствующие JSON в evidence | +| DB/Redis отказ и ресурсы/порты | dependency-failures.json, infrastructure.json | +| Автоматический dashboard и три datasource | grafana-provisioning.json, dashboardJSON в infra/observability | +| Сквозная trace с retry и связанные API/worker logs | upload-trace.json, upload-logs.json, observability.json | +| Queue/DB incidents, firing→resolved | queue-alert-*.json, db-alert-*.json | +| Backend outage и bounded memory/queue | observability.json: сообщения успешны, экспортные counters/queue и dockerstats сохранены | + +Предупреждения botocore utcnow и диагностический Pillow warning не являются падениями assertions. Code review выявил неправильный DB lease-span и отсутствие outcome/error у consumer; обе регрессии сначала упали, затем прошли после исправления. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +1. **Upload+retry.** Worker остановлен, файл принят; MinIO недоступен во время первой обработки. После восстановления тот же attachment ready, trace связывает server→producer→consumer→processor→S3/DB, включая несколько попыток. Сохранено 3 попыток; лог API и worker коррелирует с trace_id. +2. **Worker stopped.** До воздействия alert разрешён; затем остановка worker, queued upload, queue age растёт, alert firing. После запуска ready и resolved. Queue metric продолжает работать через relay. +3. **DB contention.** Операторская сессия держит ACCESS EXCLUSIVE lock на sessions, readiness остаётся живым. Запросы users/me ждут lock_timeout и возвращают503; postgres.query spans и db_operation_failed logs указывают на DB, не decoder. 9 ошибочных вызовов, затем holder прекращён через pg_terminate_backend, здоровый трафик и alert resolved. Helper отключает только собственные idle/statement timeouts и гарантированно завершается после опыта. +4. **Loki+Tempo stopped.** За 26.571s успешно записано 2575 сообщений. Collector memory: 75.03MiB / 256MiB→76.7MiB / 256MiB при256MiB limit. Bounded retry/queues и реальные accepted/sent/failed/queue counters сохранены. Это короткий опыт, не доказательство нулевых потерь telemetry или отсутствия роста при бесконечной нагрузке. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Все15 dashboard expressions проверены реальными запросами Prometheus. Данные trace/log не статические fixtures. В ходе общего прогона после rolling один latency-check Redis fallback превысил3s; отдельная воспроизводящая проверка прошла без изменения контракта, затем dependency test повторён успешно. Кроме того, публичный smoke выявил90s pending-redelivery после DB outage: reclaim сокращён до5s, после чего повтор отказа и smoke прошли. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один host, backend на локальных volumes; нет HA/backup и обещания30-дневной доступности. +- End-user response LB может отсутствовать в SDK при исключении всех API; это не считается подтверждённым API-observation. SLI определён для обработанных бизнес-попыток API. +- Экспорт bounded best effort: при исчерпании очереди или retry horizon telemetry теряется. До устойчивого ingest/backend-volume гарантий сохранности span нет. +- HTTP instrumentation измеряет handler/headers; download streaming не является полным клиентским latency. +- HTTP и localhost админ-интерфейсы допустимы только в изолированном учебном окружении. TLS/mTLS и OpenBao вводятся в lab7. +- Исследовательские sampling/overhead/burn-rate треки5 не выполнялись. Наблюдения относятся к заданному профилю и синтетическим данным. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..7a7677c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,238 @@ +name: mtusi-lab6 +services: + postgres: + image: postgres:17.5-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + redis: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.25 + migrate: + image: mtusi-lab6-api:lab6 + command: [python, -m, messenger.migrate] + environment: + OTEL_ENABLED: '1' + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + restart: 'no' + mem_limit: 256m + cpus: 0.5 + api: + image: mtusi-lab6-api:lab6 + build: . + scale: 2 + environment: + OTEL_ENABLED: '1' + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + POOL_MAX_SIZE: '8' + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [front, data] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + stop_grace_period: 10s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"] + interval: 2s + timeout: 5s + retries: 30 + mem_limit: 512m + cpus: 1.0 + minio: + image: mtusi-lab6-minio:2025-04-22 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: minio + secrets: [minio_root_user, minio_root_password] + volumes: [minio_data:/data] + networks: [data] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 10s + mem_limit: 512m + cpus: 1.0 + minio-init: + image: mtusi-lab6-mc:2025-04-16 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: mc + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key] + volumes: + - ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro + - ./infra/minio/api-policy.json:/config/api-policy.json:ro + networks: [data] + depends_on: + minio: {condition: service_healthy} + restart: 'no' + mem_limit: 128m + cpus: 0.25 + broker: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction] + volumes: [broker_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 160m + cpus: 0.5 + worker: &background + image: mtusi-lab6-api:lab6 + command: [python, -m, messenger.worker, worker] + environment: + OTEL_ENABLED: '1' + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318 + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL: redis://broker:6379/0 + JOB_MAX_ATTEMPTS: '5' + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + broker: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + stop_grace_period: 45s + restart: unless-stopped + mem_limit: 512m + cpus: 1.0 + relay: + <<: *background + command: [python, -m, messenger.worker, relay] + mem_limit: 192m + cpus: 0.25 + lb: + image: nginx:1.28.0-alpine + volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro'] + networks: [front] + ports: ['127.0.0.1:8080:8080'] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 1s + timeout: 8s + retries: 30 + mem_limit: 128m + cpus: 0.5 + collector: + image: otel/opentelemetry-collector-contrib:0.128.0 + command: [--config=/etc/otelcol/config.yaml] + volumes: [./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro] + networks: [data, observe] + mem_limit: 256m + cpus: 0.5 + prometheus: + image: prom/prometheus:v3.4.1 + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d] + volumes: + - ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + ports: ['127.0.0.1:9090:9090'] + networks: [observe, admin] + mem_limit: 512m + cpus: 0.5 + loki: + image: grafana/loki:3.5.1 + command: [-config.file=/etc/loki/local.yaml] + volumes: [./infra/observability/loki.yaml:/etc/loki/local.yaml:ro, loki_data:/loki] + networks: [observe, admin] + ports: ['127.0.0.1:3100:3100'] + mem_limit: 384m + cpus: 0.5 + tempo: + image: grafana/tempo:2.7.2 + command: [-config.file=/etc/tempo/local.yaml] + user: '0:0' + volumes: [./infra/observability/tempo.yaml:/etc/tempo/local.yaml:ro, tempo_data:/var/tempo] + networks: [observe, admin] + ports: ['127.0.0.1:3200:3200'] + mem_limit: 384m + cpus: 0.5 + grafana: + image: grafana/grafana:12.0.1 + environment: + GF_SECURITY_ADMIN_USER: operator + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password + GF_USERS_ALLOW_SIGN_UP: 'false' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + secrets: [grafana_password] + volumes: + - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: ['127.0.0.1:3000:3000'] + networks: [observe, admin] + mem_limit: 256m + cpus: 0.5 +networks: + admin: + observe: + internal: true + front: + data: + internal: true +secrets: + grafana_password: {file: ./secrets/grafana_password} + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + minio_root_user: {file: ./secrets/minio_root_user} + minio_root_password: {file: ./secrets/minio_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} +volumes: + postgres_data: + redis_data: + minio_data: + broker_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/concurrent-idempotency.json b/evidence/concurrent-idempotency.json new file mode 100644 index 0000000..f0b0a4f --- /dev/null +++ b/evidence/concurrent-idempotency.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "parallel_requests": 8, + "observed_instances": [ + "0432eff5e9d84ea99431573f134f5c23", + "bfc39b720927442aa51a68b9fe837119" + ], + "responses_201": 8, + "messages_created": 1, + "different_body_status": 409 +} diff --git a/evidence/db-alert-firing.json b/evidence/db-alert-firing.json new file mode 100644 index 0000000..9986fa9 --- /dev/null +++ b/evidence/db-alert-firing.json @@ -0,0 +1,15 @@ +[ + { + "labels": { + "alertname": "HighApiErrorRate", + "severity": "warning" + }, + "annotations": { + "runbook": "Check postgres.transaction error spans, DB pool and request logs. Restore DB and confirm resolved.", + "summary": "Business API 5xx exceeds 5% in 1 minute" + }, + "state": "firing", + "activeAt": "2026-11-09T15:56:22.478517494Z", + "value": "7.272727272727273e-01" + } +] diff --git a/evidence/db-alert-resolved.json b/evidence/db-alert-resolved.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/evidence/db-alert-resolved.json @@ -0,0 +1 @@ +[] diff --git a/evidence/dependency-failures.json b/evidence/dependency-failures.json new file mode 100644 index 0000000..bbcbb37 --- /dev/null +++ b/evidence/dependency-failures.json @@ -0,0 +1,20 @@ +{ + "result": "PASS", + "redis": { + "profile_status": 200, + "fallback_seconds": 2.2293, + "anonymous": 401, + "readiness": 200 + }, + "postgres": { + "authenticated_status": 503, + "response_seconds": 0.0177, + "anonymous_status": 401, + "internal_api_liveness": [ + 200, + 200 + ], + "note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable." + }, + "same_session_after_recovery": true +} diff --git a/evidence/fresh-migrations.txt b/evidence/fresh-migrations.txt new file mode 100644 index 0000000..bce04bc --- /dev/null +++ b/evidence/fresh-migrations.txt @@ -0,0 +1,70 @@ +./scripts/check_fresh_migrations.sh + Network mtusi-lab6-migration-check-74280_data Creating + Volume mtusi-lab6-migration-check-74280_redis_data Creating + Volume mtusi-lab6-migration-check-74280_redis_data Creating + Volume mtusi-lab6-migration-check-74280_postgres_data Creating + Volume mtusi-lab6-migration-check-74280_postgres_data Creating + Network mtusi-lab6-migration-check-74280_data Creating + Volume mtusi-lab6-migration-check-74280_minio_data Creating + Volume mtusi-lab6-migration-check-74280_minio_data Creating + Volume mtusi-lab6-migration-check-74280_redis_data Created + Volume mtusi-lab6-migration-check-74280_redis_data Created + Volume mtusi-lab6-migration-check-74280_postgres_data Created + Volume mtusi-lab6-migration-check-74280_postgres_data Created + Volume mtusi-lab6-migration-check-74280_minio_data Created + Volume mtusi-lab6-migration-check-74280_minio_data Created + Network mtusi-lab6-migration-check-74280_data Created + Network mtusi-lab6-migration-check-74280_data Created + Container mtusi-lab6-migration-check-74280-redis-1 Creating + Container mtusi-lab6-migration-check-74280-postgres-1 Creating + Container mtusi-lab6-migration-check-74280-minio-1 Creating + Container mtusi-lab6-migration-check-74280-redis-1 Created + Container mtusi-lab6-migration-check-74280-minio-1 Created + Container mtusi-lab6-migration-check-74280-minio-init-1 Creating + Container mtusi-lab6-migration-check-74280-postgres-1 Created + Container mtusi-lab6-migration-check-74280-minio-init-1 Created + Container mtusi-lab6-migration-check-74280-minio-1 Starting + Container mtusi-lab6-migration-check-74280-redis-1 Starting + Container mtusi-lab6-migration-check-74280-postgres-1 Starting + Container mtusi-lab6-migration-check-74280-minio-1 Started + Container mtusi-lab6-migration-check-74280-minio-1 Waiting + Container mtusi-lab6-migration-check-74280-redis-1 Started + Container mtusi-lab6-migration-check-74280-postgres-1 Started + Container mtusi-lab6-migration-check-74280-minio-1 Healthy + Container mtusi-lab6-migration-check-74280-minio-init-1 Starting + Container mtusi-lab6-migration-check-74280-minio-init-1 Started + Container mtusi-lab6-migration-check-74280-minio-init-1 Waiting + Container mtusi-lab6-migration-check-74280-redis-1 Waiting + Container mtusi-lab6-migration-check-74280-postgres-1 Waiting + Container mtusi-lab6-migration-check-74280-minio-1 Waiting + Container mtusi-lab6-migration-check-74280-redis-1 Healthy + Container mtusi-lab6-migration-check-74280-minio-init-1 Healthy + Container mtusi-lab6-migration-check-74280-minio-1 Healthy + Container mtusi-lab6-migration-check-74280-postgres-1 Healthy + Network mtusi-lab6-migration-check-74280_front Creating + Network mtusi-lab6-migration-check-74280_front Creating + Network mtusi-lab6-migration-check-74280_front Created + Network mtusi-lab6-migration-check-74280_front Created + Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Creating + Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Created +Applied 001_initial.sql +Applied 002_idempotency.sql +Applied 003_attachments.sql +Applied 004_jobs.sql +Migrations up to date +Migrations up to date +............................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 19 warnings +student-tests/test_idempotency.py: 1 warning +student-tests/test_jobs.py: 6 warnings + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +31 passed, 27 warnings in 13.57s +PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes. diff --git a/evidence/graceful-shutdown.json b/evidence/graceful-shutdown.json new file mode 100644 index 0000000..a205f9e --- /dev/null +++ b/evidence/graceful-shutdown.json @@ -0,0 +1,16 @@ +{ + "result": "PASS", + "target_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "target_instance": "a535000080734d769e44ed18a64615ae", + "fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards", + "in_flight_observed_in_pg_stat_activity": true, + "in_flight_response": 201, + "new_tcp_after_sigterm": { + "accepted": false, + "error_type": "ConnectionRefusedError" + }, + "target_still_running_when_new_tcp_rejected": true, + "exit_code": 0, + "shutdown_seconds_from_sigterm": 2.0642, + "confirmed_message_preserved": true +} diff --git a/evidence/grafana-provisioning.json b/evidence/grafana-provisioning.json new file mode 100644 index 0000000..80bc73b --- /dev/null +++ b/evidence/grafana-provisioning.json @@ -0,0 +1,22 @@ +{ + "result": "PASS", + "dashboard_uid": "messenger-lab6", + "panels": 15, + "datasources": [ + { + "name": "Loki", + "type": "loki", + "uid": "loki" + }, + { + "name": "Prometheus", + "type": "prometheus", + "uid": "prometheus" + }, + { + "name": "Tempo", + "type": "tempo", + "uid": "tempo" + } + ] +} diff --git a/evidence/infrastructure.json b/evidence/infrastructure.json new file mode 100644 index 0000000..1beea4c --- /dev/null +++ b/evidence/infrastructure.json @@ -0,0 +1,341 @@ +{ + "result": "PASS", + "containers": [ + { + "service": "api", + "container": "07e419b5121b83e36d525e3d218a4dfd4d5768b83531fe69ea2e459920931114", + "name": "mtusi-lab6-api-3", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:53:46.48092646Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "23ac94c138224f05b2b9ebc8962a8f79", + "release": "lab6-v1" + }, + { + "service": "api", + "container": "add61ef8a9c8a4f7e286f2b7b5ef8bc235905c843d2081ec14fc2ee2c1aee9ce", + "name": "mtusi-lab6-api-4", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:53:46.408269418Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "28bb7891222a49a4809c409c0cd15976", + "release": "lab6-v1" + }, + { + "service": "broker", + "container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6", + "name": "mtusi-lab6-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.284677889Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_broker_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "collector", + "container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3", + "name": "mtusi-lab6-collector-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:20:09.734253509Z", + "image": "otel/opentelemetry-collector-contrib:0.128.0", + "image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_observe" + ] + }, + { + "service": "grafana", + "container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c", + "name": "mtusi-lab6-grafana-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.50566143Z", + "image": "grafana/grafana:12.0.1", + "image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3000" + } + ] + }, + "volumes": [ + "mtusi-lab6_grafana_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "lb", + "container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c", + "name": "mtusi-lab6-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.236679055Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab6_front" + ] + }, + { + "service": "loki", + "container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3", + "name": "mtusi-lab6-loki-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:41:58.345305841Z", + "image": "grafana/loki:3.5.1", + "image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3100" + } + ] + }, + "volumes": [ + "mtusi-lab6_loki_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "migrate", + "container": "3c47c49b694fda4217bb34ca4f9ea5c83c534deeb2ad8cfbd92a73e191a83959", + "name": "mtusi-lab6-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:53:45.228435626Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio", + "container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d", + "name": "mtusi-lab6-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.159907764Z", + "image": "mtusi-lab6-minio:2025-04-22", + "image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_minio_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio-init", + "container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7", + "name": "mtusi-lab6-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:53:45.325678751Z", + "image": "mtusi-lab6-mc:2025-04-16", + "image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "postgres", + "container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7", + "name": "mtusi-lab6-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:53:53.727434505Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_postgres_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "prometheus", + "container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb", + "name": "mtusi-lab6-prometheus-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.39409243Z", + "image": "prom/prometheus:v3.4.1", + "image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d", + "memory_bytes": 536870912, + "nano_cpus": 500000000, + "published_ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "9090" + } + ] + }, + "volumes": [ + "mtusi-lab6_prometheus_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "redis", + "container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272", + "name": "mtusi-lab6-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:53:52.525647296Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_redis_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "relay", + "container": "ed1f47e618ca83beab6abf06e3c288c51d101bfcc53e140b2295d9b6c1a3138e", + "name": "mtusi-lab6-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:53:46.636963169Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "tempo", + "container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e", + "name": "mtusi-lab6-tempo-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:53:44.730313751Z", + "image": "grafana/tempo:2.7.2", + "image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3200" + } + ] + }, + "volumes": [ + "mtusi-lab6_tempo_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "worker", + "container": "1eed3d38f1d69ce58034efdce9c7a1867baf0f88f52146b3dad12addd25adec1", + "name": "mtusi-lab6-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:53:46.559703293Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + } + ] +} diff --git a/evidence/jobs-faults.json b/evidence/jobs-faults.json new file mode 100644 index 0000000..da28848 --- /dev/null +++ b/evidence/jobs-faults.json @@ -0,0 +1,27 @@ +{ + "result": "PASS", + "commit_publish_gap": { + "accepted_status": 202, + "outbox_unpublished": true, + "api_broker_restarted": true, + "same_job_ready": true + }, + "broker_persistence": { + "unread_job_survived_restart": true + }, + "after_put_before_ack": { + "s3_effect_written": true, + "lease_recovery_seconds": 89.096, + "attempt_after_recovery": 2, + "same_job": true + }, + "duplicate": { + "ready_noop": true + }, + "retry_replay": { + "failed_after": 5, + "code": "retry_exhausted", + "explicit_replay_ready": true, + "failed_attachment_binding": 409 + } +} diff --git a/evidence/last-stop.json b/evidence/last-stop.json new file mode 100644 index 0000000..b0c1ed6 --- /dev/null +++ b/evidence/last-stop.json @@ -0,0 +1,11 @@ +{ + "container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "name": "mtusi-lab6-api-2", + "instance": "aee7f9540f7549b88fdbb8ee4290e6af", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "stopped_at_utc": "2026-11-09T15:45:58.712702+00:00", + "stop_seconds": 0.5425, + "exit_code": 0, + "left_stopped": true +} diff --git a/evidence/nginx.json b/evidence/nginx.json new file mode 100644 index 0000000..bb6873c --- /dev/null +++ b/evidence/nginx.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "version": "nginx version: nginx/1.28.0", + "image": "nginx:1.28.0-alpine", + "public_server_header": "nginx", + "config_syntax": "PASS", + "observed_instances": [ + "0432eff5e9d84ea99431573f134f5c23", + "bfc39b720927442aa51a68b9fe837119" + ] +} diff --git a/evidence/observability.json b/evidence/observability.json new file mode 100644 index 0000000..a14143d --- /dev/null +++ b/evidence/observability.json @@ -0,0 +1,342 @@ +{ + "result": "PASS", + "trace": { + "trace_id": "69e61ec89e07ea9c4d93f81b188820fe", + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.query", + "s3.put", + "postgres.query", + "postgres.query", + "postgres.query", + "postgres.commit", + "attachments.publish", + "attachments.consume", + "postgres.query", + "postgres.query", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.query", + "postgres.commit", + "attachments.process_objects", + "s3.get", + "attachments.publish", + "attachments.consume", + "postgres.query", + "postgres.query", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.commit", + "postgres.query", + "postgres.query", + "postgres.commit", + "attachments.process_objects", + "s3.get" + ], + "retry_attempts": 3, + "api_worker_logs_correlated": true + }, + "queue_incident": { + "oldest_age": [ + { + "metric": {}, + "value": [ + 1794239772.021, + "32.563177" + ] + } + ], + "depth": [ + { + "metric": {}, + "value": [ + 1794239772.023, + "1" + ] + } + ], + "alert": "firing", + "after_restore": "resolved" + }, + "db_incident": { + "failures": [ + { + "status": 503, + "elapsed": 4.012639791006222, + "trace_id": "2248eed7575b37ebc02bc118377be094" + }, + { + "status": 503, + "elapsed": 4.009124165982939, + "trace_id": "e4a499d2482593d978976d4981e1be6c" + }, + { + "status": 503, + "elapsed": 4.010977207974065, + "trace_id": "bfa70bd527ce23c595426c74bfee607e" + }, + { + "status": 503, + "elapsed": 4.038811917009298, + "trace_id": "95bca29d2812c4cc930ac9fed93a1f7c" + }, + { + "status": 503, + "elapsed": 4.028251375013497, + "trace_id": "b9e433fdd149f2922f0b3d78386e6d5a" + }, + { + "status": 503, + "elapsed": 4.018512499984354, + "trace_id": "a11d9c704425e92132f9f5647d38bc1f" + }, + { + "status": 503, + "elapsed": 4.017416250018869, + "trace_id": "0eb74ac09ce2aa9a36d676c2eafde20e" + }, + { + "status": 503, + "elapsed": 4.0125838330131955, + "trace_id": "17f11595e2dc8fe7bbfb33e1bff4c637" + }, + { + "status": 503, + "elapsed": 4.007467999996152, + "trace_id": "ddfbc5519d3d9a746207a7d2c427364e" + } + ], + "before": "firing", + "after": "resolved", + "pool_waiting": [ + { + "metric": {}, + "value": [ + 1794239845.959, + "0" + ] + } + ] + }, + "telemetry_outage": { + "successful_messages": 2575, + "seconds": 26.571, + "collector_before": { + "BlockIO": "0B / 0B", + "CPUPerc": "1.58%", + "Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3", + "ID": "62ead4df7cf3", + "MemPerc": "29.31%", + "MemUsage": "75.03MiB / 256MiB", + "Name": "mtusi-lab6-collector-1", + "NetIO": "2.92MB / 1.2MB", + "PIDs": "16" + }, + "collector_after": { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3", + "ID": "62ead4df7cf3", + "MemPerc": "29.96%", + "MemUsage": "76.7MiB / 256MiB", + "Name": "mtusi-lab6-collector-1", + "NetIO": "9MB / 1.56MB", + "PIDs": "16" + }, + "collector_metrics": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_capacity", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "256" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_capacity", + "data_type": "traces", + "exporter": "otlp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "256" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "23" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "202" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "264" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlp/tempo", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794239872.968, + "5848" + ] + }, + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794239872.968, + "32506" + ] + } + ], + "queue_limit_batches": 256, + "collector_limit_mib": 256 + }, + "dashboard_queries": [ + { + "panel": "Business RPS", + "expression": "sum(rate(http_requests_total[1m]))", + "series": 1 + }, + { + "panel": "5xx ratio", + "expression": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)", + "series": 1 + }, + { + "panel": "HTTP p50", + "expression": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "series": 1 + }, + { + "panel": "HTTP p95", + "expression": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "series": 1 + }, + { + "panel": "HTTP p99", + "expression": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "series": 1 + }, + { + "panel": "Jobs by outcome", + "expression": "sum by(outcome)(rate(jobs_processed_total[1m]))", + "series": 1 + }, + { + "panel": "Job p95", + "expression": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))", + "series": 1 + }, + { + "panel": "Accepted to ready p95", + "expression": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))", + "series": 1 + }, + { + "panel": "Queue depth", + "expression": "max(queue_depth)", + "series": 1 + }, + { + "panel": "Oldest unfinished job", + "expression": "max(queue_oldest_age_seconds)", + "series": 1 + }, + { + "panel": "DB pool used / waiting", + "expression": "sum by(service_name)(db_pool_used)", + "series": 3 + }, + { + "panel": "DB pool waiting", + "expression": "sum by(service_name)(db_pool_waiting)", + "series": 3 + }, + { + "panel": "Worker busy / capacity", + "expression": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)", + "series": 1 + }, + { + "panel": "Process RSS", + "expression": "process_memory_bytes", + "series": 4 + }, + { + "panel": "Process CPU cores", + "expression": "rate(process_cpu_seconds_total[1m])", + "series": 4 + } + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 0000000..49c347b --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1,2 @@ +python3 scripts/cluster.py exec python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 0000000..277af8c --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,680 @@ +{ + "result": "PASS", + "before": [ + { + "service": "api", + "container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "name": "mtusi-lab6-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:43:08.631200429Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "0432eff5e9d84ea99431573f134f5c23", + "release": "lab6-v1" + }, + { + "service": "api", + "container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "name": "mtusi-lab6-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:43:08.658375471Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "bfc39b720927442aa51a68b9fe837119", + "release": "lab6-v1" + }, + { + "service": "broker", + "container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6", + "name": "mtusi-lab6-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:43:17.121717419Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_broker_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "collector", + "container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3", + "name": "mtusi-lab6-collector-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:20:09.734253509Z", + "image": "otel/opentelemetry-collector-contrib:0.128.0", + "image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_observe" + ] + }, + { + "service": "grafana", + "container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c", + "name": "mtusi-lab6-grafana-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.50566143Z", + "image": "grafana/grafana:12.0.1", + "image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3000" + } + ] + }, + "volumes": [ + "mtusi-lab6_grafana_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "lb", + "container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c", + "name": "mtusi-lab6-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:21:04.334408756Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab6_front" + ] + }, + { + "service": "loki", + "container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3", + "name": "mtusi-lab6-loki-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:41:58.345305841Z", + "image": "grafana/loki:3.5.1", + "image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3100" + } + ] + }, + "volumes": [ + "mtusi-lab6_loki_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "migrate", + "container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9", + "name": "mtusi-lab6-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:26:11.136784593Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio", + "container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d", + "name": "mtusi-lab6-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:33:32.491039297Z", + "image": "mtusi-lab6-minio:2025-04-22", + "image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_minio_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio-init", + "container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7", + "name": "mtusi-lab6-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:26:11.058089009Z", + "image": "mtusi-lab6-mc:2025-04-16", + "image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "postgres", + "container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7", + "name": "mtusi-lab6-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:20:09.5754303Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_postgres_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "prometheus", + "container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb", + "name": "mtusi-lab6-prometheus-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.39409243Z", + "image": "prom/prometheus:v3.4.1", + "image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d", + "memory_bytes": 536870912, + "nano_cpus": 500000000, + "published_ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "9090" + } + ] + }, + "volumes": [ + "mtusi-lab6_prometheus_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "redis", + "container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272", + "name": "mtusi-lab6-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:20:09.209044467Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_redis_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "relay", + "container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49", + "name": "mtusi-lab6-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:45:03.889331218Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "tempo", + "container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e", + "name": "mtusi-lab6-tempo-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:41:58.448806924Z", + "image": "grafana/tempo:2.7.2", + "image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3200" + } + ] + }, + "volumes": [ + "mtusi-lab6_tempo_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "worker", + "container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c", + "name": "mtusi-lab6-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:45:03.82336276Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + } + ], + "after": [ + { + "service": "api", + "container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "name": "mtusi-lab6-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.501501805Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "fcc51ba5c356403998a22d67f9b9be19", + "release": "lab6-v1" + }, + { + "service": "api", + "container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "name": "mtusi-lab6-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.556862139Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_front" + ], + "http_status": 200, + "instance": "aee7f9540f7549b88fdbb8ee4290e6af", + "release": "lab6-v1" + }, + { + "service": "broker", + "container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6", + "name": "mtusi-lab6-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.284677889Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_broker_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "collector", + "container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3", + "name": "mtusi-lab6-collector-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:20:09.734253509Z", + "image": "otel/opentelemetry-collector-contrib:0.128.0", + "image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data", + "mtusi-lab6_observe" + ] + }, + { + "service": "grafana", + "container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c", + "name": "mtusi-lab6-grafana-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.50566143Z", + "image": "grafana/grafana:12.0.1", + "image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3000" + } + ] + }, + "volumes": [ + "mtusi-lab6_grafana_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "lb", + "container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c", + "name": "mtusi-lab6-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.236679055Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab6_front" + ] + }, + { + "service": "loki", + "container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3", + "name": "mtusi-lab6-loki-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:41:58.345305841Z", + "image": "grafana/loki:3.5.1", + "image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3100" + } + ] + }, + "volumes": [ + "mtusi-lab6_loki_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "migrate", + "container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9", + "name": "mtusi-lab6-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:45:13.574632917Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio", + "container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d", + "name": "mtusi-lab6-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.159907764Z", + "image": "mtusi-lab6-minio:2025-04-22", + "image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_minio_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "minio-init", + "container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7", + "name": "mtusi-lab6-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:45:13.645406292Z", + "image": "mtusi-lab6-mc:2025-04-16", + "image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "postgres", + "container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7", + "name": "mtusi-lab6-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.211149514Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_postgres_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "prometheus", + "container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb", + "name": "mtusi-lab6-prometheus-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:23:49.39409243Z", + "image": "prom/prometheus:v3.4.1", + "image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d", + "memory_bytes": 536870912, + "nano_cpus": 500000000, + "published_ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "9090" + } + ] + }, + "volumes": [ + "mtusi-lab6_prometheus_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "redis", + "container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272", + "name": "mtusi-lab6-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.321981847Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_redis_data" + ], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "relay", + "container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49", + "name": "mtusi-lab6-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:45:11.757355583Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + }, + { + "service": "tempo", + "container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e", + "name": "mtusi-lab6-tempo-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:41:58.448806924Z", + "image": "grafana/tempo:2.7.2", + "image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3", + "memory_bytes": 402653184, + "nano_cpus": 500000000, + "published_ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3200" + } + ] + }, + "volumes": [ + "mtusi-lab6_tempo_data" + ], + "networks": [ + "mtusi-lab6_admin", + "mtusi-lab6_observe" + ] + }, + { + "service": "worker", + "container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c", + "name": "mtusi-lab6-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:45:12.362175042Z", + "image": "mtusi-lab6-api:lab6", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab6_data" + ] + } + ], + "public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..749fd29 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover.json b/evidence/public-failover.json new file mode 100644 index 0000000..8f7db3b --- /dev/null +++ b/evidence/public-failover.json @@ -0,0 +1,15 @@ +{ + "result": "PASS", + "public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.107}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "stopped": { + "container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "name": "mtusi-lab6-api-1", + "instance": "fcc51ba5c356403998a22d67f9b9be19", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "stopped_at_utc": "2026-11-09T15:45:19.129171+00:00", + "stop_seconds": 0.3545, + "exit_code": 0, + "left_stopped": true + } +} diff --git a/evidence/public-failover.txt b/evidence/public-failover.txt new file mode 100644 index 0000000..c7d3eb9 --- /dev/null +++ b/evidence/public-failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 1.107} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/queue-alert-firing.json b/evidence/queue-alert-firing.json new file mode 100644 index 0000000..9b2f31f --- /dev/null +++ b/evidence/queue-alert-firing.json @@ -0,0 +1,15 @@ +[ + { + "labels": { + "alertname": "QueueLagGrowing", + "severity": "warning" + }, + "annotations": { + "runbook": "Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains.", + "summary": "Oldest unfinished attachment is older than 15 seconds" + }, + "state": "firing", + "activeAt": "2026-11-09T15:55:54.478517494Z", + "value": "3.2563177e+01" + } +] diff --git a/evidence/queue-alert-resolved.json b/evidence/queue-alert-resolved.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/evidence/queue-alert-resolved.json @@ -0,0 +1 @@ +[] diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 0000000..604228c --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,171 @@ +{ + "result": "PASS", + "before": [ + { + "container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "name": "mtusi-lab6-api-1", + "instance": "f6c55a60b9d54fc49a966a29db53c7c9", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80" + }, + { + "container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "name": "mtusi-lab6-api-2", + "instance": "201f4ef054b141e89207a80a27997663", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80" + } + ], + "after": [ + { + "container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5", + "name": "mtusi-lab6-api-3", + "instance": "89984cccbfad40c8ae3896b5789dd706", + "release": "lab6-68cfb17c", + "image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f" + }, + { + "container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8", + "name": "mtusi-lab6-api-4", + "instance": "42711a4025dd43b5aa9c806b73e710ac", + "release": "lab6-68cfb17c", + "image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f" + } + ], + "target_release": "lab6-68cfb17c", + "lb_before": { + "service": "lb", + "container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c", + "name": "mtusi-lab6-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.236679055Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab6_front" + ] + }, + "lb_after": { + "service": "lb", + "container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c", + "name": "mtusi-lab6-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:45:11.236679055Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab6_front" + ] + }, + "lb_not_restarted": true, + "build_seconds": 23.6953, + "migration_output": "Migrations up to date", + "replacements": [ + { + "old_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "new_container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5", + "old_ip": "172.25.0.2", + "new_ip": "172.25.0.5", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "201f4ef054b141e89207a80a27997663", + "89984cccbfad40c8ae3896b5789dd706", + "f6c55a60b9d54fc49a966a29db53c7c9" + ], + "observed_instances": [ + "201f4ef054b141e89207a80a27997663", + "89984cccbfad40c8ae3896b5789dd706" + ], + "old_exit_code": 0, + "seconds": 5.9622 + }, + { + "old_container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "new_container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8", + "old_ip": "172.25.0.3", + "new_ip": "172.25.0.2", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "201f4ef054b141e89207a80a27997663", + "42711a4025dd43b5aa9c806b73e710ac", + "89984cccbfad40c8ae3896b5789dd706" + ], + "observed_instances": [ + "42711a4025dd43b5aa9c806b73e710ac", + "89984cccbfad40c8ae3896b5789dd706" + ], + "old_exit_code": 0, + "seconds": 6.1869 + } + ], + "during_replacement": { + "seconds": 12.1492, + "concurrency": 4, + "successful_requests": 6406, + "failed_attempts": 0, + "successful_rps": 527.28, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 5.911, + "p95": 11.297, + "p99": 56.654 + }, + "instances": [ + "201f4ef054b141e89207a80a27997663", + "42711a4025dd43b5aa9c806b73e710ac", + "89984cccbfad40c8ae3896b5789dd706", + "f6c55a60b9d54fc49a966a29db53c7c9" + ] + }, + "total": { + "seconds": 42.993, + "concurrency": 4, + "successful_requests": 24247, + "failed_attempts": 0, + "successful_rps": 563.975, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 5.93, + "p95": 9.719, + "p99": 52.049 + }, + "instances": [ + "201f4ef054b141e89207a80a27997663", + "42711a4025dd43b5aa9c806b73e710ac", + "89984cccbfad40c8ae3896b5789dd706", + "f6c55a60b9d54fc49a966a29db53c7c9" + ] + }, + "data": { + "confirmed_unique_messages": 24247, + "history_messages": 24247, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate." +} diff --git a/evidence/sequential-failover.json b/evidence/sequential-failover.json new file mode 100644 index 0000000..5a04ade --- /dev/null +++ b/evidence/sequential-failover.json @@ -0,0 +1,135 @@ +{ + "result": "PASS", + "baseline": { + "seconds": 3.0007, + "concurrency": 4, + "successful_requests": 1736, + "failed_attempts": 0, + "successful_rps": 578.527, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 5.783, + "p95": 9.151, + "p99": 37.03 + }, + "instances": [ + "2af58291b4f34b53bcd4a8c81a6c4abb", + "aee7f9540f7549b88fdbb8ee4290e6af" + ] + }, + "phases": [ + { + "stopped": { + "container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40", + "name": "mtusi-lab6-api-1", + "instance": "2af58291b4f34b53bcd4a8c81a6c4abb", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "stopped_at_utc": "2026-11-09T15:45:44.649698+00:00", + "stop_seconds": 0.4897, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.005, + "concurrency": 4, + "successful_requests": 2698, + "failed_attempts": 0, + "successful_rps": 337.039, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 9.853, + "p95": 23.34, + "p99": 65.796 + }, + "instances": [ + "2af58291b4f34b53bcd4a8c81a6c4abb", + "aee7f9540f7549b88fdbb8ee4290e6af" + ] + }, + "post_stop_write_seconds_from_action_start": 1.3292, + "post_stop_write_seconds_from_stop_completion": 0.0295, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.3137, + "seconds_from_stop_completion": 0.014 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + }, + { + "stopped": { + "container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b", + "name": "mtusi-lab6-api-2", + "instance": "aee7f9540f7549b88fdbb8ee4290e6af", + "release": "lab6-v1", + "image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80", + "stopped_at_utc": "2026-11-09T15:45:58.712702+00:00", + "stop_seconds": 0.5425, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.005, + "concurrency": 4, + "successful_requests": 2729, + "failed_attempts": 0, + "successful_rps": 340.91, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 9.895, + "p95": 22.682, + "p99": 60.64 + }, + "instances": [ + "a535000080734d769e44ed18a64615ae", + "aee7f9540f7549b88fdbb8ee4290e6af" + ] + }, + "post_stop_write_seconds_from_action_start": 1.1705, + "post_stop_write_seconds_from_stop_completion": 0.0247, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.1615, + "seconds_from_stop_completion": 0.0157 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + } + ], + "total": { + "seconds": 31.1691, + "concurrency": 4, + "successful_requests": 12705, + "failed_attempts": 0, + "successful_rps": 407.615, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 7.861, + "p95": 21.097, + "p99": 58.639 + }, + "instances": [ + "201f4ef054b141e89207a80a27997663", + "2af58291b4f34b53bcd4a8c81a6c4abb", + "a535000080734d769e44ed18a64615ae", + "aee7f9540f7549b88fdbb8ee4290e6af" + ] + }, + "data": { + "confirmed_unique_messages": 12705, + "history_messages": 12705, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "each_original_container_tested": true +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..7e5c8e2 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..6fc95ce --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,15 @@ +python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +............................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 19 warnings +student-tests/test_idempotency.py: 1 warning +student-tests/test_jobs.py: 6 warnings + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +31 passed, 27 warnings in 13.44s diff --git a/evidence/upload-logs.json b/evidence/upload-logs.json new file mode 100644 index 0000000..ac4c08b --- /dev/null +++ b/evidence/upload-logs.json @@ -0,0 +1,244 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "code_file_path": "", + "code_function_name": "", + "code_line_number": "0", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "flags": "1", + "observed_timestamp": "1794239727634612632", + "scope_name": "messenger.events", + "service_instance_id": "3d072627b1ab496c9308fc049bd79784", + "service_name": "messenger-api", + "service_version": "lab6-v1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "be90e8be55e18b54", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.34.1", + "trace_id": "69e61ec89e07ea9c4d93f81b188820fe" + }, + "values": [ + [ + "1794239727634585856", + "{\"event\":\"http_completed\",\"method\":\"POST\",\"request_id\":\"23dc38ec-dc6c-4592-9ba9-6744d664e150\",\"route\":\"/api/v1/chats/{chat_id}/attachments\",\"service_name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"be90e8be55e18b54\",\"status_class\":\"2xx\",\"timestamp\":1794239727.6345801,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}" + ] + ] + }, + { + "stream": { + "code_file_path": "", + "code_function_name": "", + "code_line_number": "0", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "flags": "1", + "observed_timestamp": "1794239729394242216", + "scope_name": "messenger.events", + "service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198", + "service_name": "messenger-worker", + "service_version": "lab6-v1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f024a4b9cef096b9", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.34.1", + "trace_id": "69e61ec89e07ea9c4d93f81b188820fe" + }, + "values": [ + [ + "1794239729394201088", + "{\"attempt\":1,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"f024a4b9cef096b9\",\"timestamp\":1794239729.3941958,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}" + ] + ] + }, + { + "stream": { + "code_file_path": "", + "code_function_name": "", + "code_line_number": "0", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "flags": "1", + "observed_timestamp": "1794239731438346217", + "scope_name": "messenger.events", + "service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198", + "service_name": "messenger-worker", + "service_version": "lab6-v1", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "866c15dacbeadbc2", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.34.1", + "trace_id": "69e61ec89e07ea9c4d93f81b188820fe" + }, + "values": [ + [ + "1794239731438312704", + "{\"attempt\":2,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"866c15dacbeadbc2\",\"timestamp\":1794239731.4383066,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 51852023, + "linesProcessedPerSecond": 82276, + "totalBytesProcessed": 24822333, + "totalLinesProcessed": 39387, + "execTime": 0.478715, + "queueTime": 0.000184, + "subqueries": 0, + "totalEntriesReturned": 3, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 3, + "totalStructuredMetadataBytesProcessed": 11794082 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 15, + "totalBatches": 2, + "totalLinesSent": 3, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 2, + "chunksDownloadTime": 4766583, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 931797, + "headChunkLines": 2188, + "decompressedBytes": 23890536, + "decompressedLines": 37199, + "compressedBytes": 2915879, + "totalDuplicates": 0, + "postFilterLines": 3, + "headChunkStructuredMetadataBytes": 262392, + "decompressedStructuredMetadataBytes": 11531690 + }, + "chunkRefsFetchTime": 385583, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/upload-trace.json b/evidence/upload-trace.json new file mode 100644 index 0000000..0d712f2 --- /dev/null +++ b/evidence/upload-trace.json @@ -0,0 +1,1584 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "3d072627b1ab496c9308fc049bd79784" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794239727618076507", + "endTimeUnixNano": "1794239727634797757", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "ojhJCim+dBc=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727618895715", + "endTimeUnixNano": "1794239727619659965", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "r5SJdA+u/NM=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727619736299", + "endTimeUnixNano": "1794239727619918132", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "SfjkcHc9Mzk=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727620362965", + "endTimeUnixNano": "1794239727620847549", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "sfU/KTTir/A=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727620901965", + "endTimeUnixNano": "1794239727621069924", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "IedaFVtN+jA=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727623803465", + "endTimeUnixNano": "1794239727624276549", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "ajhd0Gp+xQE=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "s3.put", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727624327715", + "endTimeUnixNano": "1794239727631044257", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "wbv9GRHCiYI=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727631157424", + "endTimeUnixNano": "1794239727632156757", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "oBO/3lf3uwE=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727632361257", + "endTimeUnixNano": "1794239727633165507", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "2u/P7LzkuDE=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727633261090", + "endTimeUnixNano": "1794239727633646757", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "XO8Wr5DYUXg=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239727633716215", + "endTimeUnixNano": "1794239727634161632", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-relay" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "30fd4546dd5649fbb49b88aad191f07d" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "QeL6BgJxrMs=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 768, + "name": "attachments.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794239727732107549", + "endTimeUnixNano": "1794239727733414007", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "8CSkuc7wlrk=", + "parentSpanId": "QeL6BgJxrMs=", + "flags": 768, + "name": "attachments.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794239728747935174", + "endTimeUnixNano": "1794239729396504633", + "attributes": [ + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "retry" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "KAYcmbaYEBs=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729392329425", + "endTimeUnixNano": "1794239729392782133", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "o6BGOtr3hlQ=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729392820925", + "endTimeUnixNano": "1794239729393011633", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "kofuNWIAiGw=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729393038383", + "endTimeUnixNano": "1794239729393200383", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "QNlwRyH/0g0=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729393229883", + "endTimeUnixNano": "1794239729393843008", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "9yyYMcyEnz4=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729394665591", + "endTimeUnixNano": "1794239729394964300", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "m4UIOI8lYY0=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729395011175", + "endTimeUnixNano": "1794239729395080300", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "+SOgmWWa+Ow=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729395117841", + "endTimeUnixNano": "1794239729395122633", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "av86G0WBEpQ=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728748031758", + "endTimeUnixNano": "1794239728748693216", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "+daVCBf7pOI=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728748765674", + "endTimeUnixNano": "1794239728748838383", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "fvaCC2ejQGk=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728748876549", + "endTimeUnixNano": "1794239728750897133", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "8jgum1C5cH4=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728751013799", + "endTimeUnixNano": "1794239728751125091", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "U0+XSBD8tBc=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728751168841", + "endTimeUnixNano": "1794239728751857466", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "+ycL2F4lvbI=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728751911966", + "endTimeUnixNano": "1794239728752245258", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "n6p7fC92qwQ=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239728752293799", + "endTimeUnixNano": "1794239728752728633", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "fiYoojCdoPg=", + "parentSpanId": "8CSkuc7wlrk=", + "flags": 768, + "name": "attachments.process_objects", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729281595675", + "endTimeUnixNano": "1794239729284087175", + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "7IVLMpRiTRc=", + "parentSpanId": "fiYoojCdoPg=", + "flags": 256, + "name": "s3.get", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239729281675425", + "endTimeUnixNano": "1794239729284055175", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-relay" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "30fd4546dd5649fbb49b88aad191f07d" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "DfKbmHANZeI=", + "parentSpanId": "vpDovlXhi1Q=", + "flags": 768, + "name": "attachments.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794239730756956800", + "endTimeUnixNano": "1794239730759279509", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "hmwV2svq28I=", + "parentSpanId": "DfKbmHANZeI=", + "flags": 768, + "name": "attachments.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794239730759358800", + "endTimeUnixNano": "1794239731439935509", + "attributes": [ + { + "key": "job.type", + "value": { + "stringValue": "attachment.process.v1" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "retry" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "fM1GiHTJXto=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731437115926", + "endTimeUnixNano": "1794239731437619009", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "na3Wu3nmtiw=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731437677259", + "endTimeUnixNano": "1794239731437812759", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "LDJTS69QE/0=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731437848009", + "endTimeUnixNano": "1794239731437923842", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "gw3hHf3zvVo=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731437946051", + "endTimeUnixNano": "1794239731438211092", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "2jGShS3mqY4=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731438525551", + "endTimeUnixNano": "1794239731438810467", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "Rdd3IJMq2C8=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731438855051", + "endTimeUnixNano": "1794239731438912717", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "2A2Vvwi8CjM=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731438933342", + "endTimeUnixNano": "1794239731438937009", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "yQh6xs9LEao=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730759474842", + "endTimeUnixNano": "1794239730759948759", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "k4azr7PEzdk=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730760051425", + "endTimeUnixNano": "1794239730760217800", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "YcVU3jxoo5g=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730760322592", + "endTimeUnixNano": "1794239730760742717", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "R1wRNKYrrvg=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730760839467", + "endTimeUnixNano": "1794239730760934842", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "yf7yaaVUT/E=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730761007550", + "endTimeUnixNano": "1794239730761352300", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "V7YfdYtC25g=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.query", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730761410425", + "endTimeUnixNano": "1794239730761593967", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "vMjzhuFIw20=", + "parentSpanId": "hmwV2svq28I=", + "flags": 256, + "name": "postgres.commit", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239730761668759", + "endTimeUnixNano": "1794239730761974634", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.34.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6-v1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "1bb38b267fcc402dae8d85fb5e9d5198" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "1.0" + }, + "spans": [ + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "nCUvmNlRcXc=", + "parentSpanId": "hmwV2svq28I=", + "flags": 768, + "name": "attachments.process_objects", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731348434384", + "endTimeUnixNano": "1794239731350770176", + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "aeYeyJ4H6pxNk/gbGIgg/g==", + "spanId": "fUN4gK0VVwM=", + "parentSpanId": "nCUvmNlRcXc=", + "flags": 256, + "name": "s3.get", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794239731348486384", + "endTimeUnixNano": "1794239731350747134", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + } + ] +} diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 0000000..d1a5fca --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,28 @@ +# Official public images/binaries were removed upstream; build fixed release sources. +FROM golang:1.24.2-bookworm AS minio-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio . +RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio . + +FROM golang:1.24.2-bookworm AS mc-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc . +RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc . + +FROM python:3.13.5-slim AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z" +RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data +COPY --from=minio-build /out/minio /usr/local/bin/minio +COPY infra/minio/server.sh /server.sh +USER 10002 +ENTRYPOINT ["/bin/sh", "/server.sh"] +CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"] + +FROM debian:bookworm-slim AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z" +COPY --from=mc-build /out/mc /usr/local/bin/mc +ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"] diff --git a/infra/minio/api-policy.json b/infra/minio/api-policy.json new file mode 100644 index 0000000..bd561fd --- /dev/null +++ b/infra/minio/api-policy.json @@ -0,0 +1,45 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ], + "Condition": { + "StringLike": { + "s3:prefix": [ + "originals/*", + "staging/*", + "variants/*" + ] + } + } + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + } + ] +} diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 0000000..7d9eb39 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy create local messenger-api /config/api-policy.json >/dev/null +mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).' diff --git a/infra/minio/server.sh b/infra/minio/server.sh new file mode 100644 index 0000000..aedb392 --- /dev/null +++ b/infra/minio/server.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +exec minio "$@" diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 0000000..5f0b43d --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,62 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log off; + server_tokens off; + limit_conn_zone $server_name zone=uploads:64k; + + # Docker DNS; resolve updates the upstream when replicas get new IPs. + resolver 127.0.0.11 valid=1s ipv6=off; + resolver_timeout 1s; + + upstream messenger { + zone messenger 64k; + # All A records of api participate in the default round-robin balancing. + server api:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + server_name messenger; + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + limit_conn uploads 16; + client_max_body_size 11m; + client_body_buffer_size 64k; + client_body_timeout 10s; + proxy_request_buffering on; + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + } + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + + # Passive checks. Do NOT enable non_idempotent: an already-sent POST + # must not be resent by Nginx after an ambiguous response. + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 6s; + } + } +} diff --git a/infra/observability/alerts.yaml b/infra/observability/alerts.yaml new file mode 100644 index 0000000..7634f00 --- /dev/null +++ b/infra/observability/alerts.yaml @@ -0,0 +1,17 @@ +groups: + - name: messenger + rules: + - alert: HighApiErrorRate + expr: sum(rate(http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001) > 0.05 + for: 15s + labels: {severity: warning} + annotations: + summary: Business API 5xx exceeds 5% in 1 minute + runbook: Check postgres.query error spans, DB pool and request logs. Restore DB and confirm resolved. + - alert: QueueLagGrowing + expr: max(queue_oldest_age_seconds) > 15 + for: 15s + labels: {severity: warning} + annotations: + summary: Oldest unfinished attachment is older than 15 seconds + runbook: Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains. diff --git a/infra/observability/collector.yaml b/infra/observability/collector.yaml new file mode 100644 index 0000000..56cf511 --- /dev/null +++ b/infra/observability/collector.yaml @@ -0,0 +1,80 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + resource_to_telemetry_conversion: + enabled: true + metric_expiration: 10s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 2s + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 20s +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - prometheus + logs: + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - otlphttp/loki + traces: + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - otlp/tempo diff --git a/infra/observability/grafana/dashboards/messenger.json b/infra/observability/grafana/dashboards/messenger.json new file mode 100644 index 0000000..df523b2 --- /dev/null +++ b/infra/observability/grafana/dashboards/messenger.json @@ -0,0 +1,436 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger \u2014 API and pipeline", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "lab6" + ], + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Business RPS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total[1m]))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "id": 2, + "type": "timeseries", + "title": "5xx ratio", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 4, + "type": "timeseries", + "title": "HTTP p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "HTTP p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Jobs by outcome", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(outcome)(rate(jobs_processed_total[1m]))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Job p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Accepted to ready p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_depth)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Oldest unfinished job", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "max(queue_oldest_age_seconds)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "id": 11, + "type": "timeseries", + "title": "DB pool used / waiting", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(db_pool_used)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 12, + "type": "timeseries", + "title": "DB pool waiting", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by(service_name)(db_pool_waiting)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Process RSS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "process_memory_bytes", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Process CPU cores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total[1m])", + "refId": "A", + "legendFormat": "{{service_name}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + ] +} diff --git a/infra/observability/grafana/provisioning/dashboards/dashboards.yaml b/infra/observability/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..a1c102a --- /dev/null +++ b/infra/observability/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Messenger + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/infra/observability/grafana/provisioning/datasources/datasources.yaml b/infra/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..ebfd959 --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,28 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + - name: Loki + uid: loki + type: loki + url: http://loki:3100 + access: proxy + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + url: http://tempo:3200 + access: proxy + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true diff --git a/infra/observability/loki.yaml b/infra/observability/loki.yaml new file mode 100644 index 0000000..457684b --- /dev/null +++ b/infra/observability/loki.yaml @@ -0,0 +1,28 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: {store: inmemory} + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: {prefix: index_, period: 24h} +limits_config: + allow_structured_metadata: true + retention_period: 168h +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/observability/prometheus.yaml b/infra/observability/prometheus.yaml new file mode 100644 index 0000000..699193f --- /dev/null +++ b/infra/observability/prometheus.yaml @@ -0,0 +1,9 @@ +global: + scrape_interval: 2s + evaluation_interval: 2s +rule_files: [/etc/prometheus/alerts.yaml] +scrape_configs: + - job_name: messenger + static_configs: [{targets: ['collector:9464']}] + - job_name: collector + static_configs: [{targets: ['collector:8888']}] diff --git a/infra/observability/tempo.yaml b/infra/observability/tempo.yaml new file mode 100644 index 0000000..e783e9c --- /dev/null +++ b/infra/observability/tempo.yaml @@ -0,0 +1,21 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +ingester: + trace_idle_period: 2s + max_block_duration: 10s +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: {path: /var/tempo/wal} + local: {path: /var/tempo/blocks} +usage_report: + reporting_enabled: false diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..7b8c533 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..855e66a --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +-- Additive: old lab2 tables and messages remain valid. +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..a3de2f8 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,23 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id,id); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id,attachment_id), + UNIQUE (message_id,position) +); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 0000000..51f3c12 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1), + attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0, + lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text +); +CREATE TABLE job_outbox ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz +); +CREATE TABLE job_replays ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL +); +CREATE INDEX jobs_due ON attachment_jobs(next_at); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..20d8343 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,61 @@ +annotated-types==0.8.0 +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +attrs==26.1.0 +boto3==1.38.27 +botocore==1.38.46 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.115.12 +googleapis-common-protos==1.75.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +importlib_metadata==8.7.1 +iniconfig==2.3.1 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +opentelemetry-api==1.34.1 +opentelemetry-exporter-otlp-proto-common==1.34.1 +opentelemetry-exporter-otlp-proto-http==1.34.1 +opentelemetry-proto==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-semantic-conventions==0.55b1 +packaging==26.3 +pathable==0.4.4 +pillow==11.3.0 +pluggy==1.7.0 +protobuf==5.29.6 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.21.0 +pytest==8.4.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.3 +redis==6.2.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +s3transfer==0.13.1 +six==1.17.0 +sniffio==1.3.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 +zipp==4.1.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8148c4c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +argon2-cffi==25.1.0 +pytest==8.4.0 +httpx==0.28.1 +pydantic==2.11.5 +anyio==4.9.0 +boto3==1.38.27 +Pillow==11.3.0 +opentelemetry-api==1.34.1 +opentelemetry-sdk==1.34.1 +opentelemetry-exporter-otlp-proto-http==1.34.1 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..96955fb --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +import secrets +root = Path('secrets') +p = root / 'postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +pw = p.read_text().strip() +for name, value in { + 'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger', + 'grafana_password': secrets.token_hex(24), + 'cache_url': 'redis://redis:6379/0', + 'minio_root_user': 'lab4-admin', + 'minio_root_password': secrets.token_hex(24), + 's3_access_key_id': 'messenger-api', + 's3_secret_access_key': secrets.token_hex(24), +}.items(): + path = root / name + if not path.exists(): + path.write_text(value) +# Bind-mounted Docker secrets must be readable by container uid 10001. +# Parent directory remains 0700 on the host. +for path in root.iterdir(): + path.chmod(0o444) +if not Path('.env').exists(): + Path('.env').write_text('SESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden).') +PY diff --git a/scripts/check_fresh_migrations.sh b/scripts/check_fresh_migrations.sh new file mode 100755 index 0000000..bdee148 --- /dev/null +++ b/scripts/check_fresh_migrations.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A unique disposable Compose project; never touches normal project volumes. +set -eu +cd "$(dirname "$0")/.." +project="mtusi-lab6-migration-check-$$" +if [ -n "$(docker compose -p "$project" ps -aq)" ]; then + echo 'Refusing to reuse a pre-existing scratch project' >&2 + exit 1 +fi +cleanup() { + docker compose -p "$project" down --volumes >/dev/null 2>&1 +} +trap cleanup EXIT INT TERM +docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init +docker compose -p "$project" run --rm --no-deps -e OTEL_ENABLED=0 api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache' +echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.' diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 0000000..34de692 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence.""" +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') + + +def command(args, timeout=120): + return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip() + + +def compose(*args, timeout=120): + return command(['docker', 'compose', *args], timeout=timeout) + + +def inspect(containers): + if not containers: + return [] + return json.loads(command(['docker','inspect',*containers])) + + +def container_ids(service=None, all_containers=False): + args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else []) + return compose(*args).split() + + +def probe(container, host='127.0.0.1', path='/health/live'): + code = '''import json,sys,urllib.request,urllib.error +try: + r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4) +except urllib.error.HTTPError as exc: + r=exc +print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')})) +''' + return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8)) + + +def replicas(): + items = [] + for data in inspect(container_ids('api')): + if data['State']['Running']: + result = probe(data['Id']) + items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']}) + return items + + +def snapshot(): + items = [] + for data in inspect(container_ids(all_containers=True)): + service = data['Config']['Labels']['com.docker.compose.service'] + item = { + 'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'), + 'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'], + 'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'], + 'published_ports':data['HostConfig']['PortBindings'], + 'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'), + 'networks':sorted(data['NetworkSettings']['Networks']), + } + if service == 'api' and data['State']['Running']: + result = probe(data['Id']) + item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']}) + items.append(item) + return items + + +def wait_healthy(container, timeout=60): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + data = inspect([container])[0] + if data['State'].get('Health',{}).get('Status') == 'healthy': + return + time.sleep(0.3) + raise RuntimeError('Container did not become healthy') + + +def wait_cluster(count=2, timeout=60): + deadline = time.monotonic()+timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response: + alias = response.headers.get('X-Instance-Id') + if alias: + seen.add(alias) + if len(seen) >= count: + return sorted(seen) + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise RuntimeError('Load balancer did not discover required API instances') + + +def stop_one(target): + if not re.fullmatch(r'[a-f0-9]{32}', target): + raise ValueError('Invalid target instance alias') + matching = [item for item in replicas() if item['instance']==target] + if len(matching) != 1: + raise RuntimeError('Target must match exactly one running API process') + item = matching[0] + started = time.monotonic() + command(['docker','stop','--time','7',item['container']],timeout=9) + state = inspect([item['container']])[0]['State'] + if state['Running'] or state['ExitCode'] != 0: + raise RuntimeError('API did not stop gracefully') + result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True} + (ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n') + return result + + +def restart(): + compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30) + compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40) + return wait_cluster(2,timeout=15) + + +def front_address(container): + networks = inspect([container])[0]['NetworkSettings']['Networks'] + return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front')) + + +def replace_one(container): + before = set(container_ids('api')) + if container not in before or len(before) != 2: + raise RuntimeError('Replacement expects exactly two running API containers') + old_address = front_address(container) + # Start-first ensures a genuinely different IP while the old address is still occupied. + compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api') + expanded = set(container_ids('api')) + added = expanded-before + if len(added) != 1 or not before.issubset(expanded): + raise RuntimeError('Start-first step unexpectedly changed an existing container') + replacement = added.pop() + wait_healthy(replacement) + new_address = front_address(replacement) + if new_address == old_address: + raise RuntimeError('Replacement did not get a different IP') + expected_three = {item['instance'] for item in replicas()} + if set(wait_cluster(3)) != expected_three: + raise RuntimeError('Load balancer did not discover all three live addresses') + command(['docker','stop','--time','7',container],timeout=10) + state = inspect([container])[0]['State'] + if state['ExitCode'] != 0: + raise RuntimeError('Old release did not terminate gracefully') + command(['docker','rm',container]) + compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api') + after = set(container_ids('api')) + if after != (before-{container}) | {replacement}: + raise RuntimeError('Sequential replacement changed the wrong containers') + expected = {item['instance'] for item in replicas()} + if set(wait_cluster(2)) != expected: + raise RuntimeError('Unexpected final backend aliases') + return {'old_container':container,'new_container':replacement, + 'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True, + 'instances_during_start_first':sorted(expected_three), + 'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec']) + parser.add_argument('arguments',nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.action == 'wait': + print(json.dumps({'instances':wait_cluster()})) + elif args.action == 'snapshot': + print(json.dumps(snapshot(),indent=2)) + elif args.action == 'stop-one': + print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID']))) + elif args.action == 'exec': + ids = container_ids('api') + if not ids or not args.arguments: + raise RuntimeError('A live API container and command are required') + raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT)) + elif args.action == 'restart': + print(json.dumps({'instances':restart()})) + else: + compose('run','--rm','--no-deps','migrate') + print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..3985fa9 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/cluster.py restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..ef14b25 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +exec python3 scripts/cluster.py stop-one diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py new file mode 100644 index 0000000..8d4d04b --- /dev/null +++ b/scripts/verify_cluster.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts.""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +import uuid + +import cluster + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from resilience import history + + +def save(name, value): + (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') + print('PASS:', name) + + +def percentiles(values): + values = sorted(values) + return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None + for name,p in [('p50',.50),('p95',.95),('p99',.99)]} + + +class Traffic: + def __init__(self, concurrency=4): + self.c = Client(timeout=6) + self.alice,self.bob,_,self.chat = self.c.bootstrap() + self.concurrency = concurrency + self.done = threading.Event() + self.lock = threading.Lock() + self.attempts = [] + self.confirmed = {} + self.prefix = uuid.uuid4().hex + + def worker(self, number): + client = Client(timeout=6) + index = 0 + while not self.done.is_set(): + text = self.prefix+'-'+str(number)+'-'+str(index) + key = self.prefix+'-'+str(number)+'-'+str(index) + started = time.monotonic() + try: + response = client.message(self.alice,self.chat,text,key=key) + row = response['json'] + attempt = {'started':started,'ended':time.monotonic(),'ok':True, + 'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']} + with self.lock: + self.confirmed[row['id']] = row + index += 1 + except Failure: + # Only idempotent message POST is retried, with its original body and key. + attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None} + time.sleep(0.02) + with self.lock: + self.attempts.append(attempt) + + def start(self): + self.started = time.monotonic() + self.pool = ThreadPoolExecutor(max_workers=self.concurrency) + self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)] + return self + + def stop(self): + self.done.set() + for future in self.futures: + future.result(timeout=10) + self.pool.shutdown() + self.ended = time.monotonic() + + def summary(self, start=None, end=None): + start = self.started if start is None else start + end = getattr(self,'ended',time.monotonic()) if end is None else end + with self.lock: + attempts = [a for a in self.attempts if start <= a['started'] < end] + success = [a for a in attempts if a['ok']] + duration = end-start + return {'seconds':round(duration,4),'concurrency':self.concurrency, + 'successful_requests':len(success),'failed_attempts':len(attempts)-len(success), + 'successful_rps':round(len(success)/duration,3), + 'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0, + 'successful_latency_ms':percentiles([a['latency'] for a in success]), + 'instances':sorted({a['instance'] for a in success})} + + def verify_data(self): + rows = history(self.c,self.bob,self.chat) + actual = {row['id']:row for row in rows} + check(len(actual)==len(rows),'duplicated message ids') + check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message') + check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed') + return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows), + 'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0} + + +def concurrent_replay(): + cluster.wait_cluster() + c = Client() + alice,bob,_,chat = c.bootstrap() + key = uuid.uuid4().hex + barrier = threading.Barrier(8) + def send(_): + local = Client() + barrier.wait() + return local.message(alice,chat,'Один результат через разные процессы',key=key) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send,range(8))) + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(instances)>=2,'concurrent retry did not reach different API processes') + check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice, + body={'text':'Другой body'},headers={'Idempotency-Key':key}) + check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates') + save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances), + 'responses_201':8,'messages_created':1,'different_body_status':409}) + + +def verify_persistence(): + before = cluster.snapshot() + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=190) + after = cluster.snapshot() + roles = ['api','postgres','redis','minio','broker','relay','worker','lb'] + previous = {item['container']:item for item in before} + for item in after: + if item['service'] in roles: + old = previous[item['container']] + check(old['started_at']!=item['started_at'],'service did not restart: '+item['service']) + check(old['volumes']==item['volumes'],'volume identities changed') + (ROOT/'evidence/persistence.txt').write_text(result.stdout) + save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()}) + + +def public_failover(): + cluster.wait_cluster() + try: + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=100) + (ROOT/'evidence/public-failover.txt').write_text(result.stdout) + save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(), + 'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())}) + finally: + cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90') + cluster.wait_cluster() + + +def reject_new_connection(survivor, address): + code = '''import json,socket,sys +try: + s=socket.create_connection((sys.argv[1],8080),timeout=1) +except OSError as exc: + print(json.dumps({'accepted':False,'error_type':type(exc).__name__})) +else: + s.close() + print(json.dumps({'accepted':True})) +''' + return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address])) + + +def failover_each(): + cluster.wait_cluster() + originals = cluster.replicas() + check(len(originals)==2,'two running replicas required') + traffic = Traffic().start() + phases = [] + try: + time.sleep(3) + baseline_end = time.monotonic() + baseline = traffic.summary(end=baseline_end) + for original in originals: + # Alias changes on restart, so resolve the current process of this exact container. + current = next(r for r in cluster.replicas() if r['container']==original['container']) + all_before = cluster.inspect([current['container']])[0] + address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front')) + survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container']) + read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1' + marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0] + start = time.monotonic() + try: + env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']} + subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10) + stop_returned = time.monotonic() + stopped = json.loads((ROOT/'evidence/last-stop.json').read_text()) + read_deadline = start+10 + while True: + try: + read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage') + check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage') + check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process') + read_done = time.monotonic() + break + except Failure: + check(time.monotonic()=start] + recent = [a for a in successful if a['started']>stop_returned] + recovery = min(a['ended'] for a in recent)-start if recent else None + after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None + check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start') + check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests') + phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4), + 'post_stop_write_seconds_from_stop_completion':round(after_stop,4), + 'read_during_outage':{'status':200,'confirmed_message_preserved':True, + 'seconds_from_action_start':round(read_done-start,4), + 'seconds_from_stop_completion':round(read_done-stop_returned,4)}, + 'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']}) + finally: + cluster.command(['docker','start',current['container']]) + cluster.wait_healthy(current['container']) + cluster.wait_cluster() + time.sleep(2) + finally: + traffic.stop() + save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases, + 'total':traffic.summary(),'data':traffic.verify_data(), + 'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2}) + + +def rolling_update(): + cluster.wait_cluster() + before = cluster.replicas() + check(len(before)==2,'rolling demonstration expects two replicas') + lb_before = next(item for item in cluster.snapshot() if item['service']=='lb') + target_release = 'lab6-'+uuid.uuid4().hex[:8] + traffic = Traffic().start() + replacements = [] + try: + time.sleep(3) + build_start = time.monotonic() + cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180) + build_end = time.monotonic() + migration_output = cluster.compose('run','--rm','--no-deps','migrate') + swap_start = time.monotonic() + for old in before: + step_start = time.monotonic() + result = cluster.replace_one(old['container']) + result['seconds'] = round(time.monotonic()-step_start,4) + replacements.append(result) + swap_end = time.monotonic() + time.sleep(3) + finally: + traffic.stop() + after = cluster.replicas() + lb_after = next(item for item in cluster.snapshot() if item['service']=='lb') + check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test') + check(all(r['release']==target_release for r in after),'new release not observed on all replicas') + check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement') + check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change') + save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release, + 'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True, + 'build_seconds':round(build_end-build_start,4),'migration_output':migration_output, + 'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end), + 'total':traffic.summary(),'data':traffic.verify_data(), + 'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'}) + + +def database_fixture(container, data): + code = '''import json,sys,psycopg +from psycopg import sql +from messenger.db import setting +p=json.load(sys.stdin) +with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c: + if p['action']=='install': + c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text']))) + c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) + result={'installed':True} + elif p['action']=='remove': + c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger']))) + c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) + result={'removed':True} + else: + result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]} +print(json.dumps(result)) +''' + response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT, + input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(response.stdout) + + +def graceful_shutdown(): + cluster.wait_cluster() + nodes = cluster.replicas() + target,survivor = nodes[0],nodes[1] + address = cluster.front_address(target['container']) + c = Client() + alice,bob,_,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/messages' + marker = 'drain_'+uuid.uuid4().hex + fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker} + database_fixture(survivor['container'],{**fixture,'action':'install'}) + code = '''import json,sys,urllib.request,urllib.error +p=json.load(sys.stdin) +r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']}) +try: + response=urllib.request.urlopen(r,timeout=6) +except urllib.error.HTTPError as exc: + response=exc +print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)})) +''' + request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex} + def in_flight(): + result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT, + input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(result.stdout) + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(in_flight) + try: + deadline = time.monotonic()+3 + while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']: + check(time.monotonic()0 and item['nano_cpus']>0,'resource limit missing') + admin_ports={'grafana':'3000','prometheus':'9090','loki':'3100','tempo':'3200'} + if item['service'] in admin_ports: + for bindings in (item['published_ports'] or {}).values(): + check(all(b['HostIp']=='127.0.0.1' and b['HostPort']==admin_ports[item['service']] for b in bindings),'admin port is not localhost-only') + elif item['service']!='lb': + check(not item['published_ports'],'service exposed a port bypassing LB') + check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas') + check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running') + save('infrastructure.json',{'result':'PASS','containers':state}) + + +def dependency_failures(): + c = Client(timeout=6) + actor = c.user('dependencies') + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result = {'result':'PASS'} + try: + cluster.compose('stop','-t','2','redis') + response = c.call('GET','/api/v1/users/me',actor=actor,schema='User') + check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed') + c.call('GET','/api/v1/users/me',401) + c.call('GET','/health/ready',schema='Health') + result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200} + finally: + cluster.compose('start','redis') + import urllib.request + import urllib.error + def status(path, token=None): + headers = {'Authorization':'Bearer '+token} if token else {} + request = urllib.request.Request(c.base+path,headers=headers) + started = time.monotonic() + try: + response = urllib.request.urlopen(request,timeout=6) + except urllib.error.HTTPError as error: + response = error + with response: + return response.code, round(time.monotonic()-started,4) + try: + cluster.compose('stop','-t','2','postgres') + code,elapsed = status('/api/v1/users/me',actor['token']) + check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request') + anonymous,_ = status('/api/v1/users/me') + check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage') + live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()] + check(all(code==200 for code in live),'API process liveness depends on DB') + result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous, + 'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'} + finally: + cluster.compose('start','postgres') + cluster.wait_cluster() + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result['same_session_after_recovery'] = True + save('dependency-failures.json',result) + + +def verify_nginx(): + state = cluster.snapshot() + lb = next(item for item in state if item['service']=='lb') + check(lb['image'].startswith('nginx:'),'load balancer must use Nginx') + cluster.command(['docker','exec',lb['container'],'nginx','-t']) + version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1']) + c = Client() + response = c.call('GET','/health/ready',schema='Health') + check(response['headers'].get('Server')=='nginx','public entry point is not Nginx') + check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas') + save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx', + 'config_syntax':'PASS','observed_instances':cluster.wait_cluster()}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?') + args = parser.parse_args() + modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover, + 'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure} + if args.mode=='all': + for function in modes.values(): + function() + else: + modes[args.mode]() + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_jobs.py b/scripts/verify_jobs.py new file mode 100644 index 0000000..f53bcef --- /dev/null +++ b/scripts/verify_jobs.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Explicit real fault injection; emits IDs/results, never session tokens.""" +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + + +def compose(*args): + if args and args[0]=='start': args=('up','-d','--no-deps',*args[1:]) + result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120) + if result.returncode: raise RuntimeError(result.stderr) + return result.stdout + + +def inside(body,**values): + script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()' + proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120) + if proc.returncode: raise RuntimeError(proc.stderr) + return json.loads(proc.stdout) + + +def state(aid): + return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid) + + +def wait_for(predicate,seconds=30): + end=time.monotonic()+seconds + while time.monotonic()0};obj.close()",aid=target['id']) + compose('kill','-s','SIGKILL','worker') + # unless-stopped can restart a killed worker; stop it to retain the fault boundary. + compose('stop','worker') + inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True") + trigger=False + crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt') + print('Waiting for real 90s lease expiry after killed worker.',flush=True) + started=time.monotonic() + wait_for(lambda: state(target['id'])['status']=='queued',110) + compose('start','worker') + ready(c,bob,target) + after=state(target['id']) + evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']} + # Re-deliver the completed original job. Ready metadata is immutable. + original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json'] + duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id']) + check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result') + evidence['duplicate']={'ready_noop':True} + # Retry exhaustion with real missing staging (transient storage failure), explicit replay. + compose('stop','worker','relay') + failed,failed_raw=c.attachment(alice,chat) + st=state(failed['id']) + inside("store.delete(v['key']);result=True",key=st['staging_key']) + for attempt in range(5): + inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id']) + failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json'] + check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id']) + compose('start','relay','worker') + ready(c,bob,failed) + evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409} + evidence['result']='PASS' + (ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n') + print('PASS: jobs-faults.json') + finally: + if trigger: + inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True") + compose('start','relay','worker') + + +if __name__=='__main__': main() diff --git a/scripts/verify_observability.py b/scripts/verify_observability.py new file mode 100644 index 0000000..6ffd6a1 --- /dev/null +++ b/scripts/verify_observability.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Query real backends and reproduce bounded incidents; no secret values saved.""" +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +from verify_jobs import compose,inside,wait_for,state + + +def get(url): + with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as response: + return json.load(response) + + +def query(expression): + return get('http://localhost:9090/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result'] + + +def alerts(): + return get('http://localhost:9090/api/v1/alerts')['data']['alerts'] + + +def alert_state(name,state): + rows=[a for a in alerts() if a['labels']['alertname']==name] + return any(a['state']==state for a in rows) if state=='firing' else not rows + + +def collect_names(trace): + names=[] + def walk(value): + if isinstance(value,dict): + if 'spanId' in value and 'name' in value: names.append(value['name']) + for item in value.values(): walk(item) + elif isinstance(value,list): + for item in value: walk(item) + walk(trace) + return names + + +def save(name,data): + (ROOT/'evidence'/name).write_text(json.dumps(data,indent=2)+'\n') + + +def main(): + c=Client();alice,bob,_,chat=c.bootstrap() + report={'result':'INCOMPLETE'} + lock_process=None + try: + # Controlled retry in the same causal chain. + compose('stop','worker') + response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202, + actor=alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'},schema='Attachment') + att=response['json'];trace_id=response['headers']['X-Trace-Id'] + compose('stop','minio');compose('start','worker') + wait_for(lambda: (lambda s:s['attempt']>=1 and s['status']=='queued')(state(att['id'])),30) + time.sleep(2) + compose('start','minio') + # API re-enters LB after readiness recovers. + wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0,60) + ready=c.wait_ready(bob,att) + def chain(): + try: + trace=get('http://localhost:3200/api/traces/'+trace_id) + names=collect_names(trace) + return trace if names.count('attachments.consume')>=2 and all(n in names for n in ['attachments.publish','attachments.process_objects','s3.put','postgres.query']) else None + except Exception: return None + trace=wait_for(chain,60) + save('upload-trace.json',trace) + logquery='{service_name=~"messenger-.*"} |= "'+trace_id+'"' + def correlated_logs(): + try: + result=get('http://localhost:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':logquery,'limit':100,'start':str(time.time_ns()-600_000_000_000)})) + streams=result['data']['result'] + services={item['stream'].get('service_name') for item in streams} + return result if {'messenger-api','messenger-worker'}<=services else None + except Exception: return None + + logs=wait_for(correlated_logs,60);save('upload-logs.json',logs) + report['trace']={'trace_id':trace_id,'span_names':collect_names(trace),'retry_attempts':state(att['id'])['attempt'],'api_worker_logs_correlated':True} + # Incident 1: worker stopped, queued age is measured by relay. + wait_for(lambda: alert_state('QueueLagGrowing','resolved'),40) + compose('stop','worker') + queued,_=c.attachment(alice,chat) + wait_for(lambda: alert_state('QueueLagGrowing','firing'),70) + save('queue-alert-firing.json',alerts()) + report['queue_incident']={'oldest_age':query('max(queue_oldest_age_seconds)'),'depth':query('max(queue_depth)'),'alert':'firing'} + compose('start','worker');c.wait_ready(bob,queued) + wait_for(lambda: alert_state('QueueLagGrowing','resolved'),30) + save('queue-alert-resolved.json',alerts());report['queue_incident']['after_restore']='resolved' + # Incident 2: sessions table lock affects business auth, readiness remains healthy. + code='import os\nfrom messenger.db import Database\nd=Database();d.open();c=d.pool.getconn()\nc.execute(\'SET statement_timeout=0\')\nc.execute(\'SET idle_in_transaction_session_timeout=0\')\nc.execute("SET application_name=\'lab6-fault-lock\'")\nc.commit()\nc.execute(\'LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE\')\nprint(\'locked\',flush=True)\ntry:\n c.execute(\'SELECT pg_sleep(90)\')\nfinally:\n os._exit(0)\n' + lock_process=subprocess.Popen(['docker','compose','exec','-T','api','python','-c',code],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + check(lock_process.stdout.readline().strip()=='locked','DB lock was not acquired') + failures=[] + started=time.monotonic() + while time.monotonic()-started<35: + r=c.call('GET','/api/v1/users/me',503,actor=alice) + failures.append({'status':r['status'],'elapsed':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id')}) + wait_for(lambda: alert_state('HighApiErrorRate','firing'),40) + save('db-alert-firing.json',alerts()) + # Terminate exact lock holder via PG primary, not the API container. + inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}") + lock_process.wait(timeout=10);lock_process=None + for _ in range(300): c.call('GET','/api/v1/users/me',actor=alice) + wait_for(lambda: alert_state('HighApiErrorRate','resolved'),75) + save('db-alert-resolved.json',alerts()) + report['db_incident']={'failures':failures,'before':'firing','after':'resolved','pool_waiting':query('sum(db_pool_waiting)')} + # Backend outage must not affect accepted messages or grow queues beyond configuration. + before=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip()) + compose('stop','tempo','loki') + count=0;start=time.monotonic() + while time.monotonic()-start<25: + c.message(alice,chat,'telemetry outage '+str(count),key=uuid.uuid4().hex);count+=1 + if count%20==0: time.sleep(.1) + during=query('{__name__=~"otelcol_(receiver_accepted_spans|exporter_sent_spans|exporter_send_failed_spans|exporter_queue_size|exporter_queue_capacity|exporter_enqueue_failed_spans).*"}') + after=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip()) + compose('start','tempo','loki') + report['telemetry_outage']={'successful_messages':count,'seconds':round(time.monotonic()-start,3),'collector_before':before,'collector_after':after,'collector_metrics':during,'queue_limit_batches':256,'collector_limit_mib':256} + # Every provisioned dashboard expression must return a successful Prometheus response. + dashboard=json.loads((ROOT/'infra/observability/grafana/dashboards/messenger.json').read_text()) + expressions=[] + for panel in dashboard['panels']: + for target in panel['targets']: + values=query(target['expr']);check(values,'No data for '+panel['title']) + expressions.append({'panel':panel['title'],'expression':target['expr'],'series':len(values)}) + report['dashboard_queries']=expressions + report['result']='PASS';save('observability.json',report) + print('PASS: observability.json, traces/logs and firing/resolved alerts',flush=True) + finally: + if lock_process is not None: + inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}") + compose('start','worker','minio','tempo','loki') + + +if __name__=='__main__': main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 0000000..46afc02 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,323 @@ +"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads.""" +import json +import asyncio +import time +from . import telemetry +import logging +import os +import re +from contextlib import asynccontextmanager +from uuid import uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from psycopg_pool import PoolTimeout, TooManyRequests +from redis.exceptions import RedisError +from starlette.exceptions import HTTPException +from starlette.concurrency import run_in_threadpool + +from .db import Database +from .errors import ApiError, invalid, not_found, unauthorized +from .models import CreateChat, CreateMessage, CreateUser, uuid_value +from .messages import store_message, validate_key, MESSAGE_FIELDS +from .object_store import ObjectStore, unavailable +from . import attachments +from urllib.parse import quote +from .pagination import decode_cursor, page, page_limit, public, stamp +from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session + +LOG = logging.getLogger('messenger') +USER_FIELDS = 'id, username, display_name, created_at' + + +def error_response(request, error): + return JSONResponse({'error': {'code': error.code, 'message': error.message, + 'request_id': request.state.request_id}}, + status_code=error.status, headers=error.headers) + + +def authenticate(request: Request): + token = bearer(request.headers.get('Authorization', '')) + token_hash = digest(token) + with request.app.state.db.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', + (token_hash,)).fetchone() + if not row: + raise unauthorized() + request.state.token_hash = token_hash + return row['user_id'] + + +def require_member(conn, chat_id, user_id): + found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone() + if not found: + raise not_found() + + +def chat_public(conn, row): + item = public(row) + item['member_ids'] = [str(member['user_id']) for member in conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()] + return item + + +def create_app(): + @asynccontextmanager + async def lifespan(app): + app.state.db = Database() + app.state.storage = ObjectStore() + app.state.upload_slots = asyncio.Semaphore(6) + app.state.db.open() + try: + telemetry.init("messenger-api",app.state.db) + yield + finally: + app.state.storage.close() + app.state.db.close() + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + app.state.instance_id = uuid4().hex + app.state.release = os.environ.get('APP_RELEASE', 'lab6-v1') + + async def request_context(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path) + if request.method == 'POST' and json_endpoint: + media = request.headers.get('Content-Type', '').split(';') + unsupported = media[0].strip().lower() != 'application/json' + for parameter in media[1:]: + name, _, value = parameter.strip().partition('=') + if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'): + unsupported = True + if unsupported: + if path != '/api/v1/users': + await run_in_threadpool(authenticate, request) + raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8') + try: + json.loads((await request.body()).decode('utf-8')) + except (UnicodeError, json.JSONDecodeError): + raise invalid() from None + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc) + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests): + response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно')) + except Exception as exc: + # Log exception class only: credentials, SQL and message content stay out of logs. + LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__) + response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка')) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = app.state.instance_id + response.headers['X-App-Release'] = app.state.release + return response + + @app.middleware('http') + async def traced_request(request: Request,call_next): + route=telemetry.route_name(request.url.path) + started=time.monotonic() + with telemetry.span(request.method+' '+route,kind=telemetry.SpanKind.SERVER, + context=telemetry.propagate.extract(dict(request.headers)), + attributes={'http.request.method':request.method,'http.route':route}) as active: + response=await request_context(request,call_next) + active.set_attribute('http.response.status_code',response.status_code) + if response.status_code>=500: active.set_status(telemetry.StatusCode.ERROR) + telemetry.http_done(request.method,route,response.status_code,time.monotonic()-started,request.state.request_id) + response.headers['X-Trace-Id']=format(active.get_span_context().trace_id,'032x') + return response + + @app.exception_handler(ApiError) + async def api_error(request, exc): + return error_response(request, exc) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, exc): + return error_response(request, invalid()) + + @app.exception_handler(HTTPException) + async def http_error(request, exc): + return error_response(request, not_found() if exc.status_code == 404 else invalid()) + + @app.get('/health/live') + def live(): + return {'status': 'ok'} + + @app.get('/health/ready') + def ready(request: Request): + try: + with request.app.state.db.pool.connection() as conn: + conn.execute('SELECT 1 FROM idempotency_records LIMIT 0') + conn.execute('SELECT status,variants FROM attachments LIMIT 0') + conn.execute('SELECT 1 FROM job_outbox LIMIT 0') + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn): + raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None + request.app.state.storage.check() + return {'status': 'ok'} + + @app.post('/api/v1/users', status_code=201) + def register(body: CreateUser, request: Request): + password_hash = HASHER.hash(body.password) + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute( + f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}', + (uuid4(), body.username, body.display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Имя пользователя занято') from None + return public(row) + + @app.post('/api/v1/auth/sessions', status_code=201) + async def create_session(request: Request): + # Run hashing and synchronous database calls outside the event loop. + username, password = basic_credentials(request.headers.get('Authorization', '')) + if await request.body(): + raise invalid() + return await run_in_threadpool(login, request, username, password) + + def login(request, username, password): + db = request.app.state.db + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone() + valid = check_password(row['password_hash'] if row else DUMMY_HASH, password) + if not row or not valid: + raise unauthorized(basic=True) + row.pop('password_hash') + token, expires_at = new_session(db.session_ttl) + with db.pool.connection() as conn: + conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)', + (digest(token), row['id'], expires_at)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)} + + @app.get('/api/v1/users/me') + def me(request: Request, user_id=Depends(authenticate)): + db = request.app.state.db + key = 'user:' + str(user_id) + try: + cached = db.cache.get(key) + if cached: + return json.loads(cached) + except (RedisError, ValueError): + pass + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone() + if not row: + raise unauthorized() + result = public(row) + try: + db.cache.set(key, json.dumps(result), ex=60) + except RedisError: + pass + return result + + @app.delete('/api/v1/auth/sessions/current', status_code=204) + def logout(request: Request, user_id=Depends(authenticate)): + with request.app.state.db.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,)) + return Response(status_code=204) + + @app.post('/api/v1/chats', status_code=201) + def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)): + if str(user_id) not in body.member_ids: + raise invalid() + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at', + (uuid4(), body.title)).fetchone() + # An unknown user raises AFTER the chat INSERT; transaction context rolls it all back. + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', + [(row['id'], uuid_value(member)) for member in body.member_ids]) + result = chat_public(conn, row) + except psycopg.errors.ForeignKeyViolation: + raise invalid() from None + return result + + @app.get('/api/v1/chats') + def list_chats(request: Request, user_id=Depends(authenticate)): + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'chats:' + str(user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (c.created_at,c.id) > (%s,%s)' if after else '' + params = (user_id, *after, limit+1) if after else (user_id, limit+1) + with request.app.state.db.pool.connection() as conn: + rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall() + result = page(scope, rows, limit) + result['items'] = [chat_public(conn, row) for row in rows[:limit]] + return result + + @app.get('/api/v1/chats/{chat_id}') + def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone() + return chat_public(conn, row) + + @app.post('/api/v1/chats/{chat_id}/messages', status_code=201) + def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + key = validate_key(request.headers.get('Idempotency-Key', '')) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids) + return public(row) + + @app.get('/api/v1/chats/{chat_id}/messages') + def history(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'messages:' + str(cid) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (m.created_at,m.id) > (%s,%s)' if after else '' + params = (cid, *after, limit+1) if after else (cid, limit+1) + rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+ + ' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall() + return page(scope, rows, limit) + + @app.post('/api/v1/chats/{chat_id}/attachments',status_code=202) + async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)): + cid=uuid_value(chat_id) + def access(): + with request.app.state.db.pool.connection() as conn: + require_member(conn,cid,user_id) + await run_in_threadpool(access) + return await attachments.upload(request,user_id,cid,filename) + + @app.get('/api/v1/attachments/{attachment_id}') + def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + return attachments.metadata(row) + + @app.get('/api/v1/attachments/{attachment_id}/content') + def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + if variant not in ('original','thumbnail','crop','optimized'): + raise invalid() + if row['status']!='ready': + raise ApiError(409,'conflict','Вложение ещё не готово') + item=row + key=row['object_key'] + if variant!='original': + item=next((v for v in row['variants'] if v['name']==variant),None) + if item is None: + raise not_found() + key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant + opened=request.app.state.storage.get(key) + if opened.length!=item['size_bytes']: + opened.close() + raise unavailable() + return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={ + 'Content-Length':str(item['size_bytes']), + 'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')}) + + return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 0000000..31e3b18 --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,175 @@ +"""Bounded binary upload, image validation and DB/S3 coordination.""" +import asyncio +from . import telemetry +import hashlib +import logging +import tempfile +import threading +import unicodedata +import warnings +from uuid import uuid4 + +import psycopg +from PIL import Image, PngImagePlugin, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from .errors import ApiError, invalid, not_found +from .pagination import public + +MAX_BYTES=10485760 +MAX_PIXELS=40000000 +Image.MAX_IMAGE_PIXELS=MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK=1048576 +PngImagePlugin.MAX_TEXT_MEMORY=1048576 +warnings.filterwarnings('error',category=Image.DecompressionBombWarning) +DECODER=threading.BoundedSemaphore(1) +LOG=logging.getLogger('messenger') +FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants' + + +def object_lock(conn,key,wait=True): + function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' + row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone() + return True if wait else row['locked'] + + +def filename_value(value): + if not value or not value.strip() or len(value)>255: + raise invalid() + try: + value.encode('utf-8') + except UnicodeError: + raise invalid() from None + if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value): + raise invalid() + return value + + +def metadata(row): + result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')}) + return result + + +def validate_image(file,mime): + if not DECODER.acquire(timeout=3): + raise ApiError(503,'unavailable','Обработка изображений занята') + try: + file.seek(0) + with Image.open(file) as image: + actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format) + if actual is None or actual!=mime: + raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME') + if image.width*image.height>MAX_PIXELS: + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') + image.verify() + file.seek(0) + except (Image.DecompressionBombWarning,Image.DecompressionBombError): + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None + except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError): + raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None + finally: + DECODER.release() + + +def compensate(db,store,key): + """A failed/unknown COMMIT is not proof of absence. Recheck under the same lock.""" + try: + with db.pool.connection() as conn: + if not object_lock(conn,key,wait=False): + return + exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone() + if not exists: + store.delete(key) + except Exception as exc: + # A periodic, grace-protected janitor will retry after dependency recovery. + LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__) + + +def persist(db,store,user_id,chat_id,filename,mime,file,length,digest): + aid=uuid4() + key='staging/'+aid.hex + try: + with db.pool.connection() as conn: + object_lock(conn,key) + store.put(key,file,length,mime) + row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) ' + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS, + (aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone() + job=uuid4() + ctx=telemetry.carrier() + conn.execute('INSERT INTO attachment_jobs(id,attachment_id,traceparent,tracestate) VALUES (%s,%s,%s,%s)',(job,aid,ctx.get('traceparent'),ctx.get('tracestate'))) + conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,)) + return metadata(row) + except psycopg.Error: + compensate(db,store,key) + raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None + except BaseException: + compensate(db,store,key) + raise + + +async def upload(request,user_id,chat_id,filename): + filename=filename_value(filename) + mime=request.headers.get('Content-Type','').split(';')[0].strip().lower() + if mime not in ('image/png','image/jpeg'): + raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg') + declared=request.headers.get('Content-Length') + if declared is not None: + try: + length=int(declared) + if length<0: + raise ValueError + except ValueError: + raise invalid() from None + if length>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + try: + await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1) + except TimeoutError: + raise ApiError(503,'unavailable','Слишком много загрузок') from None + try: + with tempfile.TemporaryFile(mode='w+b') as file: + total=0 + digest=hashlib.sha256() + async def read_body(): + nonlocal total + async for chunk in request.stream(): + total+=len(chunk) + if total>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + digest.update(chunk) + await run_in_threadpool(file.write,chunk) + try: + await asyncio.wait_for(read_body(),timeout=30) + except TimeoutError: + raise ApiError(400,'invalid_request','Время загрузки истекло') from None + except ClientDisconnect: + raise invalid() from None + await run_in_threadpool(validate_image,file,mime) + return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage, + user_id,chat_id,filename,mime,file,total,digest.hexdigest()) + finally: + request.app.state.upload_slots.release() + + +def lookup(conn,attachment_id,user_id): + row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+ + ' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone() + if not row: + raise not_found() + return row + + +class ObjectResponse(StreamingResponse): + def __init__(self,opened,**kwargs): + self.opened=opened + super().__init__(opened.chunks(),**kwargs) + + async def __call__(self,scope,receive,send): + try: + await super().__call__(scope,receive,send) + finally: + self.opened.close() diff --git a/src/messenger/cleanup.py b/src/messenger/cleanup.py new file mode 100644 index 0000000..d5b5dfa --- /dev/null +++ b/src/messenger/cleanup.py @@ -0,0 +1,72 @@ +"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock.""" +import argparse +from datetime import datetime,timezone,timedelta +import json +from pathlib import Path +import time + +from .attachments import object_lock +from .db import Database +from .object_store import ObjectStore + + +def sweep(db,store,grace_seconds=300,apply=False): + if grace_seconds<60: + raise ValueError('Grace must be at least60seconds') + cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds) + result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds, + 'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0} + for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)): + for obj in page.get('Contents',[]): + result['listed']+=1 + if obj['LastModified']>=cutoff: + result['fresh']+=1 + continue + with db.pool.connection() as conn: + if not object_lock(conn,obj['Key'],wait=False): + result['locked']+=1 + continue + if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone(): + result['tracked']+=1 + continue + result['orphans']+=1 + if apply: + store.delete(obj['Key']) + result['deleted']+=1 + return result + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--grace-seconds',type=int,default=300) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--dry-run',action='store_true') + parser.add_argument('--loop',action='store_true') + parser.add_argument('--interval',type=int,default=60) + args=parser.parse_args() + if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run): + parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run') + db=Database() + store=ObjectStore() + db.open() + try: + while True: + try: + result=sweep(db,store,args.grace_seconds,args.apply) + print(json.dumps(result),flush=True) + except Exception as exc: + print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True) + if not args.loop: + raise + finally: + Path('/tmp/cleanup-heartbeat').touch() + if not args.loop: + break + time.sleep(args.interval) + finally: + store.close() + db.close() + + +if __name__=='__main__': + main() diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 0000000..c90a4fe --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,62 @@ +"""Bounded dependencies; PostgreSQL is always the source of truth.""" +import os +from contextlib import contextmanager +from . import telemetry +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +def setting(name, default=None): + filename = os.environ.get(name + '_FILE') + value = Path(filename).read_text().strip() if filename else os.environ.get(name, default) + if value is None: + raise RuntimeError(f'Missing configuration: {name}') + return value + + +class TracedConnection(psycopg.Connection): + def execute(self,query,*args,**kwargs): + # Trace actual query wait, never a pooled lease containing S3/image work. + operation=str(query).split(None,1)[0].upper() if str(query).strip() else 'OTHER' + if operation not in ('SELECT','INSERT','UPDATE','DELETE'): operation='OTHER' + with telemetry.span('postgres.query',attributes={'db.system':'postgresql','db.operation.name':operation}): + try: + return super().execute(query,*args,**kwargs) + except Exception as exc: + telemetry.event('db_operation_failed',error_type=type(exc).__name__) + raise + + def commit(self): + with telemetry.span('postgres.commit',attributes={'db.system':'postgresql'}): + return super().commit() + + +class Database: + def __init__(self): + self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600')) + if self.session_ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + self.pool = ConnectionPool( + setting('DATABASE_URL'), connection_class=TracedConnection, min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')), + timeout=2, max_waiting=16, open=False, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'}, + ) + self.cache = Redis.from_url( + setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8, + retry=Retry(NoBackoff(), 0), + ) + + def open(self): + self.pool.open(wait=True, timeout=30) + + def close(self): + self.pool.close() + self.cache.close() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 0000000..082390e --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,19 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(): + return ApiError(400, 'invalid_request', 'Неверный запрос') + + +def unauthorized(basic=False): + scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme}) + + +def not_found(): + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/jobs.py b/src/messenger/jobs.py new file mode 100644 index 0000000..77dba64 --- /dev/null +++ b/src/messenger/jobs.py @@ -0,0 +1,231 @@ +"""Transactional outbox, fenced processing, bounded retries and operator replay.""" +from contextlib import nullcontext +import signal +import time +from datetime import datetime,timezone +from . import telemetry +import hashlib +import json +import logging +import os +from pathlib import Path +import random +import subprocess +import sys +import tempfile +from uuid import UUID + +from psycopg.types.json import Jsonb +from redis import Redis +from .db import setting +from .attachments import object_lock + +LOG=logging.getLogger('messenger') +STREAM='attachment-jobs' +GROUP='image-workers' +MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5')) +LEASE_SECONDS=90 + + +def broker_client(): + return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True, + socket_connect_timeout=1,socket_timeout=3,max_connections=4) + + +def job_lock(conn,job_id): + return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok', + ('job:'+str(job_id),)).fetchone()['ok'] + + +def unlock(conn,job_id): + conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),)) + + +def release_lock(conn,job_id): + # Session locks survive ROLLBACK. A failed transaction must be cleared before unlock. + try: + conn.rollback() + unlock(conn,job_id) + conn.commit() + except BaseException: + conn.close() # never return a session with an uncertain held lock to the pool + raise + + +def relay_tick(db,broker): + # Recovery uses the same session lock as the processor. A slow live worker is never fenced out. + with db.pool.connection() as conn: + expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE a.status='processing' AND j.lease_untilrow['size_bytes'] or total>10485760: raise ValueError('invalid_image') + file.write(chunk) + finally: + opened.close() + raw_hash=hashlib.sha256(source.read_bytes()).hexdigest() + if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']: + raise ValueError('invalid_image') + completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory], + capture_output=True,timeout=20) + if completed.returncode: + raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit') + variants=json.loads(completed.stdout) + with source.open('rb') as file: + store.put(row['object_key'],file,row['size_bytes'],row['content_type']) + for name,item in variants.items(): + with (Path(directory)/name).open('rb') as file: + store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type']) + return list(variants.values()) + + +def process_objects(store,row): + # A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread. + timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45')) + if not 1<=timeout<=45: raise ValueError('processing_limit') + row={**row,'telemetry_context':telemetry.carrier()} + with tempfile.TemporaryDirectory(prefix='image-job-') as directory: + process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory], + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE, + text=True,start_new_session=True) + try: + output,_=process.communicate(json.dumps(row,default=str),timeout=timeout) + except BaseException: + # Terminate processor AND decoder before lock release or another attempt. + try: os.killpg(process.pid,signal.SIGKILL) + except ProcessLookupError: pass + process.communicate() + raise + if process.returncode==65: raise ValueError('invalid_image') + if process.returncode==66: raise ValueError('processing_limit') + if process.returncode: raise RuntimeError('storage_unavailable') + return json.loads(output) + + +def work(db,store,job_id,delivery_attempt=None): + job_id=UUID(str(job_id)) + # Dedicated pooled session holds the lock, with short committed transactions between effects. + with db.pool.connection() as conn: + if not job_lock(conn,job_id): + return False + conn.commit() + try: + row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due ' + 'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone() + conn.commit() + if not row or row['status'] in ('ready','failed'): + return True + if row['status']!='queued' or not row['due']: + return False + if delivery_attempt is not None and delivery_attempt!=row['attempt']+1: + return True + attempt=row['attempt']+1 + if attempt>MAX_ATTEMPTS: + conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.commit() + return True + conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s", + (attempt,LEASE_SECONDS,job_id)) + conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.commit() + started=time.monotonic() + try: + variants=process_objects(store,row) + except Exception as exc: + permanent=isinstance(exc,ValueError) + final=permanent or attempt>=MAX_ATTEMPTS + code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable') + if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'): + code='processing_failed' + delay=min(30,2**(attempt-1))*random.uniform(.5,1.5) + conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + ('failed' if final else 'queued',code if final else None,row['id'])) + conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id)) + if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,)) + conn.commit() + telemetry.job_done('failed' if final else 'retry',time.monotonic()-started,attempt) + LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt) + return True + # Same live lock and attempt fence protect publication; no new attempt can publish concurrently. + current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone() + if current['attempt']!=attempt: + conn.rollback() + return False + conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (Jsonb(variants),row['id'])) + conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,)) + conn.commit() + telemetry.job_done('ready',time.monotonic()-started,attempt,(datetime.now(timezone.utc)-row['created_at']).total_seconds()) + LOG.info('job_completed attempt=%s',attempt) + clean_staging(db,store,row['id']) + return True + finally: + release_lock(conn,job_id) + + +def clean_staging(db,store,attachment_id=None): + with db.pool.connection() as conn: + rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL " + "AND (status='ready' OR (status='failed' AND updated_at200: raise ValueError('Short audit reason required') + with db.pool.connection() as conn: + if not job_lock(conn,job_id): raise ValueError('Job active') + try: + row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone() + if not row or row['status']!='failed' or not row['staging_key']: + raise ValueError('Replay requires failed job with retained staging') + conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,)) + conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,)) + conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason)) + conn.commit() + finally: + release_lock(conn,job_id) diff --git a/src/messenger/messages.py b/src/messenger/messages.py new file mode 100644 index 0000000..0659967 --- /dev/null +++ b/src/messenger/messages.py @@ -0,0 +1,54 @@ +"""One PostgreSQL transaction owns both a message and its retry result.""" +import hashlib +import json +import re +from uuid import uuid4 + +from .errors import ApiError, invalid, not_found +from .models import uuid_value + + +def validate_key(key): + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise invalid() + return key + + +MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids' + + +def store_message(conn, user_id, chat_id, text, key, attachment_ids=None): + attachment_ids=[] if attachment_ids is None else attachment_ids + payload={'text':text} + if attachment_ids: + payload['attachment_ids']=attachment_ids + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + params = (user_id, chat_id, key) + # A competing INSERT waits for the other transaction, then the next statement sees its result. + conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING", + (*params, fingerprint)) + record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active ' + 'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone() + if record['message_id'] is not None and record['active']: + if record['request_hash'] != fingerprint: + raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением') + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s', + (record['message_id'],)).fetchone() + ids=[uuid_value(item) for item in attachment_ids] + if ids: + allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall() + if len(allowed)!=len(ids): + raise not_found() + if any(a['status']=='failed' for a in allowed): + raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой') + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone() + if ids: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)', + [(row['id'],aid,i) for i,aid in enumerate(ids)]) + conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' " + 'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params)) + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 0000000..e591e4e --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,29 @@ +"""Versioned, atomic, repeatable migrations, serialized with an advisory lock.""" +import hashlib +from pathlib import Path + +import psycopg +from .db import setting + + +def main(): + directory = Path(__file__).resolve().parents[2] / 'migrations' + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(20261102)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(directory.glob('*.sql')): + sql = path.read_text() + checksum = hashlib.sha256(sql.encode()).hexdigest() + found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone() + if found: + if found[0] != checksum: + raise RuntimeError('Applied migration checksum changed: ' + path.name) + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum)) + print('Applied', path.name) + print('Migrations up to date') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 0000000..85e6891 --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,75 @@ +import re +from uuid import UUID +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from .errors import invalid + + +def uuid_value(value): + try: + # Require the UUID string representation, not integer/coercible values. + if not isinstance(value, str) or len(value) != 36: + raise ValueError + return UUID(value) + except (ValueError, TypeError, AttributeError): + raise invalid() from None + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + @field_validator('*') + @classmethod + def valid_strings(cls, value, info): + if isinstance(value, str): + if not value.strip() and info.field_name != 'text': + raise ValueError('Blank string') + # PostgreSQL cannot store NUL or unpaired Unicode surrogates. + if '\x00' in value: + raise ValueError('NUL is not supported') + value.encode('utf-8') + return value + + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('username') + @classmethod + def username_ascii(cls, value): + if not re.fullmatch(r'[a-z0-9_]{3,32}', value): + raise ValueError('Invalid username') + return value + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('member_ids') + @classmethod + def members(cls, value): + ids = [uuid_value(item) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('Duplicate member') + return [str(item) for item in ids] + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list,max_length=10) + + @field_validator('attachment_ids') + @classmethod + def attachment_values(cls,value): + ids=[uuid_value(item) for item in value] + if len(ids)!=len(set(ids)): + raise ValueError('Duplicate attachment') + return [str(item) for item in ids] + + @model_validator(mode='after') + def nonempty_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('Message must contain text or attachment') + return self diff --git a/src/messenger/object_store.py b/src/messenger/object_store.py new file mode 100644 index 0000000..1bfb27c --- /dev/null +++ b/src/messenger/object_store.py @@ -0,0 +1,123 @@ +"""Small bounded S3 client. API credentials never contain root/admin permissions.""" +from contextlib import contextmanager +from functools import wraps +from . import telemetry +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from .db import setting +from .errors import ApiError + +PREFIX = 'originals/' + + +def unavailable(): + return ApiError(503,'unavailable','Хранилище файлов временно недоступно') + + +class OpenObject: + def __init__(self,body,release,length): + self.body=body + self.release=release + self.length=length + self.closed=False + self.close_lock=threading.Lock() + + def chunks(self): + try: + while True: + data=self.body.read(65536) + if not data: + break + yield data + finally: + self.close() + + def close(self): + with self.close_lock: + if self.closed: + return + self.closed=True + try: + self.body.close() + finally: + self.release() + + +class ObjectStore: + def __init__(self): + self.bucket=setting('S3_BUCKET','messenger-attachments') + self.gate=threading.BoundedSemaphore(8) + self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'), + region_name=setting('S3_REGION','us-east-1'), + aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), + config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8, + retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'}, + request_checksum_calculation='when_required',response_checksum_validation='when_required')) + + @contextmanager + def operation(self): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + yield + except (BotoCoreError,ClientError): + raise unavailable() from None + finally: + self.gate.release() + + def check(self): + with self.operation(): + self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1) + + def put(self,key,body,length,mime): + with self.operation(): + body.seek(0) + self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime) + + def delete(self,key): + with self.operation(): + self.client.delete_object(Bucket=self.bucket,Key=key) + + def get(self,key): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + result=self.client.get_object(Bucket=self.bucket,Key=key) + return OpenObject(result['Body'],self.gate.release,result['ContentLength']) + except (BotoCoreError,ClientError): + self.gate.release() + raise unavailable() from None + except BaseException: + self.gate.release() + raise + + def pages(self,prefix=PREFIX): + token=None + while True: + params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000} + if token: + params['ContinuationToken']=token + with self.operation(): + result=self.client.list_objects_v2(**params) + yield result + if not result.get('IsTruncated'): + break + token=result['NextContinuationToken'] + + def close(self): + self.client.close() + + +def traced_s3(function): + @wraps(function) + def wrapped(*args,**kwargs): + with telemetry.span('s3.'+function.__name__,attributes={'rpc.system':'aws-api','rpc.service':'S3'}): + return function(*args,**kwargs) + return wrapped + + +for _operation in ('put','get','delete','check'): + setattr(ObjectStore,_operation,traced_s3(getattr(ObjectStore,_operation))) diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 0000000..14564c5 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,56 @@ +import base64 +import json +import re +from datetime import datetime, timezone +from uuid import UUID +from .errors import invalid +from .models import uuid_value + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(scope, row): + raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip('=') + + +def decode_cursor(scope, value): + if value is None: + return None + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value): + raise ValueError + data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + raise ValueError + ts = datetime.fromisoformat(data[1].replace('Z', '+00:00')) + if ts.tzinfo is None: + raise ValueError + return ts, uuid_value(data[2]) + except (ValueError, TypeError, AttributeError, OverflowError): + raise invalid() from None + + +def page_limit(value): + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + raise invalid() + return int(value) + + +def public(row): + def value(item): + if isinstance(item,datetime): + return stamp(item) + if isinstance(item,UUID): + return str(item) + if isinstance(item,list): + return [value(part) for part in item] + return item + return {key:value(item) for key,item in row.items()} + + +def page(scope, rows, limit): + return {'items': [public(row) for row in rows[:limit]], + 'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None} diff --git a/src/messenger/pipeline.py b/src/messenger/pipeline.py new file mode 100644 index 0000000..806b431 --- /dev/null +++ b/src/messenger/pipeline.py @@ -0,0 +1,49 @@ +"""Bounded image decoder subprocess. Original bytes are never re-encoded.""" +import hashlib +import json +import resource +import sys +from pathlib import Path +from PIL import Image, ImageOps, PngImagePlugin + + +def fitted(image, maximum): + scale=min(1,maximum/image.width,maximum/image.height) + size=(max(1,int(image.width*scale)),max(1,int(image.height*scale))) + return image.resize(size,Image.Resampling.LANCZOS) + + +def process_file(source, directory): + Image.MAX_IMAGE_PIXELS=40000000 + PngImagePlugin.MAX_TEXT_CHUNK=1048576 + PngImagePlugin.MAX_TEXT_MEMORY=1048576 + with Image.open(source) as original: + if original.format not in ('PNG','JPEG') or original.width*original.height>40000000: + raise ValueError('invalid_image') + image=ImageOps.exif_transpose(original) + # Alpha is composited on white; output carries no source metadata. + rgba=image.convert('RGBA') + white=Image.new('RGBA',rgba.size,'white') + white.alpha_composite(rgba) + image=white.convert('RGB') + image.info.clear() + outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'), + 'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'), + 'optimized':(fitted(image,1280),'WEBP','image/webp')} + manifest={} + for name,(variant,fmt,mime) in outputs.items(): + path=Path(directory)/name + variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'') + raw=path.read_bytes() + manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height, + 'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()} + return manifest + + +if __name__=='__main__': + resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024)) + resource.setrlimit(resource.RLIMIT_CPU,(15,15)) + try: + print(json.dumps(process_file(sys.argv[1],sys.argv[2]))) + except (OSError,ValueError,MemoryError,Image.DecompressionBombError): + sys.exit(65) diff --git a/src/messenger/processor.py b/src/messenger/processor.py new file mode 100644 index 0000000..10a5cac --- /dev/null +++ b/src/messenger/processor.py @@ -0,0 +1,33 @@ +"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds.""" +import os +import signal +import json +from . import telemetry +import sys +from uuid import UUID +from .jobs import process_objects_local +from .object_store import ObjectStore + + +def main(): + # Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline. + signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL)) + signal.alarm(45) + row=json.load(sys.stdin) + row['id']=UUID(row['id']) + telemetry.init('messenger-worker',child=True) + store=ObjectStore() + try: + with telemetry.span('attachments.process_objects',context=telemetry.propagate.extract(row.get('telemetry_context',{}))): + print(json.dumps(process_objects_local(store,row,sys.argv[1]))) + except ValueError as exc: + sys.exit(66 if str(exc)=='processing_limit' else 65) + except Exception: + # Do not export credentials, storage keys or SDK response details. + sys.exit(75) + finally: + store.close() + telemetry.shutdown() + + +if __name__=='__main__': main() diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 0000000..b82c6b0 --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,49 @@ +import base64 +import hashlib +import re +import secrets +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from .errors import unauthorized + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash('synthetic-dummy-password') + + +def digest(token): + return hashlib.sha256(token.encode()).hexdigest() + + +def basic_credentials(header): + try: + scheme, encoded = header.split(' ', 1) + if scheme.lower() != 'basic': + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError): + raise unauthorized(basic=True) from None + + +def check_password(encoded, password): + try: + return HASHER.verify(encoded, password) + except (VerificationError, InvalidHashError): + return False + + +def bearer(header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError + return token + except ValueError: + raise unauthorized() from None + + +def new_session(ttl): + return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl) diff --git a/src/messenger/telemetry.py b/src/messenger/telemetry.py new file mode 100644 index 0000000..e4a1782 --- /dev/null +++ b/src/messenger/telemetry.py @@ -0,0 +1,154 @@ +"""Explicit low-cardinality telemetry; no bodies, headers, storage keys or SQL params.""" +from contextlib import contextmanager +import json +import logging +import os +import re +import time +from uuid import uuid4 + +from opentelemetry import trace,metrics,propagate +from opentelemetry.trace import SpanKind,StatusCode + +HTTP_ROUTES={'/api/v1/users','/api/v1/users/me','/api/v1/auth/sessions','/api/v1/auth/sessions/current', + '/api/v1/chats','/health/live','/health/ready'} +ALLOWED={'event','request_id','route','method','status_class','outcome','attempt','error_type'} +INITIALIZED=False +PROVIDERS=[] +HANDLER=None +SERVICE='messenger-api' +BUSY=0 +HTTP_COUNT=HTTP_TIME=JOB_COUNT=JOB_TIME=READY_TIME=None + + +def route_name(path): + if path in HTTP_ROUTES: return path + if re.fullmatch(r'/api/v1/chats/[^/]+(?:/(?:messages|attachments))?',path): + return re.sub(r'(?<=/chats/)[^/]+','{chat_id}',path) + if re.fullmatch(r'/api/v1/attachments/[^/]+(?:/content)?',path): + return re.sub(r'(?<=/attachments/)[^/]+','{attachment_id}',path) + return 'unmatched' + + +def safe_fields(fields): + return {key:value for key,value in fields.items() if key in ALLOWED and isinstance(value,(str,int,float,bool))} + + +def tracer(): return trace.get_tracer('messenger','1.0') + + +def carrier(): + data={};propagate.inject(data);return data + + +@contextmanager +def span(name,kind=SpanKind.INTERNAL,context=None,attributes=None): + with tracer().start_as_current_span(name,kind=kind,context=context,attributes=attributes or {}, + record_exception=False,set_status_on_exception=False) as active: + try: yield active + except BaseException: + active.set_status(StatusCode.ERROR) + raise + + +def event(name,**fields): + if HANDLER is None: return + context=trace.get_current_span().get_span_context() + data=safe_fields({'event':name,**fields}) + data.update(timestamp=time.time(),severity='INFO',service_name=SERVICE, + trace_id=format(context.trace_id,'032x'),span_id=format(context.span_id,'016x')) + record=logging.LogRecord('messenger.events',logging.INFO,'',0,data,(),None) + HANDLER.emit(record) + print(json.dumps(data),flush=True) + + +def init(service,db=None,child=False): + global INITIALIZED,HANDLER,SERVICE,HTTP_COUNT,HTTP_TIME,JOB_COUNT,JOB_TIME,READY_TIME + if INITIALIZED or os.environ.get('OTEL_ENABLED')!='1': return + INITIALIZED=True;SERVICE=service + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.metrics import Observation + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import View,ExplicitBucketHistogramAggregation + from opentelemetry.sdk._logs import LoggerProvider,LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + resource=Resource.create({'service.name':service,'service.version':'lab6-v1', + 'service.instance.id':os.environ.setdefault('TELEMETRY_INSTANCE_ID',uuid4().hex), + 'deployment.environment.name':'lab'}) + endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT','http://collector:4318').rstrip('/') + tp=TracerProvider(resource=resource) + tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint+'/v1/traces',timeout=1), + max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000)) + trace.set_tracer_provider(tp);PROVIDERS.append(tp) + lp=LoggerProvider(resource=resource) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint+'/v1/logs',timeout=1), + max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000)) + HANDLER=LoggingHandler(level=logging.INFO,logger_provider=lp);PROVIDERS.append(lp) + if child: return + reader=PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint+'/v1/metrics',timeout=1), + export_interval_millis=2000,export_timeout_millis=1500) + mp=MeterProvider(resource=resource,metric_readers=[reader],views=[ + View(instrument_name='*_duration',aggregation=ExplicitBucketHistogramAggregation([.005,.01,.025,.05,.1,.25,.5,1,2,5,10,30,60,120])), + View(instrument_name='attachment_ready',aggregation=ExplicitBucketHistogramAggregation([.1,.5,1,2,5,10,30,60,120,300]))]) + metrics.set_meter_provider(mp);PROVIDERS.append(mp) + meter=metrics.get_meter('messenger','1.0') + HTTP_COUNT=meter.create_counter('http_requests') + HTTP_TIME=meter.create_histogram('http_server_duration',unit='s') + JOB_COUNT=meter.create_counter('jobs_processed') + JOB_TIME=meter.create_histogram('job_processing_duration',unit='s') + READY_TIME=meter.create_histogram('attachment_ready',unit='s') + def process_values(_): + rss=int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE') + return [Observation(rss)] + meter.create_observable_gauge('process_memory',unit='By',callbacks=[process_values]) + meter.create_observable_counter('process_cpu',unit='s',callbacks=[lambda _:[Observation(time.process_time())]]) + meter.create_observable_gauge('worker_busy',callbacks=[lambda _:[Observation(BUSY)]]) + meter.create_observable_gauge('worker_capacity',callbacks=[lambda _:[Observation(1 if service=='messenger-worker' else 0)]]) + if db: + def pool_value(name): + return lambda _:[Observation(db.pool.get_stats().get(name,0))] + meter.create_observable_gauge('db_pool_used',callbacks=[lambda _:[Observation(db.pool.get_stats().get('pool_size',0)-db.pool.get_stats().get('pool_available',0))]]) + meter.create_observable_gauge('db_pool_waiting',callbacks=[pool_value('requests_waiting')]) + def queue_values(field): + def callback(_): + try: + with db.pool.connection() as conn: + row=conn.execute("SELECT count(*) AS depth,coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0)::float AS age FROM attachments WHERE status IN ('queued','processing')").fetchone() + return [Observation(row[field],{'queue':'attachments'})] + except Exception: return [] + return callback + if service=='messenger-relay': + meter.create_observable_gauge('queue_depth',callbacks=[queue_values('depth')]) + meter.create_observable_gauge('queue_oldest_age',unit='s',callbacks=[queue_values('age')]) + + +def http_done(method,route,status,duration,request_id): + fields={'method':method,'route':route,'status_class':str(status//100)+'xx'} + if HTTP_COUNT and not route.startswith('/health/'): + HTTP_COUNT.add(1,fields);HTTP_TIME.record(duration,fields) + event('http_completed',**fields,request_id=request_id) + + +def job_done(outcome,duration,attempt,accepted_age=None): + current=trace.get_current_span() + current.set_attribute('job.type','attachment.process.v1') + current.set_attribute('job.outcome',outcome) + current.set_attribute('job.attempt',attempt) + if outcome in ('retry','failed'): current.set_status(StatusCode.ERROR) + attrs={'job_type':'attachment.process.v1','outcome':outcome} + if JOB_COUNT: + JOB_COUNT.add(1,attrs);JOB_TIME.record(duration,attrs) + if accepted_age is not None and outcome=='ready': READY_TIME.record(accepted_age,{'job_type':'attachment.process.v1'}) + event('job_attempt',outcome=outcome,attempt=attempt) + + +def shutdown(): + for provider in reversed(PROVIDERS): + try: provider.shutdown() + except Exception: pass diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 0000000..f6b87ca --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,77 @@ +"""Separate relay/consumer processes; durable Redis Streams consumer group.""" +import argparse +from . import telemetry +import json +import logging +import signal +import time +from uuid import uuid4 +from redis.exceptions import ResponseError +from .cleanup import sweep +from .db import Database +from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP +from .object_store import ObjectStore + +LOG=logging.getLogger('messenger') +STOP=False + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('role',choices=['worker','relay','replay']) + parser.add_argument('--job-id') + parser.add_argument('--reason',default='operator retry') + args=parser.parse_args() + logging.basicConfig(level=logging.INFO) + def stop(*_): + global STOP + STOP=True + signal.signal(signal.SIGTERM,stop) + signal.signal(signal.SIGINT,stop) + db=Database();db.open();store=ObjectStore();broker=broker_client() + telemetry.init('messenger-worker' if args.role=='worker' else 'messenger-relay',db) + consumer=uuid4().hex + last_cleanup=0 + try: + if args.role=='replay': + replay(db,args.job_id,args.reason) + print('Replay queued with audit') + return + if args.role=='worker': + try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True) + except ResponseError as exc: + if 'BUSYGROUP' not in str(exc): raise + while not STOP: + try: + if args.role=='relay': + relay_tick(db,broker) + clean_staging(db,store) + if time.monotonic()-last_cleanup>60: + sweep(db,store,300,True) + last_cleanup=time.monotonic() + time.sleep(.3) + else: + claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=5000,start_id='0-0',count=1) + messages=claimed[1] + if not messages: + batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000) + messages=batches[0][1] if batches else [] + for message_id,fields in messages: + payload=json.loads(fields['payload']) + with telemetry.span('attachments.consume',kind=telemetry.SpanKind.CONSUMER, + context=telemetry.propagate.extract(payload)): + telemetry.BUSY=1 + try: + if work(db,store,payload['job_id'],payload['attempt']): + broker.xack(STREAM,GROUP,message_id) + broker.xdel(STREAM,message_id) + finally: + telemetry.BUSY=0 + except Exception as exc: + LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__) + time.sleep(1) + finally: + broker.close();store.close();db.close();telemetry.shutdown() + + +if __name__=='__main__': main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..6342a6a --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,141 @@ +"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities.""" +import base64 +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fastapi.testclient import TestClient +from messenger.app import create_app + +PASSWORD = 'Тестовый:пароль🔐' * 8 + + +@pytest.fixture +def client(): + with TestClient(create_app()) as c: + yield c + + +def user(c): + body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD) + r = c.post('/api/v1/users', json=body) + assert r.status_code == 201, r.text + return r.json(), body + + +def login(c, body): + credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode() + r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials}) + assert r.status_code == 201, r.text + return {'Authorization': 'Bearer ' + r.json()['token']}, r.json() + + +def test_auth_unicode_logout_and_cache(client): + u, body = user(client) + h, session = login(client, body) + assert session['user'] == u + assert len(session['token']) >= 43 + key = 'user:' + u['id'] + client.app.state.db.cache.delete(key) + for _ in range(2): + assert client.get('/api/v1/users/me', headers=h).json() == u + cached = json.loads(client.app.state.db.cache.get(key)) + assert cached == u + assert 0 < client.app.state.db.cache.ttl(key) <= 60 + assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204 + r = client.get('/api/v1/users/me', headers=h) + assert r.status_code == 401 + assert r.headers['www-authenticate'] == 'Bearer realm="messenger"' + assert r.json()['error']['request_id'] == r.headers['x-request-id'] + + +def test_session_expiry(monkeypatch): + monkeypatch.setenv('SESSION_TTL_SECONDS', '2') + with TestClient(create_app()) as client: + _, body = user(client) + h, _ = login(client, body) + assert client.get('/api/v1/users/me', headers=h).status_code == 200 + time.sleep(2.1) + assert client.get('/api/v1/users/me', headers=h).status_code == 401 + + +def test_registration_race(client): + body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD) + with ThreadPoolExecutor(max_workers=6) as executor: + codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6))) + assert sorted(codes) == [201, 409, 409, 409, 409, 409] + + +def test_chat_rollback_after_insert(client): + u, body = user(client) + h, _ = login(client, body) + title = 'rollback_' + uuid.uuid4().hex + r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]}) + assert r.status_code == 400 + with client.app.state.db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0 + + +def test_cursor_access_and_strict_validation(client): + a, ab = user(client) + b, bb = user(client) + ah, _ = login(client, ab) + bh, _ = login(client, bb) + chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)] + path = '/api/v1/chats/' + chats[0]['id'] + for text in ['Один 👋', 'Два', 'Три']: + assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201 + p = client.get(path+'/messages?limit=2', headers=ah).json() + assert [m['text'] for m in p['items']] == ['Один 👋','Два'] + tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json() + assert [m['text'] for m in tail['items']] == ['Три'] + assert tail['next_cursor'] is None + r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']}) + assert r.status_code == 400 + assert client.get(path, headers=bh).status_code == 404 + for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]: + assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400 + for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']: + assert client.get(path+'/messages?'+query,headers=ah).status_code == 400 + for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]: + assert client.post('/api/v1/users',json=payload).status_code == 400 + assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400 + + +def test_session_storage_and_migrations(client): + from messenger.migrate import main as migrate + u, body = user(client) + _, session = login(client, body) + db = client.app.state.db + def snapshot(): + with db.pool.connection() as conn: + return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] + for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']} + before = snapshot() + migrate() + migrate() + assert snapshot() == before + with db.pool.connection() as conn: + row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone() + assert row['token_hash'] != session['token'] + assert len(row['token_hash']) == 64 + stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash'] + assert stored.startswith('$argon2id$') + assert stored != body['password'] + + +def test_json_media_type_and_utf8(client): + body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD) + raw = json.dumps(body).encode() + for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]: + r = client.post('/api/v1/users',content=raw,headers=headers) + assert r.status_code == 415 + assert r.json()['error']['code'] == 'unsupported_media_type' + for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']: + r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'}) + assert r.status_code == 400, encoding + r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'}) + assert r.status_code == 401 diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..a5b1dfa --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,187 @@ +"""Real PostgreSQL + private S3 integration; synthetic images only.""" +import hashlib +import io +import struct +import uuid +import zlib +from PIL import Image +from psycopg import sql +from test_api import client, user, login + + +def image_bytes(kind='PNG'): + stream = io.BytesIO() + Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind) + return stream.getvalue() + + +def actors(c): + a,ab=user(c) + b,bb=user(c) + e,eb=user(c) + ah,_=login(c,ab) + bh,_=login(c,bb) + eh,_=login(c,eb) + chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json() + return ah,bh,eh,chat + + +def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'): + raw=image_bytes() if raw is None else raw + return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw, + headers={**headers,'Content-Type':mime}) + + +def finish(client,attachment_id): + from messenger.jobs import work + with client.app.state.db.pool.connection() as conn: + job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id'] + work(client.app.state.db,client.app.state.storage,job) + import time + for _ in range(100): + with client.app.state.db.pool.connection() as conn: + status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status'] + if status=='ready': return + time.sleep(.1) + raise AssertionError('Job did not become ready: '+status) + + +def test_original_png_jpeg_and_message(client): + ah,bh,eh,chat=actors(client) + for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]: + raw=image_bytes(kind) + r=upload(client,ah,chat,raw,mime) + assert r.status_code==202 + att=r.json() + assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None + assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409 + finish(client,att['id']) + att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json() + assert att['status']=='ready' and len(att['variants'])==3 + assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw) + assert 'object_key' not in att and 'bucket' not in att + path='/api/v1/attachments/'+att['id'] + assert client.get(path,headers=bh).json()==att + content=client.get(path+'/content',headers=bh) + assert content.status_code==200 and content.content==raw + assert content.headers['Content-Type']==mime + for suffix in ['', '/content']: + assert client.get(path+suffix,headers=eh).status_code==404 + assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200 + response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, + json={'text':'','attachment_ids':[att['id']]}) + assert response.status_code==201 + assert response.json()['attachment_ids']==[att['id']] + + +def test_invalid_media_corruption_limits_and_filename(client): + ah,_,_,chat=actors(client) + cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415), + (image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415), + (image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)] + for raw,mime,expected in cases: + response=upload(client,ah,chat,raw,mime) + assert response.status_code==expected + assert response.json()['error']['request_id']==response.headers['X-Request-Id'] + # Valid PNG header advertises >40Mp; reject before attempting decompression. + header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0) + chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff) + bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'') + assert upload(client,ah,chat,bomb).status_code==413 + for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]: + assert upload(client,ah,chat,filename=filename).status_code==400 + + +def test_foreign_attachment_binding_and_normalized_replay(client): + ah,bh,eh,chat=actors(client) + att=upload(client,ah,chat).json() + other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json() + path='/api/v1/chats/'+other['id']+'/messages' + for aid in [att['id'],str(uuid.uuid4())]: + response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]}) + assert response.status_code==404 + assert upload(client,eh,chat).status_code==404 + h={**ah,'Idempotency-Key':uuid.uuid4().hex} + path='/api/v1/chats/'+chat['id']+'/messages' + first=client.post(path,headers=h,json={'text':'same'}) + replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'}) + assert first.status_code==replay.status_code==201 and first.json()==replay.json() + conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]}) + assert conflict.status_code==409 + duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]}) + assert duplicates.status_code==400 + + +def test_compensate_s3_when_metadata_insert_fails(client): + ah,_,_,chat=actors(client) + db=client.app.state.db + store=client.app.state.storage + before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])} + function='fail_attachment_'+uuid.uuid4().hex + trigger='fail_attachment_'+uuid.uuid4().hex + filename='fault_'+uuid.uuid4().hex+'.png' + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename))) + conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + response=upload(client,ah,chat,filename=filename) + assert response.status_code==503 + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0 + after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])} + assert after==before + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + + +def test_compensation_preserves_committed_object(client): + from messenger.attachments import compensate + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + # A retry after an unknown COMMIT result must never delete an already tracked object. + compensate(client.app.state.db,client.app.state.storage,key) + content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah) + assert content.status_code==200 and content.content==image_bytes() + + +def test_stream_close_even_if_headers_cannot_be_sent(client): + import asyncio + import pytest + from messenger.attachments import ObjectResponse + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + store=client.app.state.storage + opened=store.get(key) + response=ObjectResponse(opened,media_type='image/png') + async def fail_send(message): + raise OSError('synthetic disconnected transport') + async def receive(): + return {'type':'http.disconnect'} + with pytest.raises(Exception): + asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send)) + assert opened.closed + acquired=0 + try: + for _ in range(8): + assert store.gate.acquire(blocking=False) + acquired+=1 + finally: + for _ in range(acquired): + store.gate.release() + + +def test_chunked_upload_is_limited_without_content_length(client): + ah,_,_,chat=actors(client) + chunks=iter([b'x'*1048576]*10+[b'x']) + response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', + content=chunks,headers={**ah,'Content-Type':'image/png'}) + assert response.status_code==413 + assert response.json()['error']['code']=='payload_too_large' diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..826d3dc --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,119 @@ +"""Regressions for lab3 shared message results, real PostgreSQL (no mocks).""" +from concurrent.futures import ThreadPoolExecutor +import uuid + +from psycopg import sql +from test_api import client, user, login + + +def setup_chat(c, extra=None): + u, body = user(c) + headers, _ = login(c, body) + members = [u['id']] + (extra or []) + chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json() + return u, headers, '/api/v1/chats/'+chat['id']+'/messages' + + +def test_instance_alias_on_success_and_errors(client): + responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'), + client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})] + ids = [r.headers.get('X-Instance-Id') for r in responses] + assert ids[0] and len(set(ids)) == 1 + assert all(r.headers.get('X-Request-Id') for r in responses) + + +def test_key_required_replay_and_conflict(client): + _, headers, path = setup_chat(client) + assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400 + for key in ['', 'a b', 'bad/', 'x'*129]: + assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400 + h = {**headers, 'Idempotency-Key':'valid:._-123'} + first = client.post(path, headers=h, json={'text':'same'}) + assert first.status_code == 201 + replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ') + assert replay.status_code == 201 and replay.json() == first.json() + assert client.post(path, headers=h, json={'text':'other'}).status_code == 409 + assert len(client.get(path,headers=headers).json()['items']) == 1 + + +def test_concurrent_key_produces_one_message(client): + _, headers, path = setup_chat(client) + h = {**headers, 'Idempotency-Key':uuid.uuid4().hex} + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16))) + assert [r.status_code for r in results] == [201]*16 + assert all(r.json() == results[0].json() for r in results) + assert client.get(path,headers=headers).json()['items'] == [results[0].json()] + + +def test_scope_and_replay_rechecks_authorization(client): + b, bb = user(client) + bh, _ = login(client, bb) + a, ah, path = setup_chat(client, [b['id']]) + key = 'same-in-independent-scopes' + first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json() + second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json() + other_path = '/api/v1/chats/'+chat['id']+'/messages' + third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}) + assert third.status_code == 201 and third.json()['id'] != first['id'] + outsider, ob = user(client) + oh, _ = login(client, ob) + assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404 + assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204 + assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401 + + +def test_expired_key_can_be_reused(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json() + with client.app.state.db.pool.connection() as conn: + ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds'] + assert 86390 <= ttl <= 86400 + conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key)) + second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + assert len(client.get(path,headers=h).json()['items']) == 2 + + +def test_rollback_side_effect_and_key_on_database_error(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + text = 'fault_' + uuid.uuid4().hex + function = 'fault_' + uuid.uuid4().hex + trigger = 'trigger_' + uuid.uuid4().hex + db = client.app.state.db + # AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook. + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text))) + conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert failed.status_code == 500 + assert failed.headers['X-Instance-Id'] + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0 + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert retried.status_code == 201 + assert client.get(path,headers=h).json()['items'] == [retried.json()] + + +def test_ready_requires_compatible_schema(client): + temporary = 'temporarily_unavailable_' + uuid.uuid4().hex + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary))) + try: + r = client.get('/health/ready') + assert r.status_code == 503 + assert r.json()['error']['code'] == 'unavailable' + assert client.get('/health/live').status_code == 200 + finally: + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary))) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..f5dcd95 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,63 @@ +import io +import pytest +from messenger.jobs import work, replay +from test_attachments import actors,upload,image_bytes,finish +from test_api import client + + +def job_for(client,aid): + with client.app.state.db.pool.connection() as conn: + return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id'] + + +def test_duplicate_ready_is_noop_and_not_replayable(client): + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + path='/api/v1/attachments/'+att['id'] + before=client.get(path,headers=ah).json() + job=job_for(client,att['id']) + for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1) + assert client.get(path,headers=ah).json()==before + with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready') + with client.app.state.db.pool.connection() as conn: + row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone() + assert row['attempt']==1 + assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None + + +def test_old_delivery_does_not_consume_attempt(client): + # Direct invocation avoids consumer timing; it exercises the same real DB/S3 code. + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + job=job_for(client,att['id']) + assert work(client.app.state.db,client.app.state.storage,job,999) + with client.app.state.db.pool.connection() as conn: + row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone() + assert row['attempt'] in (0,1) # independent live consumer may already finish this job + finish(client,att['id']) + + +def test_replay_sql_failure_releases_session_lock(client): + import psycopg + from messenger.db import setting + from psycopg.rows import dict_row + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + job=job_for(client,att['id']) + db=client.app.state.db + with db.pool.connection() as conn: + conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id'])) + conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$") + conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()') + try: + with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock') + with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other: + acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok'] + if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),)) + assert acquired, 'Aborted replay leaked a session advisory lock into the pool' + finally: + with db.pool.connection() as conn: + conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays') + conn.execute('DROP FUNCTION lab5_fail_audit()') diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 0000000..eb492df --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,72 @@ +"""Real image transformations, no mock storage.""" +import hashlib +import io +import json +import subprocess +import sys +from PIL import Image + + +def test_worker_variants_and_original(tmp_path): + source=tmp_path/'input' + original=Image.new('RGBA',(320,200),(30,90,150,70)) + original.save(source,format='PNG') + before=hashlib.sha256(source.read_bytes()).hexdigest() + result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True) + assert result.returncode==0, result.stderr + manifest=json.loads(result.stdout) + assert hashlib.sha256(source.read_bytes()).hexdigest()==before + for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]: + item=manifest[kind] + with Image.open(tmp_path/kind) as image: + assert image.format==fmt and image.size==size + assert not image.getexif() + raw=(tmp_path/kind).read_bytes() + assert item['sha256']==hashlib.sha256(raw).hexdigest() + assert item['size_bytes']==len(raw) + + +def test_no_upscale_thumbnail_and_exif(tmp_path): + source=tmp_path/'input' + image=Image.new('RGB',(80,40),'red') + exif=Image.Exif();exif[274]=6 + image.save(source,format='JPEG',exif=exif) + result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True) + assert result.returncode==0, result.stderr + with Image.open(tmp_path/'thumbnail') as output: + assert output.size==(40,80) + assert not output.getexif() + + +def test_deadline_covers_slow_s3(monkeypatch): + import contextlib + import http.server + import os + import threading + import time + import uuid + import pytest + from messenger.jobs import process_objects + class Slow(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length','100') + self.end_headers() + try: + for _ in range(100): + self.wfile.write(b'x');self.wfile.flush();time.sleep(.05) + except (BrokenPipeError,ConnectionResetError): pass + def log_message(self,*args): pass + server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow) + thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start() + monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False) + monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port)) + monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2') + row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow', + 'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'} + started=time.monotonic() + try: + with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row) + assert time.monotonic()-started<4 + finally: + server.shutdown();server.server_close() diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py new file mode 100644 index 0000000..20cccc8 --- /dev/null +++ b/student-tests/test_telemetry.py @@ -0,0 +1,51 @@ +from test_api import client +from messenger.telemetry import route_name, safe_fields + + +def test_route_labels_never_include_ids_or_query(): + assert route_name('/api/v1/chats/a87e23bc-430c-4862-9da8-1de1286931aa/messages')=='/api/v1/chats/{chat_id}/messages' + assert route_name('/api/v1/attachments/a87e23bc-430c-4862-9da8-1de1286931aa/content')=='/api/v1/attachments/{attachment_id}/content' + assert route_name('/unknown/private-name')=='unmatched' + + +def test_log_attributes_allowlist_drops_sensitive_values(): + result=safe_fields({'event':'http_completed','request_id':'test','route':'/api/v1/users', + 'Authorization':'Bearer SECRET','password':'SECRET','body':'SECRET','filename':'private'}) + assert result=={'event':'http_completed','request_id':'test','route':'/api/v1/users'} + + +def test_retry_consumer_span_marks_failure(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from opentelemetry.trace import StatusCode + from messenger import telemetry + exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter)) + with provider.get_tracer('test').start_as_current_span('attachments.consume'): + telemetry.job_done('retry',.1,2) + span=exporter.get_finished_spans()[0] + assert span.attributes['job.outcome']=='retry' + assert span.attributes['job.attempt']==2 + assert span.status.status_code==StatusCode.ERROR + provider.shutdown() + + +def test_db_span_covers_query_not_image_work(client,monkeypatch): + import time + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from messenger import telemetry + exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(telemetry,'tracer',lambda:provider.get_tracer('test')) + with client.app.state.db.pool.connection() as conn: + time.sleep(.15) # non-DB work with a leased connection + conn.execute('SELECT pg_sleep(0.02)') + spans=exporter.get_finished_spans() + assert any(s.name=='postgres.query' for s in spans) + assert not any(s.name=='postgres.transaction' for s in spans) + for span in spans: + if span.name=='postgres.query': + assert (span.end_time-span.start_time)/1e9<.15 + assert 'db.statement' not in span.attributes + provider.shutdown() -- 2.54.0