ЛР 6 — БПИ2303 — Лукьянов — на 4 #17

Open
xabenskie wants to merge 1 commits from xabenskie/mtusi2026lab6:solution/bpi2303-lukyanov into main
83 changed files with 8300 additions and 47 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
.superpowers
.pytest_cache
+4 -29
View File
@@ -1,30 +1,5 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# bootstrap generates .env and local files under ignored secrets/.
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Compose enables OTEL_ENABLED=1 and sends all three signals to collector:4318.
# Grafana operator password is generated in secrets/grafana_password.
# Fixed profile: APIx2; worker concurrency1/prefetch1; processing45s/lease90s; max5 attempts.
+25
View File
@@ -0,0 +1,25 @@
name: Application acceptance
on: [push, pull_request]
permissions:
contents: read
jobs:
observability:
runs-on: ubuntu-latest
timeout-minutes: 35
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- run: make up
- run: make check validate-container test test-student test-fresh
- run: make test-jobs
- run: make test-cluster
- run: make test-observability
- uses: actions/upload-artifact@v4
if: always()
with:
name: lab6-evidence
path: evidence/
- if: always()
run: docker compose down
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.13.5-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt requirements-dev.txt requirements.lock ./
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
COPY src/ src/
COPY migrations/ migrations/
COPY student-tests/ student-tests/
COPY scripts/ scripts/
COPY tests/ tests/
COPY contracts/ contracts/
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
ARG APP_RELEASE=lab6-v1
ENV APP_RELEASE=${APP_RELEASE}
LABEL org.opencontainers.image.version=${APP_RELEASE}
USER messenger
EXPOSE 8080
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
+39
View File
@@ -28,3 +28,42 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down test-student test-cluster rolling
bootstrap:
./scripts/bootstrap.sh
up: bootstrap
docker compose build api minio minio-init
docker compose up -d --wait --wait-timeout 120
$(PYTHON) scripts/cluster.py wait
down:
docker compose down
test-student:
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
test-cluster:
$(PYTHON) scripts/verify_cluster.py
rolling:
$(PYTHON) scripts/cluster.py rolling
.PHONY: test-fresh load-evidence verify-persistence verify-failover
test-fresh:
./scripts/check_fresh_migrations.sh
load-evidence:
$(PYTHON) scripts/capture_load.py
verify-persistence:
$(PYTHON) scripts/verify_cluster.py persistence
verify-failover:
$(PYTHON) scripts/verify_cluster.py public-failover
$(PYTHON) scripts/verify_cluster.py failover
.PHONY: validate-container
validate-container:
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
.PHONY: test-jobs
test-jobs:
$(PYTHON) scripts/verify_jobs.py
.PHONY: test-observability
test-observability:
$(PYTHON) scripts/verify_observability.py
+95 -18
View File
@@ -1,37 +1,114 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Лукьянов К.Р. БПИ2303
- Целевая оценка: **4**; трек5 не заявлен.
- Commit/PR: заполняется автором после самостоятельного commit/push.
- Источники, библиотеки и помощь ИИ: FastAPI, Pydantic, Uvicorn, Psycopg, redis-py, boto3, Pillow, OpenTelemetry Python SDK1.34.1 (semantic conventions0.55b1), pytest. Документация [Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel metrics SDK](https://opentelemetry-python.readthedocs.io/en/stable/sdk/metrics.html), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker/Compose v2, Python3.9+, Make. Docker VM желательно выделить не менее8GiB RAM; ресурсы сервисов ограничены compose.yaml. Все зависимости поднимаются из репозитория. Остановить предыдущий стенд, сохранив volumes: используются те же localhost-порты.
```sh
git clone <fork-url>
cd <fork-lab6>
make up
curl -fsS http://localhost:8080/health/ready
make check
make validate-container
make test
make test-student
make test-fresh
make test-jobs
make test-cluster
make test-observability
```
Основной endpoint: http://localhost:8080. Grafana: http://localhost:3000; пользователь operator, локальный пароль в secrets/grafana_password. Prometheus9090, Loki3100, Tempo3200 доступны только через127.0.0.1 для диагностики. Значения secrets не печатать. Dashboard и три datasource импортируются автоматически. make down сохраняет volumes, down --volumes намеренно удаляет данные текущей лабораторной.
Bootstrap создаёт локальные credentials; API/MinIO/mc собираются Dockerfile. MinIO/mc — закреплённые официальные исходники с commit guard, как в lab5; registry-образы/бинарный архив были недоступны. Python и транзитивные зависимости закреплены requirements.lock. Go на хосте не нужен. First build требует registry/GitHub/package repositories.
Отказные проверки запускать последовательно. test-fresh использует отдельный случайный project и удаляет его временные volumes; в этом отдельном тесте схемы SDK выключен, поскольку telemetry backend туда не поднимается. На основном стенде smoke/31 тест и эксперименты выполняются с telemetry on. Синтетические записи тестов остаются в общей БД.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Nginx → API×2 → PG / profile cache / MinIO staging + job/outbox
relay → Streams broker → worker/processor → S3 variants + PG ready
API / relay / worker / processor → OTLP HTTP → Collector
├─ Prometheus: metrics
├─ Loki: logs
└─ Tempo: traces
Grafana → все три источника
```
Бизнес-API lab5 сохранён: 202queued, durable staging/outbox, thumbnail/crop/WebP, ACL, общий Bearer, идемпотентность, retry/failed/replay и cleanup. Nginx остаётся балансировщиком. Публичные contracts/tests/README/COURSE текущей лабы не изменялись. Служебные документы — lab6/docs.
Версии: Collector contrib0.128.0, Prometheus3.4.1, Loki3.5.1, Tempo2.7.2, Grafana12.0.1; PG17.5, Redis7.4.4, Nginx1.28.0. Backend volumes сохраняют историю (retention7d), это не HA и не backup. Collector — перенос/обработка, не долговременная БД; Grafana — интерфейс запросов.
**Трассы и контекст.** Server-span API берёт W3C context. Persist сохраняет carrier в PG job, relay создаёт producer-span и inject в payload, worker extract создаёт consumer-span. Retry связан с исходным upload и имеет job.outcome/attempt; retry/failed помечается ERROR. Дочерний processor получает carrier через stdin, экспортирует реальные S3 spans и завершает exporter перед выходом. postgres.query/commit измеряют фактические SQL/COMMIT, не аренду connection с изображениями/S3. Текст SQL, параметры, storage key и credentials не экспортируются.
**Логи.** JSON: timestamp, severity, service_name, event, trace_id/span_id и request_id для HTTP. Event attributes проходят allowlist. Body, Authorization, passwords/tokens, filenames и private keys исключены. X-Trace-Id — дополнительный диагностический заголовок, X-Request-Id сохраняется. API и worker логи найдены в Loki по trace_id.
**Ресурсы.** service.name=messenger-api/messenger-worker/messenger-relay, service.version=lab6-v1, случайный service.instance.id, deployment.environment.name=lab. Processor наследует worker instance, не создаёт отдельную серию metrics на каждую job. Processor экспортирует traces/logs, metrics обработки остаются у долгоживущего worker.
| Сигнал / имя Prometheus | Смысл / единицы |
| --- | --- |
| http_requests_total | количество попыток бизнес-запросов API; method/route template/status_class |
| http_server_duration_seconds_bucket | HTTP duration, seconds; p50/p95/p99 из histogram buckets всех API |
| jobs_processed_total | попытки обработки по job_type/outcome ready/retry/failed |
| job_processing_duration_seconds_bucket | длительность попытки, seconds |
| attachment_ready_seconds_bucket | accepted→ready, seconds |
| queue_depth / queue_oldest_age_seconds | незавершённые queued/processing, count/seconds; собирает relay |
| db_pool_used / db_pool_waiting | занятые соединения / ожидающие запросы |
| worker_busy / worker_capacity | текущая занятость / ёмкость одного worker |
| process_memory_bytes / process_cpu_seconds_total | RSS / накопленное процессорное время |
Технические UUID не являются labels. Неизвестный route — unmatched, UUID заменяются шаблонами. Histogram buckets0.005–120s, ready0.1–300s; units отображаются в dashboard. HTTP latency фиксируется до отправки тела response, а не до завершения полного скачивания клиентом. Queue age — backlog, не processing duration. Пример p95:
```promql
histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m])))
```
**SLI/SLO и alerts.** SLI=1−5xx/все бизнес-попытки API, health исключены; клиентские4xx не считаются серверным отказом. Повтор Nginx может образовать две попытки API — это SLI backend, не точный счётчик пользовательских действий. Цель>=99% за30d. Короткий учебный прогон не доказывает достижение30-дневного SLO. Alerts: error ratio>5% в окне1m for15s; oldest job>15s for15s. Правила и локальный runbook в infra/observability/alerts.yaml, доставка наружу не настраивалась.
**Ограничение telemetry.** SDK batch queues256, export batch32, задержка500ms, timeout1s; metrics interval2s. Collector memory_limiter192MiB/spike48MiB, container256MiB; batch128/max256, очереди exporter256 batches,2consumers, retry horizon20s. Неограниченного буфера нет. Prometheus exporter metric_expiration10s удаляет gauges ушедших replicas, не удерживая старый queue age5min. Backend outage может терять telemetry; запросы приложения не ждут его восстановления.
**Восстановление очереди.** Consumer reclaim неподтверждённой доставки —5s, независимо от PG processing lease90s. Живой процесс защищён advisory lock и общим deadline45s. Это устраняет задержку повторной доставки после краткого DB-сбоя, сохраняя защиту от параллельного эффекта.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Прогоны2026-11-09, Docker Desktop/macOS Apple Silicon. Ветка solution/bpi2303-lukyanov, коммитов/push агент не делает.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Проверка | Фактический результат / evidence |
| --- | --- |
| check/OpenAPI/public smoke | PASS: check.txt, openapi.txt, smoke.txt |
| Собственные регрессии |31 тест PASS: student-tests.txt |
| Пустые volumes/повтор миграций |31 тест PASS: fresh-migrations.txt |
| Сохранность pipeline lab5 | jobs-faults.json: COMMIT/publish gap, broker restart, kill послеPUT, lease, retry/replay, duplicate PASS |
| Restart/idempotency/failover/SIGTERM/rolling | соответствующие JSON в evidence |
| DB/Redis отказ и ресурсы/порты | dependency-failures.json, infrastructure.json |
| Автоматический dashboard и три datasource | grafana-provisioning.json, dashboardJSON в infra/observability |
| Сквозная trace с retry и связанные API/worker logs | upload-trace.json, upload-logs.json, observability.json |
| Queue/DB incidents, firing→resolved | queue-alert-*.json, db-alert-*.json |
| Backend outage и bounded memory/queue | observability.json: сообщения успешны, экспортные counters/queue и dockerstats сохранены |
Предупреждения botocore utcnow и диагностический Pillow warning не являются падениями assertions. Code review выявил неправильный DB lease-span и отсутствие outcome/error у consumer; обе регрессии сначала упали, затем прошли после исправления.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
1. **Upload+retry.** Worker остановлен, файл принят; MinIO недоступен во время первой обработки. После восстановления тот же attachment ready, trace связывает server→producer→consumer→processor→S3/DB, включая несколько попыток. Сохранено 3 попыток; лог API и worker коррелирует с trace_id.
2. **Worker stopped.** До воздействия alert разрешён; затем остановка worker, queued upload, queue age растёт, alert firing. После запуска ready и resolved. Queue metric продолжает работать через relay.
3. **DB contention.** Операторская сессия держит ACCESS EXCLUSIVE lock на sessions, readiness остаётся живым. Запросы users/me ждут lock_timeout и возвращают503; postgres.query spans и db_operation_failed logs указывают на DB, не decoder. 9 ошибочных вызовов, затем holder прекращён через pg_terminate_backend, здоровый трафик и alert resolved. Helper отключает только собственные idle/statement timeouts и гарантированно завершается после опыта.
4. **Loki+Tempo stopped.** За 26.571s успешно записано 2575 сообщений. Collector memory: 75.03MiB / 256MiB→76.7MiB / 256MiB при256MiB limit. Bounded retry/queues и реальные accepted/sent/failed/queue counters сохранены. Это короткий опыт, не доказательство нулевых потерь telemetry или отсутствия роста при бесконечной нагрузке.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Все15 dashboard expressions проверены реальными запросами Prometheus. Данные trace/log не статические fixtures. В ходе общего прогона после rolling один latency-check Redis fallback превысил3s; отдельная воспроизводящая проверка прошла без изменения контракта, затем dependency test повторён успешно. Кроме того, публичный smoke выявил90s pending-redelivery после DB outage: reclaim сокращён до5s, после чего повтор отказа и smoke прошли.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один host, backend на локальных volumes; нет HA/backup и обещания30-дневной доступности.
- End-user response LB может отсутствовать в SDK при исключении всех API; это не считается подтверждённым API-observation. SLI определён для обработанных бизнес-попыток API.
- Экспорт bounded best effort: при исчерпании очереди или retry horizon telemetry теряется. До устойчивого ingest/backend-volume гарантий сохранности span нет.
- HTTP instrumentation измеряет handler/headers; download streaming не является полным клиентским latency.
- HTTP и localhost админ-интерфейсы допустимы только в изолированном учебном окружении. TLS/mTLS и OpenBao вводятся в lab7.
- Исследовательские sampling/overhead/burn-rate треки5 не выполнялись. Наблюдения относятся к заданному профилю и синтетическим данным.
+238
View File
@@ -0,0 +1,238 @@
name: mtusi-lab6
services:
postgres:
image: postgres:17.5-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
redis:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 0.25
migrate:
image: mtusi-lab6-api:lab6
command: [python, -m, messenger.migrate]
environment:
OTEL_ENABLED: '1'
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
restart: 'no'
mem_limit: 256m
cpus: 0.5
api:
image: mtusi-lab6-api:lab6
build: .
scale: 2
environment:
OTEL_ENABLED: '1'
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
POOL_MAX_SIZE: '8'
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [front, data]
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
stop_grace_period: 10s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
interval: 2s
timeout: 5s
retries: 30
mem_limit: 512m
cpus: 1.0
minio:
image: mtusi-lab6-minio:2025-04-22
build:
context: .
dockerfile: infra/minio/Dockerfile
target: minio
secrets: [minio_root_user, minio_root_password]
volumes: [minio_data:/data]
networks: [data]
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
interval: 2s
timeout: 3s
retries: 30
stop_grace_period: 10s
mem_limit: 512m
cpus: 1.0
minio-init:
image: mtusi-lab6-mc:2025-04-16
build:
context: .
dockerfile: infra/minio/Dockerfile
target: mc
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
volumes:
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
networks: [data]
depends_on:
minio: {condition: service_healthy}
restart: 'no'
mem_limit: 128m
cpus: 0.25
broker:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction]
volumes: [broker_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 160m
cpus: 0.5
worker: &background
image: mtusi-lab6-api:lab6
command: [python, -m, messenger.worker, worker]
environment:
OTEL_ENABLED: '1'
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
BROKER_URL: redis://broker:6379/0
JOB_MAX_ATTEMPTS: '5'
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
broker: {condition: service_healthy}
migrate: {condition: service_completed_successfully}
minio-init: {condition: service_completed_successfully}
stop_grace_period: 45s
restart: unless-stopped
mem_limit: 512m
cpus: 1.0
relay:
<<: *background
command: [python, -m, messenger.worker, relay]
mem_limit: 192m
cpus: 0.25
lb:
image: nginx:1.28.0-alpine
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
networks: [front]
ports: ['127.0.0.1:8080:8080']
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 1s
timeout: 8s
retries: 30
mem_limit: 128m
cpus: 0.5
collector:
image: otel/opentelemetry-collector-contrib:0.128.0
command: [--config=/etc/otelcol/config.yaml]
volumes: [./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro]
networks: [data, observe]
mem_limit: 256m
cpus: 0.5
prometheus:
image: prom/prometheus:v3.4.1
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d]
volumes:
- ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
ports: ['127.0.0.1:9090:9090']
networks: [observe, admin]
mem_limit: 512m
cpus: 0.5
loki:
image: grafana/loki:3.5.1
command: [-config.file=/etc/loki/local.yaml]
volumes: [./infra/observability/loki.yaml:/etc/loki/local.yaml:ro, loki_data:/loki]
networks: [observe, admin]
ports: ['127.0.0.1:3100:3100']
mem_limit: 384m
cpus: 0.5
tempo:
image: grafana/tempo:2.7.2
command: [-config.file=/etc/tempo/local.yaml]
user: '0:0'
volumes: [./infra/observability/tempo.yaml:/etc/tempo/local.yaml:ro, tempo_data:/var/tempo]
networks: [observe, admin]
ports: ['127.0.0.1:3200:3200']
mem_limit: 384m
cpus: 0.5
grafana:
image: grafana/grafana:12.0.1
environment:
GF_SECURITY_ADMIN_USER: operator
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
secrets: [grafana_password]
volumes:
- ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
ports: ['127.0.0.1:3000:3000']
networks: [observe, admin]
mem_limit: 256m
cpus: 0.5
networks:
admin:
observe:
internal: true
front:
data:
internal: true
secrets:
grafana_password: {file: ./secrets/grafana_password}
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
minio_root_user: {file: ./secrets/minio_root_user}
minio_root_password: {file: ./secrets/minio_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
volumes:
postgres_data:
redis_data:
minio_data:
broker_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"parallel_requests": 8,
"observed_instances": [
"0432eff5e9d84ea99431573f134f5c23",
"bfc39b720927442aa51a68b9fe837119"
],
"responses_201": 8,
"messages_created": 1,
"different_body_status": 409
}
+15
View File
@@ -0,0 +1,15 @@
[
{
"labels": {
"alertname": "HighApiErrorRate",
"severity": "warning"
},
"annotations": {
"runbook": "Check postgres.transaction error spans, DB pool and request logs. Restore DB and confirm resolved.",
"summary": "Business API 5xx exceeds 5% in 1 minute"
},
"state": "firing",
"activeAt": "2026-11-09T15:56:22.478517494Z",
"value": "7.272727272727273e-01"
}
]
+1
View File
@@ -0,0 +1 @@
[]
+20
View File
@@ -0,0 +1,20 @@
{
"result": "PASS",
"redis": {
"profile_status": 200,
"fallback_seconds": 2.2293,
"anonymous": 401,
"readiness": 200
},
"postgres": {
"authenticated_status": 503,
"response_seconds": 0.0177,
"anonymous_status": 401,
"internal_api_liveness": [
200,
200
],
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
},
"same_session_after_recovery": true
}
+70
View File
@@ -0,0 +1,70 @@
./scripts/check_fresh_migrations.sh
Network mtusi-lab6-migration-check-74280_data Creating
Volume mtusi-lab6-migration-check-74280_redis_data Creating
Volume mtusi-lab6-migration-check-74280_redis_data Creating
Volume mtusi-lab6-migration-check-74280_postgres_data Creating
Volume mtusi-lab6-migration-check-74280_postgres_data Creating
Network mtusi-lab6-migration-check-74280_data Creating
Volume mtusi-lab6-migration-check-74280_minio_data Creating
Volume mtusi-lab6-migration-check-74280_minio_data Creating
Volume mtusi-lab6-migration-check-74280_redis_data Created
Volume mtusi-lab6-migration-check-74280_redis_data Created
Volume mtusi-lab6-migration-check-74280_postgres_data Created
Volume mtusi-lab6-migration-check-74280_postgres_data Created
Volume mtusi-lab6-migration-check-74280_minio_data Created
Volume mtusi-lab6-migration-check-74280_minio_data Created
Network mtusi-lab6-migration-check-74280_data Created
Network mtusi-lab6-migration-check-74280_data Created
Container mtusi-lab6-migration-check-74280-redis-1 Creating
Container mtusi-lab6-migration-check-74280-postgres-1 Creating
Container mtusi-lab6-migration-check-74280-minio-1 Creating
Container mtusi-lab6-migration-check-74280-redis-1 Created
Container mtusi-lab6-migration-check-74280-minio-1 Created
Container mtusi-lab6-migration-check-74280-minio-init-1 Creating
Container mtusi-lab6-migration-check-74280-postgres-1 Created
Container mtusi-lab6-migration-check-74280-minio-init-1 Created
Container mtusi-lab6-migration-check-74280-minio-1 Starting
Container mtusi-lab6-migration-check-74280-redis-1 Starting
Container mtusi-lab6-migration-check-74280-postgres-1 Starting
Container mtusi-lab6-migration-check-74280-minio-1 Started
Container mtusi-lab6-migration-check-74280-minio-1 Waiting
Container mtusi-lab6-migration-check-74280-redis-1 Started
Container mtusi-lab6-migration-check-74280-postgres-1 Started
Container mtusi-lab6-migration-check-74280-minio-1 Healthy
Container mtusi-lab6-migration-check-74280-minio-init-1 Starting
Container mtusi-lab6-migration-check-74280-minio-init-1 Started
Container mtusi-lab6-migration-check-74280-minio-init-1 Waiting
Container mtusi-lab6-migration-check-74280-redis-1 Waiting
Container mtusi-lab6-migration-check-74280-postgres-1 Waiting
Container mtusi-lab6-migration-check-74280-minio-1 Waiting
Container mtusi-lab6-migration-check-74280-redis-1 Healthy
Container mtusi-lab6-migration-check-74280-minio-init-1 Healthy
Container mtusi-lab6-migration-check-74280-minio-1 Healthy
Container mtusi-lab6-migration-check-74280-postgres-1 Healthy
Network mtusi-lab6-migration-check-74280_front Creating
Network mtusi-lab6-migration-check-74280_front Creating
Network mtusi-lab6-migration-check-74280_front Created
Network mtusi-lab6-migration-check-74280_front Created
Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Creating
Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Created
Applied 001_initial.sql
Applied 002_idempotency.sql
Applied 003_attachments.sql
Applied 004_jobs.sql
Migrations up to date
Migrations up to date
............................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 19 warnings
student-tests/test_idempotency.py: 1 warning
student-tests/test_jobs.py: 6 warnings
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
31 passed, 27 warnings in 13.57s
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
+16
View File
@@ -0,0 +1,16 @@
{
"result": "PASS",
"target_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"target_instance": "a535000080734d769e44ed18a64615ae",
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
"in_flight_observed_in_pg_stat_activity": true,
"in_flight_response": 201,
"new_tcp_after_sigterm": {
"accepted": false,
"error_type": "ConnectionRefusedError"
},
"target_still_running_when_new_tcp_rejected": true,
"exit_code": 0,
"shutdown_seconds_from_sigterm": 2.0642,
"confirmed_message_preserved": true
}
+22
View File
@@ -0,0 +1,22 @@
{
"result": "PASS",
"dashboard_uid": "messenger-lab6",
"panels": 15,
"datasources": [
{
"name": "Loki",
"type": "loki",
"uid": "loki"
},
{
"name": "Prometheus",
"type": "prometheus",
"uid": "prometheus"
},
{
"name": "Tempo",
"type": "tempo",
"uid": "tempo"
}
]
}
+341
View File
@@ -0,0 +1,341 @@
{
"result": "PASS",
"containers": [
{
"service": "api",
"container": "07e419b5121b83e36d525e3d218a4dfd4d5768b83531fe69ea2e459920931114",
"name": "mtusi-lab6-api-3",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:53:46.48092646Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "23ac94c138224f05b2b9ebc8962a8f79",
"release": "lab6-v1"
},
{
"service": "api",
"container": "add61ef8a9c8a4f7e286f2b7b5ef8bc235905c843d2081ec14fc2ee2c1aee9ce",
"name": "mtusi-lab6-api-4",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:53:46.408269418Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "28bb7891222a49a4809c409c0cd15976",
"release": "lab6-v1"
},
{
"service": "broker",
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
"name": "mtusi-lab6-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.284677889Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_broker_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "collector",
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
"name": "mtusi-lab6-collector-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:20:09.734253509Z",
"image": "otel/opentelemetry-collector-contrib:0.128.0",
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_observe"
]
},
{
"service": "grafana",
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
"name": "mtusi-lab6-grafana-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.50566143Z",
"image": "grafana/grafana:12.0.1",
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3000"
}
]
},
"volumes": [
"mtusi-lab6_grafana_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "lb",
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
"name": "mtusi-lab6-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.236679055Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab6_front"
]
},
{
"service": "loki",
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
"name": "mtusi-lab6-loki-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:41:58.345305841Z",
"image": "grafana/loki:3.5.1",
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3100"
}
]
},
"volumes": [
"mtusi-lab6_loki_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "migrate",
"container": "3c47c49b694fda4217bb34ca4f9ea5c83c534deeb2ad8cfbd92a73e191a83959",
"name": "mtusi-lab6-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:53:45.228435626Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio",
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
"name": "mtusi-lab6-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.159907764Z",
"image": "mtusi-lab6-minio:2025-04-22",
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_minio_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio-init",
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
"name": "mtusi-lab6-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:53:45.325678751Z",
"image": "mtusi-lab6-mc:2025-04-16",
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "postgres",
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
"name": "mtusi-lab6-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:53:53.727434505Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_postgres_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "prometheus",
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
"name": "mtusi-lab6-prometheus-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.39409243Z",
"image": "prom/prometheus:v3.4.1",
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
"memory_bytes": 536870912,
"nano_cpus": 500000000,
"published_ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "9090"
}
]
},
"volumes": [
"mtusi-lab6_prometheus_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "redis",
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
"name": "mtusi-lab6-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:53:52.525647296Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_redis_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "relay",
"container": "ed1f47e618ca83beab6abf06e3c288c51d101bfcc53e140b2295d9b6c1a3138e",
"name": "mtusi-lab6-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:53:46.636963169Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "tempo",
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
"name": "mtusi-lab6-tempo-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:53:44.730313751Z",
"image": "grafana/tempo:2.7.2",
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3200"
}
]
},
"volumes": [
"mtusi-lab6_tempo_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "worker",
"container": "1eed3d38f1d69ce58034efdce9c7a1867baf0f88f52146b3dad12addd25adec1",
"name": "mtusi-lab6-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:53:46.559703293Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
}
]
}
+27
View File
@@ -0,0 +1,27 @@
{
"result": "PASS",
"commit_publish_gap": {
"accepted_status": 202,
"outbox_unpublished": true,
"api_broker_restarted": true,
"same_job_ready": true
},
"broker_persistence": {
"unread_job_survived_restart": true
},
"after_put_before_ack": {
"s3_effect_written": true,
"lease_recovery_seconds": 89.096,
"attempt_after_recovery": 2,
"same_job": true
},
"duplicate": {
"ready_noop": true
},
"retry_replay": {
"failed_after": 5,
"code": "retry_exhausted",
"explicit_replay_ready": true,
"failed_attachment_binding": 409
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"name": "mtusi-lab6-api-2",
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"stopped_at_utc": "2026-11-09T15:45:58.712702+00:00",
"stop_seconds": 0.5425,
"exit_code": 0,
"left_stopped": true
}
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"version": "nginx version: nginx/1.28.0",
"image": "nginx:1.28.0-alpine",
"public_server_header": "nginx",
"config_syntax": "PASS",
"observed_instances": [
"0432eff5e9d84ea99431573f134f5c23",
"bfc39b720927442aa51a68b9fe837119"
]
}
+342
View File
@@ -0,0 +1,342 @@
{
"result": "PASS",
"trace": {
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe",
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.query",
"s3.put",
"postgres.query",
"postgres.query",
"postgres.query",
"postgres.commit",
"attachments.publish",
"attachments.consume",
"postgres.query",
"postgres.query",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.query",
"postgres.commit",
"attachments.process_objects",
"s3.get",
"attachments.publish",
"attachments.consume",
"postgres.query",
"postgres.query",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.commit",
"postgres.query",
"postgres.query",
"postgres.commit",
"attachments.process_objects",
"s3.get"
],
"retry_attempts": 3,
"api_worker_logs_correlated": true
},
"queue_incident": {
"oldest_age": [
{
"metric": {},
"value": [
1794239772.021,
"32.563177"
]
}
],
"depth": [
{
"metric": {},
"value": [
1794239772.023,
"1"
]
}
],
"alert": "firing",
"after_restore": "resolved"
},
"db_incident": {
"failures": [
{
"status": 503,
"elapsed": 4.012639791006222,
"trace_id": "2248eed7575b37ebc02bc118377be094"
},
{
"status": 503,
"elapsed": 4.009124165982939,
"trace_id": "e4a499d2482593d978976d4981e1be6c"
},
{
"status": 503,
"elapsed": 4.010977207974065,
"trace_id": "bfa70bd527ce23c595426c74bfee607e"
},
{
"status": 503,
"elapsed": 4.038811917009298,
"trace_id": "95bca29d2812c4cc930ac9fed93a1f7c"
},
{
"status": 503,
"elapsed": 4.028251375013497,
"trace_id": "b9e433fdd149f2922f0b3d78386e6d5a"
},
{
"status": 503,
"elapsed": 4.018512499984354,
"trace_id": "a11d9c704425e92132f9f5647d38bc1f"
},
{
"status": 503,
"elapsed": 4.017416250018869,
"trace_id": "0eb74ac09ce2aa9a36d676c2eafde20e"
},
{
"status": 503,
"elapsed": 4.0125838330131955,
"trace_id": "17f11595e2dc8fe7bbfb33e1bff4c637"
},
{
"status": 503,
"elapsed": 4.007467999996152,
"trace_id": "ddfbc5519d3d9a746207a7d2c427364e"
}
],
"before": "firing",
"after": "resolved",
"pool_waiting": [
{
"metric": {},
"value": [
1794239845.959,
"0"
]
}
]
},
"telemetry_outage": {
"successful_messages": 2575,
"seconds": 26.571,
"collector_before": {
"BlockIO": "0B / 0B",
"CPUPerc": "1.58%",
"Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
"ID": "62ead4df7cf3",
"MemPerc": "29.31%",
"MemUsage": "75.03MiB / 256MiB",
"Name": "mtusi-lab6-collector-1",
"NetIO": "2.92MB / 1.2MB",
"PIDs": "16"
},
"collector_after": {
"BlockIO": "0B / 0B",
"CPUPerc": "0.75%",
"Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
"ID": "62ead4df7cf3",
"MemPerc": "29.96%",
"MemUsage": "76.7MiB / 256MiB",
"Name": "mtusi-lab6-collector-1",
"NetIO": "9MB / 1.56MB",
"PIDs": "16"
},
"collector_metrics": [
{
"metric": {
"__name__": "otelcol_exporter_queue_capacity",
"data_type": "logs",
"exporter": "otlphttp/loki",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"256"
]
},
{
"metric": {
"__name__": "otelcol_exporter_queue_capacity",
"data_type": "traces",
"exporter": "otlp/tempo",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"256"
]
},
{
"metric": {
"__name__": "otelcol_exporter_queue_size",
"data_type": "logs",
"exporter": "otlphttp/loki",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"23"
]
},
{
"metric": {
"__name__": "otelcol_exporter_queue_size",
"data_type": "traces",
"exporter": "otlp/tempo",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"202"
]
},
{
"metric": {
"__name__": "otelcol_exporter_send_failed_spans_total",
"exporter": "otlp/tempo",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"264"
]
},
{
"metric": {
"__name__": "otelcol_exporter_sent_spans_total",
"exporter": "otlp/tempo",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
},
"value": [
1794239872.968,
"5848"
]
},
{
"metric": {
"__name__": "otelcol_receiver_accepted_spans_total",
"instance": "collector:8888",
"job": "collector",
"otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper",
"receiver": "otlp",
"transport": "http"
},
"value": [
1794239872.968,
"32506"
]
}
],
"queue_limit_batches": 256,
"collector_limit_mib": 256
},
"dashboard_queries": [
{
"panel": "Business RPS",
"expression": "sum(rate(http_requests_total[1m]))",
"series": 1
},
{
"panel": "5xx ratio",
"expression": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)",
"series": 1
},
{
"panel": "HTTP p50",
"expression": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"series": 1
},
{
"panel": "HTTP p95",
"expression": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"series": 1
},
{
"panel": "HTTP p99",
"expression": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"series": 1
},
{
"panel": "Jobs by outcome",
"expression": "sum by(outcome)(rate(jobs_processed_total[1m]))",
"series": 1
},
{
"panel": "Job p95",
"expression": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))",
"series": 1
},
{
"panel": "Accepted to ready p95",
"expression": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))",
"series": 1
},
{
"panel": "Queue depth",
"expression": "max(queue_depth)",
"series": 1
},
{
"panel": "Oldest unfinished job",
"expression": "max(queue_oldest_age_seconds)",
"series": 1
},
{
"panel": "DB pool used / waiting",
"expression": "sum by(service_name)(db_pool_used)",
"series": 3
},
{
"panel": "DB pool waiting",
"expression": "sum by(service_name)(db_pool_waiting)",
"series": 3
},
{
"panel": "Worker busy / capacity",
"expression": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
"series": 1
},
{
"panel": "Process RSS",
"expression": "process_memory_bytes",
"series": 4
},
{
"panel": "Process CPU cores",
"expression": "rate(process_cpu_seconds_total[1m])",
"series": 4
}
]
}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/cluster.py exec python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+680
View File
@@ -0,0 +1,680 @@
{
"result": "PASS",
"before": [
{
"service": "api",
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"name": "mtusi-lab6-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:43:08.631200429Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "0432eff5e9d84ea99431573f134f5c23",
"release": "lab6-v1"
},
{
"service": "api",
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"name": "mtusi-lab6-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:43:08.658375471Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "bfc39b720927442aa51a68b9fe837119",
"release": "lab6-v1"
},
{
"service": "broker",
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
"name": "mtusi-lab6-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:43:17.121717419Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_broker_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "collector",
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
"name": "mtusi-lab6-collector-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:20:09.734253509Z",
"image": "otel/opentelemetry-collector-contrib:0.128.0",
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_observe"
]
},
{
"service": "grafana",
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
"name": "mtusi-lab6-grafana-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.50566143Z",
"image": "grafana/grafana:12.0.1",
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3000"
}
]
},
"volumes": [
"mtusi-lab6_grafana_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "lb",
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
"name": "mtusi-lab6-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:21:04.334408756Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab6_front"
]
},
{
"service": "loki",
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
"name": "mtusi-lab6-loki-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:41:58.345305841Z",
"image": "grafana/loki:3.5.1",
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3100"
}
]
},
"volumes": [
"mtusi-lab6_loki_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "migrate",
"container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9",
"name": "mtusi-lab6-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:26:11.136784593Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio",
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
"name": "mtusi-lab6-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:33:32.491039297Z",
"image": "mtusi-lab6-minio:2025-04-22",
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_minio_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio-init",
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
"name": "mtusi-lab6-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:26:11.058089009Z",
"image": "mtusi-lab6-mc:2025-04-16",
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "postgres",
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
"name": "mtusi-lab6-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:20:09.5754303Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_postgres_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "prometheus",
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
"name": "mtusi-lab6-prometheus-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.39409243Z",
"image": "prom/prometheus:v3.4.1",
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
"memory_bytes": 536870912,
"nano_cpus": 500000000,
"published_ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "9090"
}
]
},
"volumes": [
"mtusi-lab6_prometheus_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "redis",
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
"name": "mtusi-lab6-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:20:09.209044467Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_redis_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "relay",
"container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49",
"name": "mtusi-lab6-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:45:03.889331218Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "tempo",
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
"name": "mtusi-lab6-tempo-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:41:58.448806924Z",
"image": "grafana/tempo:2.7.2",
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3200"
}
]
},
"volumes": [
"mtusi-lab6_tempo_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "worker",
"container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c",
"name": "mtusi-lab6-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:45:03.82336276Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
}
],
"after": [
{
"service": "api",
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"name": "mtusi-lab6-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.501501805Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "fcc51ba5c356403998a22d67f9b9be19",
"release": "lab6-v1"
},
{
"service": "api",
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"name": "mtusi-lab6-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.556862139Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_front"
],
"http_status": 200,
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
"release": "lab6-v1"
},
{
"service": "broker",
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
"name": "mtusi-lab6-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.284677889Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_broker_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "collector",
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
"name": "mtusi-lab6-collector-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:20:09.734253509Z",
"image": "otel/opentelemetry-collector-contrib:0.128.0",
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data",
"mtusi-lab6_observe"
]
},
{
"service": "grafana",
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
"name": "mtusi-lab6-grafana-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.50566143Z",
"image": "grafana/grafana:12.0.1",
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3000"
}
]
},
"volumes": [
"mtusi-lab6_grafana_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "lb",
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
"name": "mtusi-lab6-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.236679055Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab6_front"
]
},
{
"service": "loki",
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
"name": "mtusi-lab6-loki-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:41:58.345305841Z",
"image": "grafana/loki:3.5.1",
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3100"
}
]
},
"volumes": [
"mtusi-lab6_loki_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "migrate",
"container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9",
"name": "mtusi-lab6-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:45:13.574632917Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio",
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
"name": "mtusi-lab6-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.159907764Z",
"image": "mtusi-lab6-minio:2025-04-22",
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_minio_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "minio-init",
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
"name": "mtusi-lab6-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:45:13.645406292Z",
"image": "mtusi-lab6-mc:2025-04-16",
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "postgres",
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
"name": "mtusi-lab6-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.211149514Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_postgres_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "prometheus",
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
"name": "mtusi-lab6-prometheus-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:23:49.39409243Z",
"image": "prom/prometheus:v3.4.1",
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
"memory_bytes": 536870912,
"nano_cpus": 500000000,
"published_ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "9090"
}
]
},
"volumes": [
"mtusi-lab6_prometheus_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "redis",
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
"name": "mtusi-lab6-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.321981847Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab6_redis_data"
],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "relay",
"container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49",
"name": "mtusi-lab6-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:45:11.757355583Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
},
{
"service": "tempo",
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
"name": "mtusi-lab6-tempo-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:41:58.448806924Z",
"image": "grafana/tempo:2.7.2",
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
"memory_bytes": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "3200"
}
]
},
"volumes": [
"mtusi-lab6_tempo_data"
],
"networks": [
"mtusi-lab6_admin",
"mtusi-lab6_observe"
]
},
{
"service": "worker",
"container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c",
"name": "mtusi-lab6-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:45:12.362175042Z",
"image": "mtusi-lab6-api:lab6",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab6_data"
]
}
],
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
}
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
{
"result": "PASS",
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.107}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"stopped": {
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"name": "mtusi-lab6-api-1",
"instance": "fcc51ba5c356403998a22d67f9b9be19",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"stopped_at_utc": "2026-11-09T15:45:19.129171+00:00",
"stop_seconds": 0.3545,
"exit_code": 0,
"left_stopped": true
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 1.107}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
[
{
"labels": {
"alertname": "QueueLagGrowing",
"severity": "warning"
},
"annotations": {
"runbook": "Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains.",
"summary": "Oldest unfinished attachment is older than 15 seconds"
},
"state": "firing",
"activeAt": "2026-11-09T15:55:54.478517494Z",
"value": "3.2563177e+01"
}
]
+1
View File
@@ -0,0 +1 @@
[]
+171
View File
@@ -0,0 +1,171 @@
{
"result": "PASS",
"before": [
{
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"name": "mtusi-lab6-api-1",
"instance": "f6c55a60b9d54fc49a966a29db53c7c9",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80"
},
{
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"name": "mtusi-lab6-api-2",
"instance": "201f4ef054b141e89207a80a27997663",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80"
}
],
"after": [
{
"container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5",
"name": "mtusi-lab6-api-3",
"instance": "89984cccbfad40c8ae3896b5789dd706",
"release": "lab6-68cfb17c",
"image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f"
},
{
"container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8",
"name": "mtusi-lab6-api-4",
"instance": "42711a4025dd43b5aa9c806b73e710ac",
"release": "lab6-68cfb17c",
"image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f"
}
],
"target_release": "lab6-68cfb17c",
"lb_before": {
"service": "lb",
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
"name": "mtusi-lab6-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.236679055Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab6_front"
]
},
"lb_after": {
"service": "lb",
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
"name": "mtusi-lab6-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:45:11.236679055Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab6_front"
]
},
"lb_not_restarted": true,
"build_seconds": 23.6953,
"migration_output": "Migrations up to date",
"replacements": [
{
"old_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"new_container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5",
"old_ip": "172.25.0.2",
"new_ip": "172.25.0.5",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"201f4ef054b141e89207a80a27997663",
"89984cccbfad40c8ae3896b5789dd706",
"f6c55a60b9d54fc49a966a29db53c7c9"
],
"observed_instances": [
"201f4ef054b141e89207a80a27997663",
"89984cccbfad40c8ae3896b5789dd706"
],
"old_exit_code": 0,
"seconds": 5.9622
},
{
"old_container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"new_container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8",
"old_ip": "172.25.0.3",
"new_ip": "172.25.0.2",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"201f4ef054b141e89207a80a27997663",
"42711a4025dd43b5aa9c806b73e710ac",
"89984cccbfad40c8ae3896b5789dd706"
],
"observed_instances": [
"42711a4025dd43b5aa9c806b73e710ac",
"89984cccbfad40c8ae3896b5789dd706"
],
"old_exit_code": 0,
"seconds": 6.1869
}
],
"during_replacement": {
"seconds": 12.1492,
"concurrency": 4,
"successful_requests": 6406,
"failed_attempts": 0,
"successful_rps": 527.28,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 5.911,
"p95": 11.297,
"p99": 56.654
},
"instances": [
"201f4ef054b141e89207a80a27997663",
"42711a4025dd43b5aa9c806b73e710ac",
"89984cccbfad40c8ae3896b5789dd706",
"f6c55a60b9d54fc49a966a29db53c7c9"
]
},
"total": {
"seconds": 42.993,
"concurrency": 4,
"successful_requests": 24247,
"failed_attempts": 0,
"successful_rps": 563.975,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 5.93,
"p95": 9.719,
"p99": 52.049
},
"instances": [
"201f4ef054b141e89207a80a27997663",
"42711a4025dd43b5aa9c806b73e710ac",
"89984cccbfad40c8ae3896b5789dd706",
"f6c55a60b9d54fc49a966a29db53c7c9"
]
},
"data": {
"confirmed_unique_messages": 24247,
"history_messages": 24247,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
}
+135
View File
@@ -0,0 +1,135 @@
{
"result": "PASS",
"baseline": {
"seconds": 3.0007,
"concurrency": 4,
"successful_requests": 1736,
"failed_attempts": 0,
"successful_rps": 578.527,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 5.783,
"p95": 9.151,
"p99": 37.03
},
"instances": [
"2af58291b4f34b53bcd4a8c81a6c4abb",
"aee7f9540f7549b88fdbb8ee4290e6af"
]
},
"phases": [
{
"stopped": {
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
"name": "mtusi-lab6-api-1",
"instance": "2af58291b4f34b53bcd4a8c81a6c4abb",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"stopped_at_utc": "2026-11-09T15:45:44.649698+00:00",
"stop_seconds": 0.4897,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.005,
"concurrency": 4,
"successful_requests": 2698,
"failed_attempts": 0,
"successful_rps": 337.039,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 9.853,
"p95": 23.34,
"p99": 65.796
},
"instances": [
"2af58291b4f34b53bcd4a8c81a6c4abb",
"aee7f9540f7549b88fdbb8ee4290e6af"
]
},
"post_stop_write_seconds_from_action_start": 1.3292,
"post_stop_write_seconds_from_stop_completion": 0.0295,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.3137,
"seconds_from_stop_completion": 0.014
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
},
{
"stopped": {
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
"name": "mtusi-lab6-api-2",
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
"release": "lab6-v1",
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
"stopped_at_utc": "2026-11-09T15:45:58.712702+00:00",
"stop_seconds": 0.5425,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.005,
"concurrency": 4,
"successful_requests": 2729,
"failed_attempts": 0,
"successful_rps": 340.91,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 9.895,
"p95": 22.682,
"p99": 60.64
},
"instances": [
"a535000080734d769e44ed18a64615ae",
"aee7f9540f7549b88fdbb8ee4290e6af"
]
},
"post_stop_write_seconds_from_action_start": 1.1705,
"post_stop_write_seconds_from_stop_completion": 0.0247,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.1615,
"seconds_from_stop_completion": 0.0157
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
}
],
"total": {
"seconds": 31.1691,
"concurrency": 4,
"successful_requests": 12705,
"failed_attempts": 0,
"successful_rps": 407.615,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 7.861,
"p95": 21.097,
"p99": 58.639
},
"instances": [
"201f4ef054b141e89207a80a27997663",
"2af58291b4f34b53bcd4a8c81a6c4abb",
"a535000080734d769e44ed18a64615ae",
"aee7f9540f7549b88fdbb8ee4290e6af"
]
},
"data": {
"confirmed_unique_messages": 12705,
"history_messages": 12705,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"each_original_container_tested": true
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+15
View File
@@ -0,0 +1,15 @@
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
............................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 19 warnings
student-tests/test_idempotency.py: 1 warning
student-tests/test_jobs.py: 6 warnings
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
31 passed, 27 warnings in 13.44s
+244
View File
@@ -0,0 +1,244 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"code_file_path": "",
"code_function_name": "",
"code_line_number": "0",
"deployment_environment_name": "lab",
"detected_level": "INFO",
"flags": "1",
"observed_timestamp": "1794239727634612632",
"scope_name": "messenger.events",
"service_instance_id": "3d072627b1ab496c9308fc049bd79784",
"service_name": "messenger-api",
"service_version": "lab6-v1",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "be90e8be55e18b54",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.34.1",
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
},
"values": [
[
"1794239727634585856",
"{\"event\":\"http_completed\",\"method\":\"POST\",\"request_id\":\"23dc38ec-dc6c-4592-9ba9-6744d664e150\",\"route\":\"/api/v1/chats/{chat_id}/attachments\",\"service_name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"be90e8be55e18b54\",\"status_class\":\"2xx\",\"timestamp\":1794239727.6345801,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
]
]
},
{
"stream": {
"code_file_path": "",
"code_function_name": "",
"code_line_number": "0",
"deployment_environment_name": "lab",
"detected_level": "INFO",
"flags": "1",
"observed_timestamp": "1794239729394242216",
"scope_name": "messenger.events",
"service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198",
"service_name": "messenger-worker",
"service_version": "lab6-v1",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "f024a4b9cef096b9",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.34.1",
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
},
"values": [
[
"1794239729394201088",
"{\"attempt\":1,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"f024a4b9cef096b9\",\"timestamp\":1794239729.3941958,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
]
]
},
{
"stream": {
"code_file_path": "",
"code_function_name": "",
"code_line_number": "0",
"deployment_environment_name": "lab",
"detected_level": "INFO",
"flags": "1",
"observed_timestamp": "1794239731438346217",
"scope_name": "messenger.events",
"service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198",
"service_name": "messenger-worker",
"service_version": "lab6-v1",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "866c15dacbeadbc2",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.34.1",
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
},
"values": [
[
"1794239731438312704",
"{\"attempt\":2,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"866c15dacbeadbc2\",\"timestamp\":1794239731.4383066,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 51852023,
"linesProcessedPerSecond": 82276,
"totalBytesProcessed": 24822333,
"totalLinesProcessed": 39387,
"execTime": 0.478715,
"queueTime": 0.000184,
"subqueries": 0,
"totalEntriesReturned": 3,
"splits": 0,
"shards": 1,
"totalPostFilterLines": 3,
"totalStructuredMetadataBytesProcessed": 11794082
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 1,
"totalChunksMatched": 15,
"totalBatches": 2,
"totalLinesSent": 3,
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 2,
"chunksDownloadTime": 4766583,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 931797,
"headChunkLines": 2188,
"decompressedBytes": 23890536,
"decompressedLines": 37199,
"compressedBytes": 2915879,
"totalDuplicates": 0,
"postFilterLines": 3,
"headChunkStructuredMetadataBytes": 262392,
"decompressedStructuredMetadataBytes": 11531690
},
"chunkRefsFetchTime": 385583,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0,
"usedBloomFilters": false
}
}
}
}
File diff suppressed because it is too large Load Diff
+28
View File
@@ -0,0 +1,28 @@
# Official public images/binaries were removed upstream; build fixed release sources.
FROM golang:1.24.2-bookworm AS minio-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
FROM golang:1.24.2-bookworm AS mc-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
FROM python:3.13.5-slim AS minio
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY infra/minio/server.sh /server.sh
USER 10002
ENTRYPOINT ["/bin/sh", "/server.sh"]
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
FROM debian:bookworm-slim AS mc
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
COPY --from=mc-build /out/mc /usr/local/bin/mc
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
+45
View File
@@ -0,0 +1,45 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"originals/*",
"staging/*",
"variants/*"
]
}
}
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/variants/*"
]
}
]
}
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
exec minio "$@"
+62
View File
@@ -0,0 +1,62 @@
worker_processes 1;
error_log /dev/stderr warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
server_tokens off;
limit_conn_zone $server_name zone=uploads:64k;
# Docker DNS; resolve updates the upstream when replicas get new IPs.
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
upstream messenger {
zone messenger 64k;
# All A records of api participate in the default round-robin balancing.
server api:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
server_name messenger;
location ~ ^/api/v1/chats/[^/]+/attachments$ {
limit_conn uploads 16;
client_max_body_size 11m;
client_body_buffer_size 64k;
client_body_timeout 10s;
proxy_request_buffering on;
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
}
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
# must not be resent by Nginx after an ambiguous response.
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
}
}
}
+17
View File
@@ -0,0 +1,17 @@
groups:
- name: messenger
rules:
- alert: HighApiErrorRate
expr: sum(rate(http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001) > 0.05
for: 15s
labels: {severity: warning}
annotations:
summary: Business API 5xx exceeds 5% in 1 minute
runbook: Check postgres.query error spans, DB pool and request logs. Restore DB and confirm resolved.
- alert: QueueLagGrowing
expr: max(queue_oldest_age_seconds) > 15
for: 15s
labels: {severity: warning}
annotations:
summary: Oldest unfinished attachment is older than 15 seconds
runbook: Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains.
+80
View File
@@ -0,0 +1,80 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:9464
resource_to_telemetry_conversion:
enabled: true
metric_expiration: 10s
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
timeout: 2s
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 20s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 20s
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- prometheus
logs:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlphttp/loki
traces:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp/tempo
@@ -0,0 +1,436 @@
{
"uid": "messenger-lab6",
"title": "Messenger \u2014 API and pipeline",
"schemaVersion": 41,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"tags": [
"lab6"
],
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Business RPS",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(http_requests_total[1m]))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 2,
"type": "timeseries",
"title": "5xx ratio",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 3,
"type": "timeseries",
"title": "HTTP p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 4,
"type": "timeseries",
"title": "HTTP p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 5,
"type": "timeseries",
"title": "HTTP p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 6,
"type": "timeseries",
"title": "Jobs by outcome",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(outcome)(rate(jobs_processed_total[1m]))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 7,
"type": "timeseries",
"title": "Job p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"type": "timeseries",
"title": "Accepted to ready p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 9,
"type": "timeseries",
"title": "Queue depth",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(queue_depth)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 10,
"type": "timeseries",
"title": "Oldest unfinished job",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(queue_oldest_age_seconds)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 11,
"type": "timeseries",
"title": "DB pool used / waiting",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name)(db_pool_used)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 12,
"type": "timeseries",
"title": "DB pool waiting",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name)(db_pool_waiting)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 13,
"type": "timeseries",
"title": "Worker busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 14,
"type": "timeseries",
"title": "Process RSS",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "process_memory_bytes",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 15,
"type": "timeseries",
"title": "Process CPU cores",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "rate(process_cpu_seconds_total[1m])",
"refId": "A",
"legendFormat": "{{service_name}} {{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
}
]
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,28 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
access: proxy
- name: Loki
uid: loki
type: loki
url: http://loki:3100
access: proxy
jsonData:
derivedFields:
- name: TraceID
matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
url: http://tempo:3200
access: proxy
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
+28
View File
@@ -0,0 +1,28 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore: {store: inmemory}
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index: {prefix: index_, period: 24h}
limits_config:
allow_structured_metadata: true
retention_period: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+9
View File
@@ -0,0 +1,9 @@
global:
scrape_interval: 2s
evaluation_interval: 2s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: messenger
static_configs: [{targets: ['collector:9464']}]
- job_name: collector
static_configs: [{targets: ['collector:8888']}]
+21
View File
@@ -0,0 +1,21 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
ingester:
trace_idle_period: 2s
max_block_duration: 10s
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal: {path: /var/tempo/wal}
local: {path: /var/tempo/blocks}
usage_report:
reporting_enabled: false
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_history_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
CREATE TABLE sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+11
View File
@@ -0,0 +1,11 @@
-- Additive: old lab2 tables and messages remain valid.
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
+23
View File
@@ -0,0 +1,23 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
attachment_id uuid NOT NULL REFERENCES attachments(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position)
);
+19
View File
@@ -0,0 +1,19 @@
ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN error_code text;
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1),
attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0,
lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text
);
CREATE TABLE job_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz
);
CREATE TABLE job_replays (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL
);
CREATE INDEX jobs_due ON attachment_jobs(next_at);
+61
View File
@@ -0,0 +1,61 @@
annotated-types==0.8.0
anyio==4.9.0
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
attrs==26.1.0
boto3==1.38.27
botocore==1.38.46
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.115.12
googleapis-common-protos==1.75.0
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.20
importlib_metadata==8.7.1
iniconfig==2.3.1
jmespath==1.1.0
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.13.0
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
opentelemetry-api==1.34.1
opentelemetry-exporter-otlp-proto-common==1.34.1
opentelemetry-exporter-otlp-proto-http==1.34.1
opentelemetry-proto==1.34.1
opentelemetry-sdk==1.34.1
opentelemetry-semantic-conventions==0.55b1
packaging==26.3
pathable==0.4.4
pillow==11.3.0
pluggy==1.7.0
protobuf==5.29.6
psycopg==3.2.9
psycopg-binary==3.2.9
psycopg-pool==3.2.6
pycparser==3.12
pydantic==2.11.5
pydantic_core==2.33.2
Pygments==2.21.0
pytest==8.4.0
python-dateutil==2.9.0.post0
PyYAML==6.0.3
redis==6.2.0
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
s3transfer==0.13.1
six==1.17.0
sniffio==1.3.1
starlette==0.46.2
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.3
zipp==4.1.2
+15
View File
@@ -0,0 +1,15 @@
fastapi==0.115.12
uvicorn==0.34.3
psycopg[binary]==3.2.9
psycopg-pool==3.2.6
redis==6.2.0
argon2-cffi==25.1.0
pytest==8.4.0
httpx==0.28.1
pydantic==2.11.5
anyio==4.9.0
boto3==1.38.27
Pillow==11.3.0
opentelemetry-api==1.34.1
opentelemetry-sdk==1.34.1
opentelemetry-exporter-otlp-proto-http==1.34.1
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
python3 - <<'PY'
from pathlib import Path
import secrets
root = Path('secrets')
p = root / 'postgres_password'
if not p.exists():
p.write_text(secrets.token_hex(24))
pw = p.read_text().strip()
for name, value in {
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
'grafana_password': secrets.token_hex(24),
'cache_url': 'redis://redis:6379/0',
'minio_root_user': 'lab4-admin',
'minio_root_password': secrets.token_hex(24),
's3_access_key_id': 'messenger-api',
's3_secret_access_key': secrets.token_hex(24),
}.items():
path = root / name
if not path.exists():
path.write_text(value)
# Bind-mounted Docker secrets must be readable by container uid 10001.
# Parent directory remains 0700 on the host.
for path in root.iterdir():
path.chmod(0o444)
if not Path('.env').exists():
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready (values hidden).')
PY
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# A unique disposable Compose project; never touches normal project volumes.
set -eu
cd "$(dirname "$0")/.."
project="mtusi-lab6-migration-check-$$"
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
echo 'Refusing to reuse a pre-existing scratch project' >&2
exit 1
fi
cleanup() {
docker compose -p "$project" down --volumes >/dev/null 2>&1
}
trap cleanup EXIT INT TERM
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
docker compose -p "$project" run --rm --no-deps -e OTEL_ENABLED=0 api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/cluster.py restart
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec python3 scripts/cluster.py stop-one
+447
View File
@@ -0,0 +1,447 @@
#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','minio','broker','relay','worker','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab6-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
admin_ports={'grafana':'3000','prometheus':'9090','loki':'3100','tempo':'3200'}
if item['service'] in admin_ports:
for bindings in (item['published_ports'] or {}).values():
check(all(b['HostIp']=='127.0.0.1' and b['HostPort']==admin_ports[item['service']] for b in bindings),'admin port is not localhost-only')
elif item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env python3
"""Explicit real fault injection; emits IDs/results, never session tokens."""
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
def compose(*args):
if args and args[0]=='start': args=('up','-d','--no-deps',*args[1:])
result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120)
if result.returncode: raise RuntimeError(result.stderr)
return result.stdout
def inside(body,**values):
script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()'
proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120)
if proc.returncode: raise RuntimeError(proc.stderr)
return json.loads(proc.stdout)
def state(aid):
return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid)
def wait_for(predicate,seconds=30):
end=time.monotonic()+seconds
while time.monotonic()<end:
result=predicate()
if result: return result
time.sleep(.2)
raise AssertionError('Condition timed out')
def ready(c,actor,att):
return c.wait_ready(actor,att)
def main():
c=Client();alice,bob,_,chat=c.bootstrap()
evidence={'result':'INCOMPLETE'}
trigger=False
try:
# Durable acceptance while both relay and worker are absent: real COMMIT/publish gap.
compose('stop','worker','relay')
att,raw=c.attachment(alice,chat)
c.message(alice,chat,'',attachments=[att['id']])
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
before=state(att['id']);check(before['dispatched_at'] is None,'outbox already published')
compose('restart','api','broker')
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0)
c.call('GET','/api/v1/users/me',actor=alice)
compose('start','relay','worker')
meta=ready(c,bob,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original changed')
evidence['commit_publish_gap']={'accepted_status':202,'outbox_unpublished':True,'api_broker_restarted':True,'same_job_ready':state(att['id'])['job_id']==before['job_id']}
# Publisher has already confirmed acceptance, broker restart preserves unread work.
compose('stop','worker')
att2,_=c.attachment(alice,chat)
wait_for(lambda: state(att2['id'])['dispatched_at'])
compose('restart','broker')
compose('start','worker')
ready(c,bob,att2)
evidence['broker_persistence']={'unread_job_survived_restart':True}
# Delay ready COMMIT in real PostgreSQL AFTER all S3 PUTs, kill before ack.
compose('stop','worker')
target,_=c.attachment(alice,chat)
inside("with db.pool.connection() as c:\n c.execute(\"CREATE OR REPLACE FUNCTION lab5_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='\"+v['aid']+\"'::uuid AND NEW.status='ready' THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$\")\n c.execute('CREATE TRIGGER lab5_pause BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_pause_ready()')\nresult=True",aid=target['id'])
trigger=True
compose('start','worker')
def at_commit():
return inside("with db.pool.connection() as c:\n result=bool(c.execute(\"SELECT 1 FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'UPDATE attachments SET status=%'\").fetchone())")
wait_for(at_commit)
observed=inside("with db.pool.connection() as c:\n row=c.execute('SELECT object_key FROM attachments WHERE id=%s',(v['aid'],)).fetchone()\nobj=store.get(row['object_key']);result={'s3_effect_written':obj.length>0};obj.close()",aid=target['id'])
compose('kill','-s','SIGKILL','worker')
# unless-stopped can restart a killed worker; stop it to retain the fault boundary.
compose('stop','worker')
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True")
trigger=False
crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt')
print('Waiting for real 90s lease expiry after killed worker.',flush=True)
started=time.monotonic()
wait_for(lambda: state(target['id'])['status']=='queued',110)
compose('start','worker')
ready(c,bob,target)
after=state(target['id'])
evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']}
# Re-deliver the completed original job. Ready metadata is immutable.
original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']
duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id'])
check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result')
evidence['duplicate']={'ready_noop':True}
# Retry exhaustion with real missing staging (transient storage failure), explicit replay.
compose('stop','worker','relay')
failed,failed_raw=c.attachment(alice,chat)
st=state(failed['id'])
inside("store.delete(v['key']);result=True",key=st['staging_key'])
for attempt in range(5):
inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id'])
failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json']
check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id'])
compose('start','relay','worker')
ready(c,bob,failed)
evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409}
evidence['result']='PASS'
(ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n')
print('PASS: jobs-faults.json')
finally:
if trigger:
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True")
compose('start','relay','worker')
if __name__=='__main__': main()
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Query real backends and reproduce bounded incidents; no secret values saved."""
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
from verify_jobs import compose,inside,wait_for,state
def get(url):
with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as response:
return json.load(response)
def query(expression):
return get('http://localhost:9090/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result']
def alerts():
return get('http://localhost:9090/api/v1/alerts')['data']['alerts']
def alert_state(name,state):
rows=[a for a in alerts() if a['labels']['alertname']==name]
return any(a['state']==state for a in rows) if state=='firing' else not rows
def collect_names(trace):
names=[]
def walk(value):
if isinstance(value,dict):
if 'spanId' in value and 'name' in value: names.append(value['name'])
for item in value.values(): walk(item)
elif isinstance(value,list):
for item in value: walk(item)
walk(trace)
return names
def save(name,data):
(ROOT/'evidence'/name).write_text(json.dumps(data,indent=2)+'\n')
def main():
c=Client();alice,bob,_,chat=c.bootstrap()
report={'result':'INCOMPLETE'}
lock_process=None
try:
# Controlled retry in the same causal chain.
compose('stop','worker')
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,
actor=alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'},schema='Attachment')
att=response['json'];trace_id=response['headers']['X-Trace-Id']
compose('stop','minio');compose('start','worker')
wait_for(lambda: (lambda s:s['attempt']>=1 and s['status']=='queued')(state(att['id'])),30)
time.sleep(2)
compose('start','minio')
# API re-enters LB after readiness recovers.
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0,60)
ready=c.wait_ready(bob,att)
def chain():
try:
trace=get('http://localhost:3200/api/traces/'+trace_id)
names=collect_names(trace)
return trace if names.count('attachments.consume')>=2 and all(n in names for n in ['attachments.publish','attachments.process_objects','s3.put','postgres.query']) else None
except Exception: return None
trace=wait_for(chain,60)
save('upload-trace.json',trace)
logquery='{service_name=~"messenger-.*"} |= "'+trace_id+'"'
def correlated_logs():
try:
result=get('http://localhost:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':logquery,'limit':100,'start':str(time.time_ns()-600_000_000_000)}))
streams=result['data']['result']
services={item['stream'].get('service_name') for item in streams}
return result if {'messenger-api','messenger-worker'}<=services else None
except Exception: return None
logs=wait_for(correlated_logs,60);save('upload-logs.json',logs)
report['trace']={'trace_id':trace_id,'span_names':collect_names(trace),'retry_attempts':state(att['id'])['attempt'],'api_worker_logs_correlated':True}
# Incident 1: worker stopped, queued age is measured by relay.
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),40)
compose('stop','worker')
queued,_=c.attachment(alice,chat)
wait_for(lambda: alert_state('QueueLagGrowing','firing'),70)
save('queue-alert-firing.json',alerts())
report['queue_incident']={'oldest_age':query('max(queue_oldest_age_seconds)'),'depth':query('max(queue_depth)'),'alert':'firing'}
compose('start','worker');c.wait_ready(bob,queued)
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),30)
save('queue-alert-resolved.json',alerts());report['queue_incident']['after_restore']='resolved'
# Incident 2: sessions table lock affects business auth, readiness remains healthy.
code='import os\nfrom messenger.db import Database\nd=Database();d.open();c=d.pool.getconn()\nc.execute(\'SET statement_timeout=0\')\nc.execute(\'SET idle_in_transaction_session_timeout=0\')\nc.execute("SET application_name=\'lab6-fault-lock\'")\nc.commit()\nc.execute(\'LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE\')\nprint(\'locked\',flush=True)\ntry:\n c.execute(\'SELECT pg_sleep(90)\')\nfinally:\n os._exit(0)\n'
lock_process=subprocess.Popen(['docker','compose','exec','-T','api','python','-c',code],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
check(lock_process.stdout.readline().strip()=='locked','DB lock was not acquired')
failures=[]
started=time.monotonic()
while time.monotonic()-started<35:
r=c.call('GET','/api/v1/users/me',503,actor=alice)
failures.append({'status':r['status'],'elapsed':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id')})
wait_for(lambda: alert_state('HighApiErrorRate','firing'),40)
save('db-alert-firing.json',alerts())
# Terminate exact lock holder via PG primary, not the API container.
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
lock_process.wait(timeout=10);lock_process=None
for _ in range(300): c.call('GET','/api/v1/users/me',actor=alice)
wait_for(lambda: alert_state('HighApiErrorRate','resolved'),75)
save('db-alert-resolved.json',alerts())
report['db_incident']={'failures':failures,'before':'firing','after':'resolved','pool_waiting':query('sum(db_pool_waiting)')}
# Backend outage must not affect accepted messages or grow queues beyond configuration.
before=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
compose('stop','tempo','loki')
count=0;start=time.monotonic()
while time.monotonic()-start<25:
c.message(alice,chat,'telemetry outage '+str(count),key=uuid.uuid4().hex);count+=1
if count%20==0: time.sleep(.1)
during=query('{__name__=~"otelcol_(receiver_accepted_spans|exporter_sent_spans|exporter_send_failed_spans|exporter_queue_size|exporter_queue_capacity|exporter_enqueue_failed_spans).*"}')
after=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
compose('start','tempo','loki')
report['telemetry_outage']={'successful_messages':count,'seconds':round(time.monotonic()-start,3),'collector_before':before,'collector_after':after,'collector_metrics':during,'queue_limit_batches':256,'collector_limit_mib':256}
# Every provisioned dashboard expression must return a successful Prometheus response.
dashboard=json.loads((ROOT/'infra/observability/grafana/dashboards/messenger.json').read_text())
expressions=[]
for panel in dashboard['panels']:
for target in panel['targets']:
values=query(target['expr']);check(values,'No data for '+panel['title'])
expressions.append({'panel':panel['title'],'expression':target['expr'],'series':len(values)})
report['dashboard_queries']=expressions
report['result']='PASS';save('observability.json',report)
print('PASS: observability.json, traces/logs and firing/resolved alerts',flush=True)
finally:
if lock_process is not None:
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
compose('start','worker','minio','tempo','loki')
if __name__=='__main__': main()
View File
+323
View File
@@ -0,0 +1,323 @@
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
import json
import asyncio
import time
from . import telemetry
import logging
import os
import re
from contextlib import asynccontextmanager
from uuid import uuid4
import psycopg
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from starlette.exceptions import HTTPException
from starlette.concurrency import run_in_threadpool
from .db import Database
from .errors import ApiError, invalid, not_found, unauthorized
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
from .messages import store_message, validate_key, MESSAGE_FIELDS
from .object_store import ObjectStore, unavailable
from . import attachments
from urllib.parse import quote
from .pagination import decode_cursor, page, page_limit, public, stamp
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
LOG = logging.getLogger('messenger')
USER_FIELDS = 'id, username, display_name, created_at'
def error_response(request, error):
return JSONResponse({'error': {'code': error.code, 'message': error.message,
'request_id': request.state.request_id}},
status_code=error.status, headers=error.headers)
def authenticate(request: Request):
token = bearer(request.headers.get('Authorization', ''))
token_hash = digest(token)
with request.app.state.db.pool.connection() as conn:
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
(token_hash,)).fetchone()
if not row:
raise unauthorized()
request.state.token_hash = token_hash
return row['user_id']
def require_member(conn, chat_id, user_id):
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
if not found:
raise not_found()
def chat_public(conn, row):
item = public(row)
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
return item
def create_app():
@asynccontextmanager
async def lifespan(app):
app.state.db = Database()
app.state.storage = ObjectStore()
app.state.upload_slots = asyncio.Semaphore(6)
app.state.db.open()
try:
telemetry.init("messenger-api",app.state.db)
yield
finally:
app.state.storage.close()
app.state.db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.state.release = os.environ.get('APP_RELEASE', 'lab6-v1')
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
path = request.url.path
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
if request.method == 'POST' and json_endpoint:
media = request.headers.get('Content-Type', '').split(';')
unsupported = media[0].strip().lower() != 'application/json'
for parameter in media[1:]:
name, _, value = parameter.strip().partition('=')
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
unsupported = True
if unsupported:
if path != '/api/v1/users':
await run_in_threadpool(authenticate, request)
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
try:
json.loads((await request.body()).decode('utf-8'))
except (UnicodeError, json.JSONDecodeError):
raise invalid() from None
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc)
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
except Exception as exc:
# Log exception class only: credentials, SQL and message content stay out of logs.
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = app.state.instance_id
response.headers['X-App-Release'] = app.state.release
return response
@app.middleware('http')
async def traced_request(request: Request,call_next):
route=telemetry.route_name(request.url.path)
started=time.monotonic()
with telemetry.span(request.method+' '+route,kind=telemetry.SpanKind.SERVER,
context=telemetry.propagate.extract(dict(request.headers)),
attributes={'http.request.method':request.method,'http.route':route}) as active:
response=await request_context(request,call_next)
active.set_attribute('http.response.status_code',response.status_code)
if response.status_code>=500: active.set_status(telemetry.StatusCode.ERROR)
telemetry.http_done(request.method,route,response.status_code,time.monotonic()-started,request.state.request_id)
response.headers['X-Trace-Id']=format(active.get_span_context().trace_id,'032x')
return response
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, invalid())
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error_response(request, not_found() if exc.status_code == 404 else invalid())
@app.get('/health/live')
def live():
return {'status': 'ok'}
@app.get('/health/ready')
def ready(request: Request):
try:
with request.app.state.db.pool.connection() as conn:
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
conn.execute('SELECT status,variants FROM attachments LIMIT 0')
conn.execute('SELECT 1 FROM job_outbox LIMIT 0')
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
request.app.state.storage.check()
return {'status': 'ok'}
@app.post('/api/v1/users', status_code=201)
def register(body: CreateUser, request: Request):
password_hash = HASHER.hash(body.password)
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute(
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
return public(row)
@app.post('/api/v1/auth/sessions', status_code=201)
async def create_session(request: Request):
# Run hashing and synchronous database calls outside the event loop.
username, password = basic_credentials(request.headers.get('Authorization', ''))
if await request.body():
raise invalid()
return await run_in_threadpool(login, request, username, password)
def login(request, username, password):
db = request.app.state.db
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
if not row or not valid:
raise unauthorized(basic=True)
row.pop('password_hash')
token, expires_at = new_session(db.session_ttl)
with db.pool.connection() as conn:
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
(digest(token), row['id'], expires_at))
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
@app.get('/api/v1/users/me')
def me(request: Request, user_id=Depends(authenticate)):
db = request.app.state.db
key = 'user:' + str(user_id)
try:
cached = db.cache.get(key)
if cached:
return json.loads(cached)
except (RedisError, ValueError):
pass
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
if not row:
raise unauthorized()
result = public(row)
try:
db.cache.set(key, json.dumps(result), ex=60)
except RedisError:
pass
return result
@app.delete('/api/v1/auth/sessions/current', status_code=204)
def logout(request: Request, user_id=Depends(authenticate)):
with request.app.state.db.pool.connection() as conn:
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
return Response(status_code=204)
@app.post('/api/v1/chats', status_code=201)
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
if str(user_id) not in body.member_ids:
raise invalid()
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
(uuid4(), body.title)).fetchone()
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
[(row['id'], uuid_value(member)) for member in body.member_ids])
result = chat_public(conn, row)
except psycopg.errors.ForeignKeyViolation:
raise invalid() from None
return result
@app.get('/api/v1/chats')
def list_chats(request: Request, user_id=Depends(authenticate)):
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'chats:' + str(user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
with request.app.state.db.pool.connection() as conn:
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
result = page(scope, rows, limit)
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
return result
@app.get('/api/v1/chats/{chat_id}')
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
return chat_public(conn, row)
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
key = validate_key(request.headers.get('Idempotency-Key', ''))
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
return public(row)
@app.get('/api/v1/chats/{chat_id}/messages')
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'messages:' + str(cid)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
params = (cid, *after, limit+1) if after else (cid, limit+1)
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
return page(scope, rows, limit)
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
cid=uuid_value(chat_id)
def access():
with request.app.state.db.pool.connection() as conn:
require_member(conn,cid,user_id)
await run_in_threadpool(access)
return await attachments.upload(request,user_id,cid,filename)
@app.get('/api/v1/attachments/{attachment_id}')
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
return attachments.metadata(row)
@app.get('/api/v1/attachments/{attachment_id}/content')
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
if variant not in ('original','thumbnail','crop','optimized'):
raise invalid()
if row['status']!='ready':
raise ApiError(409,'conflict','Вложение ещё не готово')
item=row
key=row['object_key']
if variant!='original':
item=next((v for v in row['variants'] if v['name']==variant),None)
if item is None:
raise not_found()
key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant
opened=request.app.state.storage.get(key)
if opened.length!=item['size_bytes']:
opened.close()
raise unavailable()
return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={
'Content-Length':str(item['size_bytes']),
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
return app
+175
View File
@@ -0,0 +1,175 @@
"""Bounded binary upload, image validation and DB/S3 coordination."""
import asyncio
from . import telemetry
import hashlib
import logging
import tempfile
import threading
import unicodedata
import warnings
from uuid import uuid4
import psycopg
from PIL import Image, PngImagePlugin, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from .errors import ApiError, invalid, not_found
from .pagination import public
MAX_BYTES=10485760
MAX_PIXELS=40000000
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
PngImagePlugin.MAX_TEXT_CHUNK=1048576
PngImagePlugin.MAX_TEXT_MEMORY=1048576
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
DECODER=threading.BoundedSemaphore(1)
LOG=logging.getLogger('messenger')
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants'
def object_lock(conn,key,wait=True):
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
return True if wait else row['locked']
def filename_value(value):
if not value or not value.strip() or len(value)>255:
raise invalid()
try:
value.encode('utf-8')
except UnicodeError:
raise invalid() from None
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
raise invalid()
return value
def metadata(row):
result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')})
return result
def validate_image(file,mime):
if not DECODER.acquire(timeout=3):
raise ApiError(503,'unavailable','Обработка изображений занята')
try:
file.seek(0)
with Image.open(file) as image:
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
if actual is None or actual!=mime:
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
if image.width*image.height>MAX_PIXELS:
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
image.verify()
file.seek(0)
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
finally:
DECODER.release()
def compensate(db,store,key):
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
try:
with db.pool.connection() as conn:
if not object_lock(conn,key,wait=False):
return
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone()
if not exists:
store.delete(key)
except Exception as exc:
# A periodic, grace-protected janitor will retry after dependency recovery.
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
aid=uuid4()
key='staging/'+aid.hex
try:
with db.pool.connection() as conn:
object_lock(conn,key)
store.put(key,file,length,mime)
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) '
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS,
(aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone()
job=uuid4()
ctx=telemetry.carrier()
conn.execute('INSERT INTO attachment_jobs(id,attachment_id,traceparent,tracestate) VALUES (%s,%s,%s,%s)',(job,aid,ctx.get('traceparent'),ctx.get('tracestate')))
conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,))
return metadata(row)
except psycopg.Error:
compensate(db,store,key)
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
except BaseException:
compensate(db,store,key)
raise
async def upload(request,user_id,chat_id,filename):
filename=filename_value(filename)
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
if mime not in ('image/png','image/jpeg'):
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
declared=request.headers.get('Content-Length')
if declared is not None:
try:
length=int(declared)
if length<0:
raise ValueError
except ValueError:
raise invalid() from None
if length>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
try:
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
except TimeoutError:
raise ApiError(503,'unavailable','Слишком много загрузок') from None
try:
with tempfile.TemporaryFile(mode='w+b') as file:
total=0
digest=hashlib.sha256()
async def read_body():
nonlocal total
async for chunk in request.stream():
total+=len(chunk)
if total>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
digest.update(chunk)
await run_in_threadpool(file.write,chunk)
try:
await asyncio.wait_for(read_body(),timeout=30)
except TimeoutError:
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
except ClientDisconnect:
raise invalid() from None
await run_in_threadpool(validate_image,file,mime)
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
finally:
request.app.state.upload_slots.release()
def lookup(conn,attachment_id,user_id):
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
if not row:
raise not_found()
return row
class ObjectResponse(StreamingResponse):
def __init__(self,opened,**kwargs):
self.opened=opened
super().__init__(opened.chunks(),**kwargs)
async def __call__(self,scope,receive,send):
try:
await super().__call__(scope,receive,send)
finally:
self.opened.close()
+72
View File
@@ -0,0 +1,72 @@
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
import argparse
from datetime import datetime,timezone,timedelta
import json
from pathlib import Path
import time
from .attachments import object_lock
from .db import Database
from .object_store import ObjectStore
def sweep(db,store,grace_seconds=300,apply=False):
if grace_seconds<60:
raise ValueError('Grace must be at least60seconds')
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)):
for obj in page.get('Contents',[]):
result['listed']+=1
if obj['LastModified']>=cutoff:
result['fresh']+=1
continue
with db.pool.connection() as conn:
if not object_lock(conn,obj['Key'],wait=False):
result['locked']+=1
continue
if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone():
result['tracked']+=1
continue
result['orphans']+=1
if apply:
store.delete(obj['Key'])
result['deleted']+=1
return result
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--grace-seconds',type=int,default=300)
parser.add_argument('--apply',action='store_true')
parser.add_argument('--dry-run',action='store_true')
parser.add_argument('--loop',action='store_true')
parser.add_argument('--interval',type=int,default=60)
args=parser.parse_args()
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
db=Database()
store=ObjectStore()
db.open()
try:
while True:
try:
result=sweep(db,store,args.grace_seconds,args.apply)
print(json.dumps(result),flush=True)
except Exception as exc:
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
if not args.loop:
raise
finally:
Path('/tmp/cleanup-heartbeat').touch()
if not args.loop:
break
time.sleep(args.interval)
finally:
store.close()
db.close()
if __name__=='__main__':
main()
+62
View File
@@ -0,0 +1,62 @@
"""Bounded dependencies; PostgreSQL is always the source of truth."""
import os
from contextlib import contextmanager
from . import telemetry
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from redis import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
def setting(name, default=None):
filename = os.environ.get(name + '_FILE')
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
if value is None:
raise RuntimeError(f'Missing configuration: {name}')
return value
class TracedConnection(psycopg.Connection):
def execute(self,query,*args,**kwargs):
# Trace actual query wait, never a pooled lease containing S3/image work.
operation=str(query).split(None,1)[0].upper() if str(query).strip() else 'OTHER'
if operation not in ('SELECT','INSERT','UPDATE','DELETE'): operation='OTHER'
with telemetry.span('postgres.query',attributes={'db.system':'postgresql','db.operation.name':operation}):
try:
return super().execute(query,*args,**kwargs)
except Exception as exc:
telemetry.event('db_operation_failed',error_type=type(exc).__name__)
raise
def commit(self):
with telemetry.span('postgres.commit',attributes={'db.system':'postgresql'}):
return super().commit()
class Database:
def __init__(self):
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
if self.session_ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
self.pool = ConnectionPool(
setting('DATABASE_URL'), connection_class=TracedConnection, min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
timeout=2, max_waiting=16, open=False,
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
)
self.cache = Redis.from_url(
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
retry=Retry(NoBackoff(), 0),
)
def open(self):
self.pool.open(wait=True, timeout=30)
def close(self):
self.pool.close()
self.cache.close()
+19
View File
@@ -0,0 +1,19 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid():
return ApiError(400, 'invalid_request', 'Неверный запрос')
def unauthorized(basic=False):
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
def not_found():
return ApiError(404, 'not_found', 'Объект не найден')
+231
View File
@@ -0,0 +1,231 @@
"""Transactional outbox, fenced processing, bounded retries and operator replay."""
from contextlib import nullcontext
import signal
import time
from datetime import datetime,timezone
from . import telemetry
import hashlib
import json
import logging
import os
from pathlib import Path
import random
import subprocess
import sys
import tempfile
from uuid import UUID
from psycopg.types.json import Jsonb
from redis import Redis
from .db import setting
from .attachments import object_lock
LOG=logging.getLogger('messenger')
STREAM='attachment-jobs'
GROUP='image-workers'
MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5'))
LEASE_SECONDS=90
def broker_client():
return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True,
socket_connect_timeout=1,socket_timeout=3,max_connections=4)
def job_lock(conn,job_id):
return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
('job:'+str(job_id),)).fetchone()['ok']
def unlock(conn,job_id):
conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),))
def release_lock(conn,job_id):
# Session locks survive ROLLBACK. A failed transaction must be cleared before unlock.
try:
conn.rollback()
unlock(conn,job_id)
conn.commit()
except BaseException:
conn.close() # never return a session with an uncertain held lock to the pool
raise
def relay_tick(db,broker):
# Recovery uses the same session lock as the processor. A slow live worker is never fenced out.
with db.pool.connection() as conn:
expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='processing' AND j.lease_until<clock_timestamp() LIMIT 50").fetchall()
for row in expired:
with db.pool.connection() as conn:
if not job_lock(conn,row['id']):
continue
try:
conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id="
"(SELECT attachment_id FROM attachment_jobs WHERE id=%s AND lease_until<clock_timestamp()) "
"AND status='processing'",(row['id'],))
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(row['id'],))
conn.execute('UPDATE attachment_jobs SET next_at=clock_timestamp(),lease_until=NULL WHERE id=%s',(row['id'],))
conn.commit()
finally:
release_lock(conn,row['id'])
with db.pool.connection() as conn:
rows=conn.execute("SELECT j.*,o.job_id FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"JOIN attachments a ON a.id=j.attachment_id WHERE o.dispatched_at IS NULL AND a.status='queued' "
"AND j.next_at<=clock_timestamp() ORDER BY j.next_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED").fetchall()
for row in rows:
payload={'schema_version':1,'type':'attachment.process.v1','job_id':str(row['id']),
'attachment_id':str(row['attachment_id']),'pipeline_version':row['pipeline_version'],
'attempt':row['attempt']+1,'created_at':row['created_at'].isoformat()}
for key in ('traceparent','tracestate'):
if row[key]: payload[key]=row[key]
with telemetry.span('attachments.publish',kind=telemetry.SpanKind.PRODUCER,context=telemetry.propagate.extract(payload)):
payload.update(telemetry.carrier())
broker.xadd(STREAM,{'payload':json.dumps(payload)})
conn.execute('UPDATE job_outbox SET dispatched_at=clock_timestamp() WHERE job_id=%s',(row['id'],))
return len(rows)
def process_objects_local(store,row,directory=None):
with (nullcontext(directory) if directory else tempfile.TemporaryDirectory(prefix='image-job-')) as directory:
source=Path(directory)/'input'
opened=store.get(row['staging_key'])
try:
with source.open('wb') as file:
total=0
for chunk in opened.chunks():
total+=len(chunk)
if total>row['size_bytes'] or total>10485760: raise ValueError('invalid_image')
file.write(chunk)
finally:
opened.close()
raw_hash=hashlib.sha256(source.read_bytes()).hexdigest()
if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']:
raise ValueError('invalid_image')
completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory],
capture_output=True,timeout=20)
if completed.returncode:
raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit')
variants=json.loads(completed.stdout)
with source.open('rb') as file:
store.put(row['object_key'],file,row['size_bytes'],row['content_type'])
for name,item in variants.items():
with (Path(directory)/name).open('rb') as file:
store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type'])
return list(variants.values())
def process_objects(store,row):
# A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread.
timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45'))
if not 1<=timeout<=45: raise ValueError('processing_limit')
row={**row,'telemetry_context':telemetry.carrier()}
with tempfile.TemporaryDirectory(prefix='image-job-') as directory:
process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory],
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
text=True,start_new_session=True)
try:
output,_=process.communicate(json.dumps(row,default=str),timeout=timeout)
except BaseException:
# Terminate processor AND decoder before lock release or another attempt.
try: os.killpg(process.pid,signal.SIGKILL)
except ProcessLookupError: pass
process.communicate()
raise
if process.returncode==65: raise ValueError('invalid_image')
if process.returncode==66: raise ValueError('processing_limit')
if process.returncode: raise RuntimeError('storage_unavailable')
return json.loads(output)
def work(db,store,job_id,delivery_attempt=None):
job_id=UUID(str(job_id))
# Dedicated pooled session holds the lock, with short committed transactions between effects.
with db.pool.connection() as conn:
if not job_lock(conn,job_id):
return False
conn.commit()
try:
row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due '
'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone()
conn.commit()
if not row or row['status'] in ('ready','failed'):
return True
if row['status']!='queued' or not row['due']:
return False
if delivery_attempt is not None and delivery_attempt!=row['attempt']+1:
return True
attempt=row['attempt']+1
if attempt>MAX_ATTEMPTS:
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.commit()
return True
conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s",
(attempt,LEASE_SECONDS,job_id))
conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.commit()
started=time.monotonic()
try:
variants=process_objects(store,row)
except Exception as exc:
permanent=isinstance(exc,ValueError)
final=permanent or attempt>=MAX_ATTEMPTS
code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable')
if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'):
code='processing_failed'
delay=min(30,2**(attempt-1))*random.uniform(.5,1.5)
conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
('failed' if final else 'queued',code if final else None,row['id']))
conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id))
if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
conn.commit()
telemetry.job_done('failed' if final else 'retry',time.monotonic()-started,attempt)
LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt)
return True
# Same live lock and attempt fence protect publication; no new attempt can publish concurrently.
current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone()
if current['attempt']!=attempt:
conn.rollback()
return False
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(Jsonb(variants),row['id']))
conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,))
conn.commit()
telemetry.job_done('ready',time.monotonic()-started,attempt,(datetime.now(timezone.utc)-row['created_at']).total_seconds())
LOG.info('job_completed attempt=%s',attempt)
clean_staging(db,store,row['id'])
return True
finally:
release_lock(conn,job_id)
def clean_staging(db,store,attachment_id=None):
with db.pool.connection() as conn:
rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL "
"AND (status='ready' OR (status='failed' AND updated_at<clock_timestamp()-interval '24 hours')) "
+('AND id=%s ' if attachment_id else '')+'LIMIT 50', (attachment_id,) if attachment_id else ()).fetchall()
for row in rows:
with db.pool.connection() as conn:
if not object_lock(conn,row['staging_key'],wait=False): continue
current=conn.execute("SELECT status FROM attachments WHERE id=%s FOR UPDATE",(row['id'],)).fetchone()
if current['status'] not in ('ready','failed'): continue
store.delete(row['staging_key'])
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s',(row['id'],))
def replay(db,job_id,reason):
if not reason.strip() or len(reason)>200: raise ValueError('Short audit reason required')
with db.pool.connection() as conn:
if not job_lock(conn,job_id): raise ValueError('Job active')
try:
row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone()
if not row or row['status']!='failed' or not row['staging_key']:
raise ValueError('Replay requires failed job with retained staging')
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,))
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason))
conn.commit()
finally:
release_lock(conn,job_id)
+54
View File
@@ -0,0 +1,54 @@
"""One PostgreSQL transaction owns both a message and its retry result."""
import hashlib
import json
import re
from uuid import uuid4
from .errors import ApiError, invalid, not_found
from .models import uuid_value
def validate_key(key):
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
raise invalid()
return key
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
attachment_ids=[] if attachment_ids is None else attachment_ids
payload={'text':text}
if attachment_ids:
payload['attachment_ids']=attachment_ids
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
params = (user_id, chat_id, key)
# A competing INSERT waits for the other transaction, then the next statement sees its result.
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
(*params, fingerprint))
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
if record['message_id'] is not None and record['active']:
if record['request_hash'] != fingerprint:
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
(record['message_id'],)).fetchone()
ids=[uuid_value(item) for item in attachment_ids]
if ids:
allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
if len(allowed)!=len(ids):
raise not_found()
if any(a['status']=='failed' for a in allowed):
raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой')
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
if ids:
with conn.cursor() as cur:
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
[(row['id'],aid,i) for i,aid in enumerate(ids)])
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
+29
View File
@@ -0,0 +1,29 @@
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
import hashlib
from pathlib import Path
import psycopg
from .db import setting
def main():
directory = Path(__file__).resolve().parents[2] / 'migrations'
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(directory.glob('*.sql')):
sql = path.read_text()
checksum = hashlib.sha256(sql.encode()).hexdigest()
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
if found:
if found[0] != checksum:
raise RuntimeError('Applied migration checksum changed: ' + path.name)
continue
conn.execute(sql)
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
print('Applied', path.name)
print('Migrations up to date')
if __name__ == '__main__':
main()
+75
View File
@@ -0,0 +1,75 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from .errors import invalid
def uuid_value(value):
try:
# Require the UUID string representation, not integer/coercible values.
if not isinstance(value, str) or len(value) != 36:
raise ValueError
return UUID(value)
except (ValueError, TypeError, AttributeError):
raise invalid() from None
class StrictModel(BaseModel):
model_config = ConfigDict(extra='forbid', strict=True)
@field_validator('*')
@classmethod
def valid_strings(cls, value, info):
if isinstance(value, str):
if not value.strip() and info.field_name != 'text':
raise ValueError('Blank string')
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
if '\x00' in value:
raise ValueError('NUL is not supported')
value.encode('utf-8')
return value
class CreateUser(StrictModel):
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator('username')
@classmethod
def username_ascii(cls, value):
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
raise ValueError('Invalid username')
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator('member_ids')
@classmethod
def members(cls, value):
ids = [uuid_value(item) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('Duplicate member')
return [str(item) for item in ids]
class CreateMessage(StrictModel):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
@field_validator('attachment_ids')
@classmethod
def attachment_values(cls,value):
ids=[uuid_value(item) for item in value]
if len(ids)!=len(set(ids)):
raise ValueError('Duplicate attachment')
return [str(item) for item in ids]
@model_validator(mode='after')
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('Message must contain text or attachment')
return self
+123
View File
@@ -0,0 +1,123 @@
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
from contextlib import contextmanager
from functools import wraps
from . import telemetry
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .db import setting
from .errors import ApiError
PREFIX = 'originals/'
def unavailable():
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
class OpenObject:
def __init__(self,body,release,length):
self.body=body
self.release=release
self.length=length
self.closed=False
self.close_lock=threading.Lock()
def chunks(self):
try:
while True:
data=self.body.read(65536)
if not data:
break
yield data
finally:
self.close()
def close(self):
with self.close_lock:
if self.closed:
return
self.closed=True
try:
self.body.close()
finally:
self.release()
class ObjectStore:
def __init__(self):
self.bucket=setting('S3_BUCKET','messenger-attachments')
self.gate=threading.BoundedSemaphore(8)
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
region_name=setting('S3_REGION','us-east-1'),
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
@contextmanager
def operation(self):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
yield
except (BotoCoreError,ClientError):
raise unavailable() from None
finally:
self.gate.release()
def check(self):
with self.operation():
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
def put(self,key,body,length,mime):
with self.operation():
body.seek(0)
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
def delete(self,key):
with self.operation():
self.client.delete_object(Bucket=self.bucket,Key=key)
def get(self,key):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
result=self.client.get_object(Bucket=self.bucket,Key=key)
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
except (BotoCoreError,ClientError):
self.gate.release()
raise unavailable() from None
except BaseException:
self.gate.release()
raise
def pages(self,prefix=PREFIX):
token=None
while True:
params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000}
if token:
params['ContinuationToken']=token
with self.operation():
result=self.client.list_objects_v2(**params)
yield result
if not result.get('IsTruncated'):
break
token=result['NextContinuationToken']
def close(self):
self.client.close()
def traced_s3(function):
@wraps(function)
def wrapped(*args,**kwargs):
with telemetry.span('s3.'+function.__name__,attributes={'rpc.system':'aws-api','rpc.service':'S3'}):
return function(*args,**kwargs)
return wrapped
for _operation in ('put','get','delete','check'):
setattr(ObjectStore,_operation,traced_s3(getattr(ObjectStore,_operation)))
+56
View File
@@ -0,0 +1,56 @@
import base64
import json
import re
from datetime import datetime, timezone
from uuid import UUID
from .errors import invalid
from .models import uuid_value
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def encode_cursor(scope, row):
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
def decode_cursor(scope, value):
if value is None:
return None
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
raise ValueError
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
raise ValueError
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
if ts.tzinfo is None:
raise ValueError
return ts, uuid_value(data[2])
except (ValueError, TypeError, AttributeError, OverflowError):
raise invalid() from None
def page_limit(value):
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
raise invalid()
return int(value)
def public(row):
def value(item):
if isinstance(item,datetime):
return stamp(item)
if isinstance(item,UUID):
return str(item)
if isinstance(item,list):
return [value(part) for part in item]
return item
return {key:value(item) for key,item in row.items()}
def page(scope, rows, limit):
return {'items': [public(row) for row in rows[:limit]],
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
+49
View File
@@ -0,0 +1,49 @@
"""Bounded image decoder subprocess. Original bytes are never re-encoded."""
import hashlib
import json
import resource
import sys
from pathlib import Path
from PIL import Image, ImageOps, PngImagePlugin
def fitted(image, maximum):
scale=min(1,maximum/image.width,maximum/image.height)
size=(max(1,int(image.width*scale)),max(1,int(image.height*scale)))
return image.resize(size,Image.Resampling.LANCZOS)
def process_file(source, directory):
Image.MAX_IMAGE_PIXELS=40000000
PngImagePlugin.MAX_TEXT_CHUNK=1048576
PngImagePlugin.MAX_TEXT_MEMORY=1048576
with Image.open(source) as original:
if original.format not in ('PNG','JPEG') or original.width*original.height>40000000:
raise ValueError('invalid_image')
image=ImageOps.exif_transpose(original)
# Alpha is composited on white; output carries no source metadata.
rgba=image.convert('RGBA')
white=Image.new('RGBA',rgba.size,'white')
white.alpha_composite(rgba)
image=white.convert('RGB')
image.info.clear()
outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'),
'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'),
'optimized':(fitted(image,1280),'WEBP','image/webp')}
manifest={}
for name,(variant,fmt,mime) in outputs.items():
path=Path(directory)/name
variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'')
raw=path.read_bytes()
manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height,
'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}
return manifest
if __name__=='__main__':
resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024))
resource.setrlimit(resource.RLIMIT_CPU,(15,15))
try:
print(json.dumps(process_file(sys.argv[1],sys.argv[2])))
except (OSError,ValueError,MemoryError,Image.DecompressionBombError):
sys.exit(65)
+33
View File
@@ -0,0 +1,33 @@
"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds."""
import os
import signal
import json
from . import telemetry
import sys
from uuid import UUID
from .jobs import process_objects_local
from .object_store import ObjectStore
def main():
# Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline.
signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL))
signal.alarm(45)
row=json.load(sys.stdin)
row['id']=UUID(row['id'])
telemetry.init('messenger-worker',child=True)
store=ObjectStore()
try:
with telemetry.span('attachments.process_objects',context=telemetry.propagate.extract(row.get('telemetry_context',{}))):
print(json.dumps(process_objects_local(store,row,sys.argv[1])))
except ValueError as exc:
sys.exit(66 if str(exc)=='processing_limit' else 65)
except Exception:
# Do not export credentials, storage keys or SDK response details.
sys.exit(75)
finally:
store.close()
telemetry.shutdown()
if __name__=='__main__': main()
+49
View File
@@ -0,0 +1,49 @@
import base64
import hashlib
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from .errors import unauthorized
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
def digest(token):
return hashlib.sha256(token.encode()).hexdigest()
def basic_credentials(header):
try:
scheme, encoded = header.split(' ', 1)
if scheme.lower() != 'basic':
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def check_password(encoded, password):
try:
return HASHER.verify(encoded, password)
except (VerificationError, InvalidHashError):
return False
def bearer(header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError
return token
except ValueError:
raise unauthorized() from None
def new_session(ttl):
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
+154
View File
@@ -0,0 +1,154 @@
"""Explicit low-cardinality telemetry; no bodies, headers, storage keys or SQL params."""
from contextlib import contextmanager
import json
import logging
import os
import re
import time
from uuid import uuid4
from opentelemetry import trace,metrics,propagate
from opentelemetry.trace import SpanKind,StatusCode
HTTP_ROUTES={'/api/v1/users','/api/v1/users/me','/api/v1/auth/sessions','/api/v1/auth/sessions/current',
'/api/v1/chats','/health/live','/health/ready'}
ALLOWED={'event','request_id','route','method','status_class','outcome','attempt','error_type'}
INITIALIZED=False
PROVIDERS=[]
HANDLER=None
SERVICE='messenger-api'
BUSY=0
HTTP_COUNT=HTTP_TIME=JOB_COUNT=JOB_TIME=READY_TIME=None
def route_name(path):
if path in HTTP_ROUTES: return path
if re.fullmatch(r'/api/v1/chats/[^/]+(?:/(?:messages|attachments))?',path):
return re.sub(r'(?<=/chats/)[^/]+','{chat_id}',path)
if re.fullmatch(r'/api/v1/attachments/[^/]+(?:/content)?',path):
return re.sub(r'(?<=/attachments/)[^/]+','{attachment_id}',path)
return 'unmatched'
def safe_fields(fields):
return {key:value for key,value in fields.items() if key in ALLOWED and isinstance(value,(str,int,float,bool))}
def tracer(): return trace.get_tracer('messenger','1.0')
def carrier():
data={};propagate.inject(data);return data
@contextmanager
def span(name,kind=SpanKind.INTERNAL,context=None,attributes=None):
with tracer().start_as_current_span(name,kind=kind,context=context,attributes=attributes or {},
record_exception=False,set_status_on_exception=False) as active:
try: yield active
except BaseException:
active.set_status(StatusCode.ERROR)
raise
def event(name,**fields):
if HANDLER is None: return
context=trace.get_current_span().get_span_context()
data=safe_fields({'event':name,**fields})
data.update(timestamp=time.time(),severity='INFO',service_name=SERVICE,
trace_id=format(context.trace_id,'032x'),span_id=format(context.span_id,'016x'))
record=logging.LogRecord('messenger.events',logging.INFO,'',0,data,(),None)
HANDLER.emit(record)
print(json.dumps(data),flush=True)
def init(service,db=None,child=False):
global INITIALIZED,HANDLER,SERVICE,HTTP_COUNT,HTTP_TIME,JOB_COUNT,JOB_TIME,READY_TIME
if INITIALIZED or os.environ.get('OTEL_ENABLED')!='1': return
INITIALIZED=True;SERVICE=service
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.metrics import Observation
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View,ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider,LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
resource=Resource.create({'service.name':service,'service.version':'lab6-v1',
'service.instance.id':os.environ.setdefault('TELEMETRY_INSTANCE_ID',uuid4().hex),
'deployment.environment.name':'lab'})
endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT','http://collector:4318').rstrip('/')
tp=TracerProvider(resource=resource)
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint+'/v1/traces',timeout=1),
max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000))
trace.set_tracer_provider(tp);PROVIDERS.append(tp)
lp=LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint+'/v1/logs',timeout=1),
max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000))
HANDLER=LoggingHandler(level=logging.INFO,logger_provider=lp);PROVIDERS.append(lp)
if child: return
reader=PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint+'/v1/metrics',timeout=1),
export_interval_millis=2000,export_timeout_millis=1500)
mp=MeterProvider(resource=resource,metric_readers=[reader],views=[
View(instrument_name='*_duration',aggregation=ExplicitBucketHistogramAggregation([.005,.01,.025,.05,.1,.25,.5,1,2,5,10,30,60,120])),
View(instrument_name='attachment_ready',aggregation=ExplicitBucketHistogramAggregation([.1,.5,1,2,5,10,30,60,120,300]))])
metrics.set_meter_provider(mp);PROVIDERS.append(mp)
meter=metrics.get_meter('messenger','1.0')
HTTP_COUNT=meter.create_counter('http_requests')
HTTP_TIME=meter.create_histogram('http_server_duration',unit='s')
JOB_COUNT=meter.create_counter('jobs_processed')
JOB_TIME=meter.create_histogram('job_processing_duration',unit='s')
READY_TIME=meter.create_histogram('attachment_ready',unit='s')
def process_values(_):
rss=int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE')
return [Observation(rss)]
meter.create_observable_gauge('process_memory',unit='By',callbacks=[process_values])
meter.create_observable_counter('process_cpu',unit='s',callbacks=[lambda _:[Observation(time.process_time())]])
meter.create_observable_gauge('worker_busy',callbacks=[lambda _:[Observation(BUSY)]])
meter.create_observable_gauge('worker_capacity',callbacks=[lambda _:[Observation(1 if service=='messenger-worker' else 0)]])
if db:
def pool_value(name):
return lambda _:[Observation(db.pool.get_stats().get(name,0))]
meter.create_observable_gauge('db_pool_used',callbacks=[lambda _:[Observation(db.pool.get_stats().get('pool_size',0)-db.pool.get_stats().get('pool_available',0))]])
meter.create_observable_gauge('db_pool_waiting',callbacks=[pool_value('requests_waiting')])
def queue_values(field):
def callback(_):
try:
with db.pool.connection() as conn:
row=conn.execute("SELECT count(*) AS depth,coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0)::float AS age FROM attachments WHERE status IN ('queued','processing')").fetchone()
return [Observation(row[field],{'queue':'attachments'})]
except Exception: return []
return callback
if service=='messenger-relay':
meter.create_observable_gauge('queue_depth',callbacks=[queue_values('depth')])
meter.create_observable_gauge('queue_oldest_age',unit='s',callbacks=[queue_values('age')])
def http_done(method,route,status,duration,request_id):
fields={'method':method,'route':route,'status_class':str(status//100)+'xx'}
if HTTP_COUNT and not route.startswith('/health/'):
HTTP_COUNT.add(1,fields);HTTP_TIME.record(duration,fields)
event('http_completed',**fields,request_id=request_id)
def job_done(outcome,duration,attempt,accepted_age=None):
current=trace.get_current_span()
current.set_attribute('job.type','attachment.process.v1')
current.set_attribute('job.outcome',outcome)
current.set_attribute('job.attempt',attempt)
if outcome in ('retry','failed'): current.set_status(StatusCode.ERROR)
attrs={'job_type':'attachment.process.v1','outcome':outcome}
if JOB_COUNT:
JOB_COUNT.add(1,attrs);JOB_TIME.record(duration,attrs)
if accepted_age is not None and outcome=='ready': READY_TIME.record(accepted_age,{'job_type':'attachment.process.v1'})
event('job_attempt',outcome=outcome,attempt=attempt)
def shutdown():
for provider in reversed(PROVIDERS):
try: provider.shutdown()
except Exception: pass
+77
View File
@@ -0,0 +1,77 @@
"""Separate relay/consumer processes; durable Redis Streams consumer group."""
import argparse
from . import telemetry
import json
import logging
import signal
import time
from uuid import uuid4
from redis.exceptions import ResponseError
from .cleanup import sweep
from .db import Database
from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP
from .object_store import ObjectStore
LOG=logging.getLogger('messenger')
STOP=False
def main():
parser=argparse.ArgumentParser()
parser.add_argument('role',choices=['worker','relay','replay'])
parser.add_argument('--job-id')
parser.add_argument('--reason',default='operator retry')
args=parser.parse_args()
logging.basicConfig(level=logging.INFO)
def stop(*_):
global STOP
STOP=True
signal.signal(signal.SIGTERM,stop)
signal.signal(signal.SIGINT,stop)
db=Database();db.open();store=ObjectStore();broker=broker_client()
telemetry.init('messenger-worker' if args.role=='worker' else 'messenger-relay',db)
consumer=uuid4().hex
last_cleanup=0
try:
if args.role=='replay':
replay(db,args.job_id,args.reason)
print('Replay queued with audit')
return
if args.role=='worker':
try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except ResponseError as exc:
if 'BUSYGROUP' not in str(exc): raise
while not STOP:
try:
if args.role=='relay':
relay_tick(db,broker)
clean_staging(db,store)
if time.monotonic()-last_cleanup>60:
sweep(db,store,300,True)
last_cleanup=time.monotonic()
time.sleep(.3)
else:
claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=5000,start_id='0-0',count=1)
messages=claimed[1]
if not messages:
batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
messages=batches[0][1] if batches else []
for message_id,fields in messages:
payload=json.loads(fields['payload'])
with telemetry.span('attachments.consume',kind=telemetry.SpanKind.CONSUMER,
context=telemetry.propagate.extract(payload)):
telemetry.BUSY=1
try:
if work(db,store,payload['job_id'],payload['attempt']):
broker.xack(STREAM,GROUP,message_id)
broker.xdel(STREAM,message_id)
finally:
telemetry.BUSY=0
except Exception as exc:
LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__)
time.sleep(1)
finally:
broker.close();store.close();db.close();telemetry.shutdown()
if __name__=='__main__': main()
+141
View File
@@ -0,0 +1,141 @@
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
import base64
import json
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
from fastapi.testclient import TestClient
from messenger.app import create_app
PASSWORD = 'Тестовый:пароль🔐' * 8
@pytest.fixture
def client():
with TestClient(create_app()) as c:
yield c
def user(c):
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
r = c.post('/api/v1/users', json=body)
assert r.status_code == 201, r.text
return r.json(), body
def login(c, body):
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
assert r.status_code == 201, r.text
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
def test_auth_unicode_logout_and_cache(client):
u, body = user(client)
h, session = login(client, body)
assert session['user'] == u
assert len(session['token']) >= 43
key = 'user:' + u['id']
client.app.state.db.cache.delete(key)
for _ in range(2):
assert client.get('/api/v1/users/me', headers=h).json() == u
cached = json.loads(client.app.state.db.cache.get(key))
assert cached == u
assert 0 < client.app.state.db.cache.ttl(key) <= 60
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
r = client.get('/api/v1/users/me', headers=h)
assert r.status_code == 401
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
assert r.json()['error']['request_id'] == r.headers['x-request-id']
def test_session_expiry(monkeypatch):
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
with TestClient(create_app()) as client:
_, body = user(client)
h, _ = login(client, body)
assert client.get('/api/v1/users/me', headers=h).status_code == 200
time.sleep(2.1)
assert client.get('/api/v1/users/me', headers=h).status_code == 401
def test_registration_race(client):
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
with ThreadPoolExecutor(max_workers=6) as executor:
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
def test_chat_rollback_after_insert(client):
u, body = user(client)
h, _ = login(client, body)
title = 'rollback_' + uuid.uuid4().hex
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
assert r.status_code == 400
with client.app.state.db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
def test_cursor_access_and_strict_validation(client):
a, ab = user(client)
b, bb = user(client)
ah, _ = login(client, ab)
bh, _ = login(client, bb)
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
path = '/api/v1/chats/' + chats[0]['id']
for text in ['Один 👋', 'Два', 'Три']:
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
p = client.get(path+'/messages?limit=2', headers=ah).json()
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
assert [m['text'] for m in tail['items']] == ['Три']
assert tail['next_cursor'] is None
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
assert r.status_code == 400
assert client.get(path, headers=bh).status_code == 404
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
assert client.post('/api/v1/users',json=payload).status_code == 400
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
def test_session_storage_and_migrations(client):
from messenger.migrate import main as migrate
u, body = user(client)
_, session = login(client, body)
db = client.app.state.db
def snapshot():
with db.pool.connection() as conn:
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
before = snapshot()
migrate()
migrate()
assert snapshot() == before
with db.pool.connection() as conn:
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
assert row['token_hash'] != session['token']
assert len(row['token_hash']) == 64
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
assert stored.startswith('$argon2id$')
assert stored != body['password']
def test_json_media_type_and_utf8(client):
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
raw = json.dumps(body).encode()
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
r = client.post('/api/v1/users',content=raw,headers=headers)
assert r.status_code == 415
assert r.json()['error']['code'] == 'unsupported_media_type'
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
assert r.status_code == 400, encoding
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
assert r.status_code == 401
+187
View File
@@ -0,0 +1,187 @@
"""Real PostgreSQL + private S3 integration; synthetic images only."""
import hashlib
import io
import struct
import uuid
import zlib
from PIL import Image
from psycopg import sql
from test_api import client, user, login
def image_bytes(kind='PNG'):
stream = io.BytesIO()
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
return stream.getvalue()
def actors(c):
a,ab=user(c)
b,bb=user(c)
e,eb=user(c)
ah,_=login(c,ab)
bh,_=login(c,bb)
eh,_=login(c,eb)
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
return ah,bh,eh,chat
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
raw=image_bytes() if raw is None else raw
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
headers={**headers,'Content-Type':mime})
def finish(client,attachment_id):
from messenger.jobs import work
with client.app.state.db.pool.connection() as conn:
job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id']
work(client.app.state.db,client.app.state.storage,job)
import time
for _ in range(100):
with client.app.state.db.pool.connection() as conn:
status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status']
if status=='ready': return
time.sleep(.1)
raise AssertionError('Job did not become ready: '+status)
def test_original_png_jpeg_and_message(client):
ah,bh,eh,chat=actors(client)
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
raw=image_bytes(kind)
r=upload(client,ah,chat,raw,mime)
assert r.status_code==202
att=r.json()
assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None
assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409
finish(client,att['id'])
att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json()
assert att['status']=='ready' and len(att['variants'])==3
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
assert 'object_key' not in att and 'bucket' not in att
path='/api/v1/attachments/'+att['id']
assert client.get(path,headers=bh).json()==att
content=client.get(path+'/content',headers=bh)
assert content.status_code==200 and content.content==raw
assert content.headers['Content-Type']==mime
for suffix in ['', '/content']:
assert client.get(path+suffix,headers=eh).status_code==404
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
json={'text':'','attachment_ids':[att['id']]})
assert response.status_code==201
assert response.json()['attachment_ids']==[att['id']]
def test_invalid_media_corruption_limits_and_filename(client):
ah,_,_,chat=actors(client)
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
for raw,mime,expected in cases:
response=upload(client,ah,chat,raw,mime)
assert response.status_code==expected
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
# Valid PNG header advertises >40Mp; reject before attempting decompression.
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
assert upload(client,ah,chat,bomb).status_code==413
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
assert upload(client,ah,chat,filename=filename).status_code==400
def test_foreign_attachment_binding_and_normalized_replay(client):
ah,bh,eh,chat=actors(client)
att=upload(client,ah,chat).json()
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
path='/api/v1/chats/'+other['id']+'/messages'
for aid in [att['id'],str(uuid.uuid4())]:
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
assert response.status_code==404
assert upload(client,eh,chat).status_code==404
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
path='/api/v1/chats/'+chat['id']+'/messages'
first=client.post(path,headers=h,json={'text':'same'})
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
assert conflict.status_code==409
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
assert duplicates.status_code==400
def test_compensate_s3_when_metadata_insert_fails(client):
ah,_,_,chat=actors(client)
db=client.app.state.db
store=client.app.state.storage
before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
function='fail_attachment_'+uuid.uuid4().hex
trigger='fail_attachment_'+uuid.uuid4().hex
filename='fault_'+uuid.uuid4().hex+'.png'
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
response=upload(client,ah,chat,filename=filename)
assert response.status_code==503
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
assert after==before
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
def test_compensation_preserves_committed_object(client):
from messenger.attachments import compensate
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
# A retry after an unknown COMMIT result must never delete an already tracked object.
compensate(client.app.state.db,client.app.state.storage,key)
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
assert content.status_code==200 and content.content==image_bytes()
def test_stream_close_even_if_headers_cannot_be_sent(client):
import asyncio
import pytest
from messenger.attachments import ObjectResponse
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
store=client.app.state.storage
opened=store.get(key)
response=ObjectResponse(opened,media_type='image/png')
async def fail_send(message):
raise OSError('synthetic disconnected transport')
async def receive():
return {'type':'http.disconnect'}
with pytest.raises(Exception):
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
assert opened.closed
acquired=0
try:
for _ in range(8):
assert store.gate.acquire(blocking=False)
acquired+=1
finally:
for _ in range(acquired):
store.gate.release()
def test_chunked_upload_is_limited_without_content_length(client):
ah,_,_,chat=actors(client)
chunks=iter([b'x'*1048576]*10+[b'x'])
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
content=chunks,headers={**ah,'Content-Type':'image/png'})
assert response.status_code==413
assert response.json()['error']['code']=='payload_too_large'
+119
View File
@@ -0,0 +1,119 @@
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from psycopg import sql
from test_api import client, user, login
def setup_chat(c, extra=None):
u, body = user(c)
headers, _ = login(c, body)
members = [u['id']] + (extra or [])
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
def test_instance_alias_on_success_and_errors(client):
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
ids = [r.headers.get('X-Instance-Id') for r in responses]
assert ids[0] and len(set(ids)) == 1
assert all(r.headers.get('X-Request-Id') for r in responses)
def test_key_required_replay_and_conflict(client):
_, headers, path = setup_chat(client)
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
for key in ['', 'a b', 'bad/', 'x'*129]:
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
h = {**headers, 'Idempotency-Key':'valid:._-123'}
first = client.post(path, headers=h, json={'text':'same'})
assert first.status_code == 201
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
assert replay.status_code == 201 and replay.json() == first.json()
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
assert len(client.get(path,headers=headers).json()['items']) == 1
def test_concurrent_key_produces_one_message(client):
_, headers, path = setup_chat(client)
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
assert [r.status_code for r in results] == [201]*16
assert all(r.json() == results[0].json() for r in results)
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
def test_scope_and_replay_rechecks_authorization(client):
b, bb = user(client)
bh, _ = login(client, bb)
a, ah, path = setup_chat(client, [b['id']])
key = 'same-in-independent-scopes'
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
assert second.status_code == 201 and second.json()['id'] != first['id']
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
other_path = '/api/v1/chats/'+chat['id']+'/messages'
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
assert third.status_code == 201 and third.json()['id'] != first['id']
outsider, ob = user(client)
oh, _ = login(client, ob)
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
def test_expired_key_can_be_reused(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
with client.app.state.db.pool.connection() as conn:
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
assert 86390 <= ttl <= 86400
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
assert second.status_code == 201 and second.json()['id'] != first['id']
assert len(client.get(path,headers=h).json()['items']) == 2
def test_rollback_side_effect_and_key_on_database_error(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
text = 'fault_' + uuid.uuid4().hex
function = 'fault_' + uuid.uuid4().hex
trigger = 'trigger_' + uuid.uuid4().hex
db = client.app.state.db
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert failed.status_code == 500
assert failed.headers['X-Instance-Id']
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert retried.status_code == 201
assert client.get(path,headers=h).json()['items'] == [retried.json()]
def test_ready_requires_compatible_schema(client):
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
try:
r = client.get('/health/ready')
assert r.status_code == 503
assert r.json()['error']['code'] == 'unavailable'
assert client.get('/health/live').status_code == 200
finally:
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
+63
View File
@@ -0,0 +1,63 @@
import io
import pytest
from messenger.jobs import work, replay
from test_attachments import actors,upload,image_bytes,finish
from test_api import client
def job_for(client,aid):
with client.app.state.db.pool.connection() as conn:
return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id']
def test_duplicate_ready_is_noop_and_not_replayable(client):
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
path='/api/v1/attachments/'+att['id']
before=client.get(path,headers=ah).json()
job=job_for(client,att['id'])
for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1)
assert client.get(path,headers=ah).json()==before
with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready')
with client.app.state.db.pool.connection() as conn:
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
assert row['attempt']==1
assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None
def test_old_delivery_does_not_consume_attempt(client):
# Direct invocation avoids consumer timing; it exercises the same real DB/S3 code.
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
job=job_for(client,att['id'])
assert work(client.app.state.db,client.app.state.storage,job,999)
with client.app.state.db.pool.connection() as conn:
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
assert row['attempt'] in (0,1) # independent live consumer may already finish this job
finish(client,att['id'])
def test_replay_sql_failure_releases_session_lock(client):
import psycopg
from messenger.db import setting
from psycopg.rows import dict_row
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
job=job_for(client,att['id'])
db=client.app.state.db
with db.pool.connection() as conn:
conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id']))
conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$")
conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()')
try:
with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock')
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other:
acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok']
if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),))
assert acquired, 'Aborted replay leaked a session advisory lock into the pool'
finally:
with db.pool.connection() as conn:
conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays')
conn.execute('DROP FUNCTION lab5_fail_audit()')
+72
View File
@@ -0,0 +1,72 @@
"""Real image transformations, no mock storage."""
import hashlib
import io
import json
import subprocess
import sys
from PIL import Image
def test_worker_variants_and_original(tmp_path):
source=tmp_path/'input'
original=Image.new('RGBA',(320,200),(30,90,150,70))
original.save(source,format='PNG')
before=hashlib.sha256(source.read_bytes()).hexdigest()
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
assert result.returncode==0, result.stderr
manifest=json.loads(result.stdout)
assert hashlib.sha256(source.read_bytes()).hexdigest()==before
for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]:
item=manifest[kind]
with Image.open(tmp_path/kind) as image:
assert image.format==fmt and image.size==size
assert not image.getexif()
raw=(tmp_path/kind).read_bytes()
assert item['sha256']==hashlib.sha256(raw).hexdigest()
assert item['size_bytes']==len(raw)
def test_no_upscale_thumbnail_and_exif(tmp_path):
source=tmp_path/'input'
image=Image.new('RGB',(80,40),'red')
exif=Image.Exif();exif[274]=6
image.save(source,format='JPEG',exif=exif)
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
assert result.returncode==0, result.stderr
with Image.open(tmp_path/'thumbnail') as output:
assert output.size==(40,80)
assert not output.getexif()
def test_deadline_covers_slow_s3(monkeypatch):
import contextlib
import http.server
import os
import threading
import time
import uuid
import pytest
from messenger.jobs import process_objects
class Slow(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-Length','100')
self.end_headers()
try:
for _ in range(100):
self.wfile.write(b'x');self.wfile.flush();time.sleep(.05)
except (BrokenPipeError,ConnectionResetError): pass
def log_message(self,*args): pass
server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow)
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False)
monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port))
monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2')
row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow',
'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'}
started=time.monotonic()
try:
with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row)
assert time.monotonic()-started<4
finally:
server.shutdown();server.server_close()
+51
View File
@@ -0,0 +1,51 @@
from test_api import client
from messenger.telemetry import route_name, safe_fields
def test_route_labels_never_include_ids_or_query():
assert route_name('/api/v1/chats/a87e23bc-430c-4862-9da8-1de1286931aa/messages')=='/api/v1/chats/{chat_id}/messages'
assert route_name('/api/v1/attachments/a87e23bc-430c-4862-9da8-1de1286931aa/content')=='/api/v1/attachments/{attachment_id}/content'
assert route_name('/unknown/private-name')=='unmatched'
def test_log_attributes_allowlist_drops_sensitive_values():
result=safe_fields({'event':'http_completed','request_id':'test','route':'/api/v1/users',
'Authorization':'Bearer SECRET','password':'SECRET','body':'SECRET','filename':'private'})
assert result=={'event':'http_completed','request_id':'test','route':'/api/v1/users'}
def test_retry_consumer_span_marks_failure():
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import StatusCode
from messenger import telemetry
exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter))
with provider.get_tracer('test').start_as_current_span('attachments.consume'):
telemetry.job_done('retry',.1,2)
span=exporter.get_finished_spans()[0]
assert span.attributes['job.outcome']=='retry'
assert span.attributes['job.attempt']==2
assert span.status.status_code==StatusCode.ERROR
provider.shutdown()
def test_db_span_covers_query_not_image_work(client,monkeypatch):
import time
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from messenger import telemetry
exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter))
monkeypatch.setattr(telemetry,'tracer',lambda:provider.get_tracer('test'))
with client.app.state.db.pool.connection() as conn:
time.sleep(.15) # non-DB work with a leased connection
conn.execute('SELECT pg_sleep(0.02)')
spans=exporter.get_finished_spans()
assert any(s.name=='postgres.query' for s in spans)
assert not any(s.name=='postgres.transaction' for s in spans)
for span in spans:
if span.name=='postgres.query':
assert (span.end_time-span.start_time)/1e9<.15
assert 'db.statement' not in span.attributes
provider.shutdown()