ЛР 6 — БПИ2303 — Лукьянов — на 4 #17
@@ -18,3 +18,6 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
.superpowers
|
||||
.pytest_cache
|
||||
|
||||
+4
-29
@@ -1,30 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# bootstrap generates .env and local files under ignored secrets/.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# Compose enables OTEL_ENABLED=1 and sends all three signals to collector:4318.
|
||||
# Grafana operator password is generated in secrets/grafana_password.
|
||||
# Fixed profile: APIx2; worker concurrency1/prefetch1; processing45s/lease90s; max5 attempts.
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Application acceptance
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
observability:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- run: make up
|
||||
- run: make check validate-container test test-student test-fresh
|
||||
- run: make test-jobs
|
||||
- run: make test-cluster
|
||||
- run: make test-observability
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: lab6-evidence
|
||||
path: evidence/
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
FROM python:3.13.5-slim
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements-dev.txt requirements.lock ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
|
||||
COPY src/ src/
|
||||
COPY migrations/ migrations/
|
||||
COPY student-tests/ student-tests/
|
||||
COPY scripts/ scripts/
|
||||
COPY tests/ tests/
|
||||
COPY contracts/ contracts/
|
||||
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
|
||||
ARG APP_RELEASE=lab6-v1
|
||||
ENV APP_RELEASE=${APP_RELEASE}
|
||||
LABEL org.opencontainers.image.version=${APP_RELEASE}
|
||||
USER messenger
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
|
||||
@@ -28,3 +28,42 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
.PHONY: bootstrap up down test-student test-cluster rolling
|
||||
bootstrap:
|
||||
./scripts/bootstrap.sh
|
||||
up: bootstrap
|
||||
docker compose build api minio minio-init
|
||||
docker compose up -d --wait --wait-timeout 120
|
||||
$(PYTHON) scripts/cluster.py wait
|
||||
down:
|
||||
docker compose down
|
||||
test-student:
|
||||
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
test-cluster:
|
||||
$(PYTHON) scripts/verify_cluster.py
|
||||
rolling:
|
||||
$(PYTHON) scripts/cluster.py rolling
|
||||
|
||||
.PHONY: test-fresh load-evidence verify-persistence verify-failover
|
||||
test-fresh:
|
||||
./scripts/check_fresh_migrations.sh
|
||||
load-evidence:
|
||||
$(PYTHON) scripts/capture_load.py
|
||||
verify-persistence:
|
||||
$(PYTHON) scripts/verify_cluster.py persistence
|
||||
verify-failover:
|
||||
$(PYTHON) scripts/verify_cluster.py public-failover
|
||||
$(PYTHON) scripts/verify_cluster.py failover
|
||||
|
||||
.PHONY: validate-container
|
||||
validate-container:
|
||||
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
|
||||
|
||||
.PHONY: test-jobs
|
||||
test-jobs:
|
||||
$(PYTHON) scripts/verify_jobs.py
|
||||
|
||||
.PHONY: test-observability
|
||||
test-observability:
|
||||
$(PYTHON) scripts/verify_observability.py
|
||||
|
||||
@@ -1,37 +1,114 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Лукьянов К.Р. БПИ2303
|
||||
- Целевая оценка: **4**; трек5 не заявлен.
|
||||
- Commit/PR: заполняется автором после самостоятельного commit/push.
|
||||
- Источники, библиотеки и помощь ИИ: FastAPI, Pydantic, Uvicorn, Psycopg, redis-py, boto3, Pillow, OpenTelemetry Python SDK1.34.1 (semantic conventions0.55b1), pytest. Документация [Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel metrics SDK](https://opentelemetry-python.readthedocs.io/en/stable/sdk/metrics.html), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker/Compose v2, Python3.9+, Make. Docker VM желательно выделить не менее8GiB RAM; ресурсы сервисов ограничены compose.yaml. Все зависимости поднимаются из репозитория. Остановить предыдущий стенд, сохранив volumes: используются те же localhost-порты.
|
||||
|
||||
```sh
|
||||
git clone <fork-url>
|
||||
cd <fork-lab6>
|
||||
make up
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
make check
|
||||
make validate-container
|
||||
make test
|
||||
make test-student
|
||||
make test-fresh
|
||||
make test-jobs
|
||||
make test-cluster
|
||||
make test-observability
|
||||
```
|
||||
|
||||
Основной endpoint: http://localhost:8080. Grafana: http://localhost:3000; пользователь operator, локальный пароль в secrets/grafana_password. Prometheus9090, Loki3100, Tempo3200 доступны только через127.0.0.1 для диагностики. Значения secrets не печатать. Dashboard и три datasource импортируются автоматически. make down сохраняет volumes, down --volumes намеренно удаляет данные текущей лабораторной.
|
||||
|
||||
Bootstrap создаёт локальные credentials; API/MinIO/mc собираются Dockerfile. MinIO/mc — закреплённые официальные исходники с commit guard, как в lab5; registry-образы/бинарный архив были недоступны. Python и транзитивные зависимости закреплены requirements.lock. Go на хосте не нужен. First build требует registry/GitHub/package repositories.
|
||||
|
||||
Отказные проверки запускать последовательно. test-fresh использует отдельный случайный project и удаляет его временные volumes; в этом отдельном тесте схемы SDK выключен, поскольку telemetry backend туда не поднимается. На основном стенде smoke/31 тест и эксперименты выполняются с telemetry on. Синтетические записи тестов остаются в общей БД.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Nginx → API×2 → PG / profile cache / MinIO staging + job/outbox
|
||||
relay → Streams broker → worker/processor → S3 variants + PG ready
|
||||
API / relay / worker / processor → OTLP HTTP → Collector
|
||||
├─ Prometheus: metrics
|
||||
├─ Loki: logs
|
||||
└─ Tempo: traces
|
||||
Grafana → все три источника
|
||||
```
|
||||
|
||||
Бизнес-API lab5 сохранён: 202queued, durable staging/outbox, thumbnail/crop/WebP, ACL, общий Bearer, идемпотентность, retry/failed/replay и cleanup. Nginx остаётся балансировщиком. Публичные contracts/tests/README/COURSE текущей лабы не изменялись. Служебные документы — lab6/docs.
|
||||
|
||||
Версии: Collector contrib0.128.0, Prometheus3.4.1, Loki3.5.1, Tempo2.7.2, Grafana12.0.1; PG17.5, Redis7.4.4, Nginx1.28.0. Backend volumes сохраняют историю (retention7d), это не HA и не backup. Collector — перенос/обработка, не долговременная БД; Grafana — интерфейс запросов.
|
||||
|
||||
**Трассы и контекст.** Server-span API берёт W3C context. Persist сохраняет carrier в PG job, relay создаёт producer-span и inject в payload, worker extract создаёт consumer-span. Retry связан с исходным upload и имеет job.outcome/attempt; retry/failed помечается ERROR. Дочерний processor получает carrier через stdin, экспортирует реальные S3 spans и завершает exporter перед выходом. postgres.query/commit измеряют фактические SQL/COMMIT, не аренду connection с изображениями/S3. Текст SQL, параметры, storage key и credentials не экспортируются.
|
||||
|
||||
**Логи.** JSON: timestamp, severity, service_name, event, trace_id/span_id и request_id для HTTP. Event attributes проходят allowlist. Body, Authorization, passwords/tokens, filenames и private keys исключены. X-Trace-Id — дополнительный диагностический заголовок, X-Request-Id сохраняется. API и worker логи найдены в Loki по trace_id.
|
||||
|
||||
**Ресурсы.** service.name=messenger-api/messenger-worker/messenger-relay, service.version=lab6-v1, случайный service.instance.id, deployment.environment.name=lab. Processor наследует worker instance, не создаёт отдельную серию metrics на каждую job. Processor экспортирует traces/logs, metrics обработки остаются у долгоживущего worker.
|
||||
|
||||
| Сигнал / имя Prometheus | Смысл / единицы |
|
||||
| --- | --- |
|
||||
| http_requests_total | количество попыток бизнес-запросов API; method/route template/status_class |
|
||||
| http_server_duration_seconds_bucket | HTTP duration, seconds; p50/p95/p99 из histogram buckets всех API |
|
||||
| jobs_processed_total | попытки обработки по job_type/outcome ready/retry/failed |
|
||||
| job_processing_duration_seconds_bucket | длительность попытки, seconds |
|
||||
| attachment_ready_seconds_bucket | accepted→ready, seconds |
|
||||
| queue_depth / queue_oldest_age_seconds | незавершённые queued/processing, count/seconds; собирает relay |
|
||||
| db_pool_used / db_pool_waiting | занятые соединения / ожидающие запросы |
|
||||
| worker_busy / worker_capacity | текущая занятость / ёмкость одного worker |
|
||||
| process_memory_bytes / process_cpu_seconds_total | RSS / накопленное процессорное время |
|
||||
|
||||
Технические UUID не являются labels. Неизвестный route — unmatched, UUID заменяются шаблонами. Histogram buckets0.005–120s, ready0.1–300s; units отображаются в dashboard. HTTP latency фиксируется до отправки тела response, а не до завершения полного скачивания клиентом. Queue age — backlog, не processing duration. Пример p95:
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m])))
|
||||
```
|
||||
|
||||
**SLI/SLO и alerts.** SLI=1−5xx/все бизнес-попытки API, health исключены; клиентские4xx не считаются серверным отказом. Повтор Nginx может образовать две попытки API — это SLI backend, не точный счётчик пользовательских действий. Цель>=99% за30d. Короткий учебный прогон не доказывает достижение30-дневного SLO. Alerts: error ratio>5% в окне1m for15s; oldest job>15s for15s. Правила и локальный runbook в infra/observability/alerts.yaml, доставка наружу не настраивалась.
|
||||
|
||||
**Ограничение telemetry.** SDK batch queues256, export batch32, задержка500ms, timeout1s; metrics interval2s. Collector memory_limiter192MiB/spike48MiB, container256MiB; batch128/max256, очереди exporter256 batches,2consumers, retry horizon20s. Неограниченного буфера нет. Prometheus exporter metric_expiration10s удаляет gauges ушедших replicas, не удерживая старый queue age5min. Backend outage может терять telemetry; запросы приложения не ждут его восстановления.
|
||||
|
||||
**Восстановление очереди.** Consumer reclaim неподтверждённой доставки —5s, независимо от PG processing lease90s. Живой процесс защищён advisory lock и общим deadline45s. Это устраняет задержку повторной доставки после краткого DB-сбоя, сохраняя защиту от параллельного эффекта.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Прогоны2026-11-09, Docker Desktop/macOS Apple Silicon. Ветка solution/bpi2303-lukyanov, коммитов/push агент не делает.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Проверка | Фактический результат / evidence |
|
||||
| --- | --- |
|
||||
| check/OpenAPI/public smoke | PASS: check.txt, openapi.txt, smoke.txt |
|
||||
| Собственные регрессии |31 тест PASS: student-tests.txt |
|
||||
| Пустые volumes/повтор миграций |31 тест PASS: fresh-migrations.txt |
|
||||
| Сохранность pipeline lab5 | jobs-faults.json: COMMIT/publish gap, broker restart, kill послеPUT, lease, retry/replay, duplicate PASS |
|
||||
| Restart/idempotency/failover/SIGTERM/rolling | соответствующие JSON в evidence |
|
||||
| DB/Redis отказ и ресурсы/порты | dependency-failures.json, infrastructure.json |
|
||||
| Автоматический dashboard и три datasource | grafana-provisioning.json, dashboardJSON в infra/observability |
|
||||
| Сквозная trace с retry и связанные API/worker logs | upload-trace.json, upload-logs.json, observability.json |
|
||||
| Queue/DB incidents, firing→resolved | queue-alert-*.json, db-alert-*.json |
|
||||
| Backend outage и bounded memory/queue | observability.json: сообщения успешны, экспортные counters/queue и dockerstats сохранены |
|
||||
|
||||
Предупреждения botocore utcnow и диагностический Pillow warning не являются падениями assertions. Code review выявил неправильный DB lease-span и отсутствие outcome/error у consumer; обе регрессии сначала упали, затем прошли после исправления.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
1. **Upload+retry.** Worker остановлен, файл принят; MinIO недоступен во время первой обработки. После восстановления тот же attachment ready, trace связывает server→producer→consumer→processor→S3/DB, включая несколько попыток. Сохранено 3 попыток; лог API и worker коррелирует с trace_id.
|
||||
2. **Worker stopped.** До воздействия alert разрешён; затем остановка worker, queued upload, queue age растёт, alert firing. После запуска ready и resolved. Queue metric продолжает работать через relay.
|
||||
3. **DB contention.** Операторская сессия держит ACCESS EXCLUSIVE lock на sessions, readiness остаётся живым. Запросы users/me ждут lock_timeout и возвращают503; postgres.query spans и db_operation_failed logs указывают на DB, не decoder. 9 ошибочных вызовов, затем holder прекращён через pg_terminate_backend, здоровый трафик и alert resolved. Helper отключает только собственные idle/statement timeouts и гарантированно завершается после опыта.
|
||||
4. **Loki+Tempo stopped.** За 26.571s успешно записано 2575 сообщений. Collector memory: 75.03MiB / 256MiB→76.7MiB / 256MiB при256MiB limit. Bounded retry/queues и реальные accepted/sent/failed/queue counters сохранены. Это короткий опыт, не доказательство нулевых потерь telemetry или отсутствия роста при бесконечной нагрузке.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Все15 dashboard expressions проверены реальными запросами Prometheus. Данные trace/log не статические fixtures. В ходе общего прогона после rolling один latency-check Redis fallback превысил3s; отдельная воспроизводящая проверка прошла без изменения контракта, затем dependency test повторён успешно. Кроме того, публичный smoke выявил90s pending-redelivery после DB outage: reclaim сокращён до5s, после чего повтор отказа и smoke прошли.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один host, backend на локальных volumes; нет HA/backup и обещания30-дневной доступности.
|
||||
- End-user response LB может отсутствовать в SDK при исключении всех API; это не считается подтверждённым API-observation. SLI определён для обработанных бизнес-попыток API.
|
||||
- Экспорт bounded best effort: при исчерпании очереди или retry horizon telemetry теряется. До устойчивого ingest/backend-volume гарантий сохранности span нет.
|
||||
- HTTP instrumentation измеряет handler/headers; download streaming не является полным клиентским latency.
|
||||
- HTTP и localhost админ-интерфейсы допустимы только в изолированном учебном окружении. TLS/mTLS и OpenBao вводятся в lab7.
|
||||
- Исследовательские sampling/overhead/burn-rate треки5 не выполнялись. Наблюдения относятся к заданному профилю и синтетическим данным.
|
||||
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
name: mtusi-lab6
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.5-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
redis:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
migrate:
|
||||
image: mtusi-lab6-api:lab6
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment:
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
api:
|
||||
image: mtusi-lab6-api:lab6
|
||||
build: .
|
||||
scale: 2
|
||||
environment:
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
POOL_MAX_SIZE: '8'
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [front, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio:
|
||||
image: mtusi-lab6-minio:2025-04-22
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: minio
|
||||
secrets: [minio_root_user, minio_root_password]
|
||||
volumes: [minio_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
stop_grace_period: 10s
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio-init:
|
||||
image: mtusi-lab6-mc:2025-04-16
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: mc
|
||||
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
|
||||
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
|
||||
networks: [data]
|
||||
depends_on:
|
||||
minio: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
broker:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction]
|
||||
volumes: [broker_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 160m
|
||||
cpus: 0.5
|
||||
worker: &background
|
||||
image: mtusi-lab6-api:lab6
|
||||
command: [python, -m, messenger.worker, worker]
|
||||
environment:
|
||||
OTEL_ENABLED: '1'
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4318
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
JOB_MAX_ATTEMPTS: '5'
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
broker: {condition: service_healthy}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
stop_grace_period: 45s
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
relay:
|
||||
<<: *background
|
||||
command: [python, -m, messenger.worker, relay]
|
||||
mem_limit: 192m
|
||||
cpus: 0.25
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
|
||||
networks: [front]
|
||||
ports: ['127.0.0.1:8080:8080']
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 1s
|
||||
timeout: 8s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.128.0
|
||||
command: [--config=/etc/otelcol/config.yaml]
|
||||
volumes: [./infra/observability/collector.yaml:/etc/otelcol/config.yaml:ro]
|
||||
networks: [data, observe]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.4.1
|
||||
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d]
|
||||
volumes:
|
||||
- ./infra/observability/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/observability/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports: ['127.0.0.1:9090:9090']
|
||||
networks: [observe, admin]
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
loki:
|
||||
image: grafana/loki:3.5.1
|
||||
command: [-config.file=/etc/loki/local.yaml]
|
||||
volumes: [./infra/observability/loki.yaml:/etc/loki/local.yaml:ro, loki_data:/loki]
|
||||
networks: [observe, admin]
|
||||
ports: ['127.0.0.1:3100:3100']
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: [-config.file=/etc/tempo/local.yaml]
|
||||
user: '0:0'
|
||||
volumes: [./infra/observability/tempo.yaml:/etc/tempo/local.yaml:ro, tempo_data:/var/tempo]
|
||||
networks: [observe, admin]
|
||||
ports: ['127.0.0.1:3200:3200']
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
grafana:
|
||||
image: grafana/grafana:12.0.1
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: operator
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||
secrets: [grafana_password]
|
||||
volumes:
|
||||
- ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports: ['127.0.0.1:3000:3000']
|
||||
networks: [observe, admin]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
admin:
|
||||
observe:
|
||||
internal: true
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
grafana_password: {file: ./secrets/grafana_password}
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
minio_root_user: {file: ./secrets/minio_root_user}
|
||||
minio_root_password: {file: ./secrets/minio_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
broker_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"parallel_requests": 8,
|
||||
"observed_instances": [
|
||||
"0432eff5e9d84ea99431573f134f5c23",
|
||||
"bfc39b720927442aa51a68b9fe837119"
|
||||
],
|
||||
"responses_201": 8,
|
||||
"messages_created": 1,
|
||||
"different_body_status": 409
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
[
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "HighApiErrorRate",
|
||||
"severity": "warning"
|
||||
},
|
||||
"annotations": {
|
||||
"runbook": "Check postgres.transaction error spans, DB pool and request logs. Restore DB and confirm resolved.",
|
||||
"summary": "Business API 5xx exceeds 5% in 1 minute"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-09T15:56:22.478517494Z",
|
||||
"value": "7.272727272727273e-01"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"redis": {
|
||||
"profile_status": 200,
|
||||
"fallback_seconds": 2.2293,
|
||||
"anonymous": 401,
|
||||
"readiness": 200
|
||||
},
|
||||
"postgres": {
|
||||
"authenticated_status": 503,
|
||||
"response_seconds": 0.0177,
|
||||
"anonymous_status": 401,
|
||||
"internal_api_liveness": [
|
||||
200,
|
||||
200
|
||||
],
|
||||
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
|
||||
},
|
||||
"same_session_after_recovery": true
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
./scripts/check_fresh_migrations.sh
|
||||
Network mtusi-lab6-migration-check-74280_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_redis_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_redis_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_postgres_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_postgres_data Creating
|
||||
Network mtusi-lab6-migration-check-74280_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_minio_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_minio_data Creating
|
||||
Volume mtusi-lab6-migration-check-74280_redis_data Created
|
||||
Volume mtusi-lab6-migration-check-74280_redis_data Created
|
||||
Volume mtusi-lab6-migration-check-74280_postgres_data Created
|
||||
Volume mtusi-lab6-migration-check-74280_postgres_data Created
|
||||
Volume mtusi-lab6-migration-check-74280_minio_data Created
|
||||
Volume mtusi-lab6-migration-check-74280_minio_data Created
|
||||
Network mtusi-lab6-migration-check-74280_data Created
|
||||
Network mtusi-lab6-migration-check-74280_data Created
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Creating
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Creating
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Creating
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Created
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Created
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Creating
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Created
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Created
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Starting
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Starting
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Starting
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Started
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Waiting
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Started
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Started
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Healthy
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Starting
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Started
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Waiting
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Waiting
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Waiting
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Waiting
|
||||
Container mtusi-lab6-migration-check-74280-redis-1 Healthy
|
||||
Container mtusi-lab6-migration-check-74280-minio-init-1 Healthy
|
||||
Container mtusi-lab6-migration-check-74280-minio-1 Healthy
|
||||
Container mtusi-lab6-migration-check-74280-postgres-1 Healthy
|
||||
Network mtusi-lab6-migration-check-74280_front Creating
|
||||
Network mtusi-lab6-migration-check-74280_front Creating
|
||||
Network mtusi-lab6-migration-check-74280_front Created
|
||||
Network mtusi-lab6-migration-check-74280_front Created
|
||||
Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Creating
|
||||
Container mtusi-lab6-migration-check-74280-api-run-5f6645bc6b07 Created
|
||||
Applied 001_initial.sql
|
||||
Applied 002_idempotency.sql
|
||||
Applied 003_attachments.sql
|
||||
Applied 004_jobs.sql
|
||||
Migrations up to date
|
||||
Migrations up to date
|
||||
............................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 19 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
student-tests/test_jobs.py: 6 warnings
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
31 passed, 27 warnings in 13.57s
|
||||
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"target_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"target_instance": "a535000080734d769e44ed18a64615ae",
|
||||
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
|
||||
"in_flight_observed_in_pg_stat_activity": true,
|
||||
"in_flight_response": 201,
|
||||
"new_tcp_after_sigterm": {
|
||||
"accepted": false,
|
||||
"error_type": "ConnectionRefusedError"
|
||||
},
|
||||
"target_still_running_when_new_tcp_rejected": true,
|
||||
"exit_code": 0,
|
||||
"shutdown_seconds_from_sigterm": 2.0642,
|
||||
"confirmed_message_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"dashboard_uid": "messenger-lab6",
|
||||
"panels": 15,
|
||||
"datasources": [
|
||||
{
|
||||
"name": "Loki",
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
{
|
||||
"name": "Prometheus",
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
{
|
||||
"name": "Tempo",
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "07e419b5121b83e36d525e3d218a4dfd4d5768b83531fe69ea2e459920931114",
|
||||
"name": "mtusi-lab6-api-3",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:53:46.48092646Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "23ac94c138224f05b2b9ebc8962a8f79",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "add61ef8a9c8a4f7e286f2b7b5ef8bc235905c843d2081ec14fc2ee2c1aee9ce",
|
||||
"name": "mtusi-lab6-api-4",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:53:46.408269418Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "28bb7891222a49a4809c409c0cd15976",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
|
||||
"name": "mtusi-lab6-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.284677889Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "collector",
|
||||
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
|
||||
"name": "mtusi-lab6-collector-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:20:09.734253509Z",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.128.0",
|
||||
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "grafana",
|
||||
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
|
||||
"name": "mtusi-lab6-grafana-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.50566143Z",
|
||||
"image": "grafana/grafana:12.0.1",
|
||||
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3000"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_grafana_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
|
||||
"name": "mtusi-lab6-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.236679055Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "loki",
|
||||
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
|
||||
"name": "mtusi-lab6-loki-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:41:58.345305841Z",
|
||||
"image": "grafana/loki:3.5.1",
|
||||
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3100/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3100"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_loki_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "3c47c49b694fda4217bb34ca4f9ea5c83c534deeb2ad8cfbd92a73e191a83959",
|
||||
"name": "mtusi-lab6-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:53:45.228435626Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
|
||||
"name": "mtusi-lab6-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.159907764Z",
|
||||
"image": "mtusi-lab6-minio:2025-04-22",
|
||||
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
|
||||
"name": "mtusi-lab6-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:53:45.325678751Z",
|
||||
"image": "mtusi-lab6-mc:2025-04-16",
|
||||
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
|
||||
"name": "mtusi-lab6-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:53:53.727434505Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "prometheus",
|
||||
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
|
||||
"name": "mtusi-lab6-prometheus-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.39409243Z",
|
||||
"image": "prom/prometheus:v3.4.1",
|
||||
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"9090/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "9090"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_prometheus_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
|
||||
"name": "mtusi-lab6-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:53:52.525647296Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "ed1f47e618ca83beab6abf06e3c288c51d101bfcc53e140b2295d9b6c1a3138e",
|
||||
"name": "mtusi-lab6-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:53:46.636963169Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "tempo",
|
||||
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
|
||||
"name": "mtusi-lab6-tempo-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:53:44.730313751Z",
|
||||
"image": "grafana/tempo:2.7.2",
|
||||
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3200"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_tempo_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "1eed3d38f1d69ce58034efdce9c7a1867baf0f88f52146b3dad12addd25adec1",
|
||||
"name": "mtusi-lab6-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:53:46.559703293Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:dd37275e2e85a08e0ed48f59da3c6ba1db5c4324b52a16ad3dba45d844e78806",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"commit_publish_gap": {
|
||||
"accepted_status": 202,
|
||||
"outbox_unpublished": true,
|
||||
"api_broker_restarted": true,
|
||||
"same_job_ready": true
|
||||
},
|
||||
"broker_persistence": {
|
||||
"unread_job_survived_restart": true
|
||||
},
|
||||
"after_put_before_ack": {
|
||||
"s3_effect_written": true,
|
||||
"lease_recovery_seconds": 89.096,
|
||||
"attempt_after_recovery": 2,
|
||||
"same_job": true
|
||||
},
|
||||
"duplicate": {
|
||||
"ready_noop": true
|
||||
},
|
||||
"retry_replay": {
|
||||
"failed_after": 5,
|
||||
"code": "retry_exhausted",
|
||||
"explicit_replay_ready": true,
|
||||
"failed_attachment_binding": 409
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"name": "mtusi-lab6-api-2",
|
||||
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"stopped_at_utc": "2026-11-09T15:45:58.712702+00:00",
|
||||
"stop_seconds": 0.5425,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"version": "nginx version: nginx/1.28.0",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"public_server_header": "nginx",
|
||||
"config_syntax": "PASS",
|
||||
"observed_instances": [
|
||||
"0432eff5e9d84ea99431573f134f5c23",
|
||||
"bfc39b720927442aa51a68b9fe837119"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"trace": {
|
||||
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe",
|
||||
"span_names": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"s3.put",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"attachments.publish",
|
||||
"attachments.consume",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"attachments.process_objects",
|
||||
"s3.get",
|
||||
"attachments.publish",
|
||||
"attachments.consume",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"postgres.query",
|
||||
"postgres.query",
|
||||
"postgres.commit",
|
||||
"attachments.process_objects",
|
||||
"s3.get"
|
||||
],
|
||||
"retry_attempts": 3,
|
||||
"api_worker_logs_correlated": true
|
||||
},
|
||||
"queue_incident": {
|
||||
"oldest_age": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794239772.021,
|
||||
"32.563177"
|
||||
]
|
||||
}
|
||||
],
|
||||
"depth": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794239772.023,
|
||||
"1"
|
||||
]
|
||||
}
|
||||
],
|
||||
"alert": "firing",
|
||||
"after_restore": "resolved"
|
||||
},
|
||||
"db_incident": {
|
||||
"failures": [
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.012639791006222,
|
||||
"trace_id": "2248eed7575b37ebc02bc118377be094"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.009124165982939,
|
||||
"trace_id": "e4a499d2482593d978976d4981e1be6c"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.010977207974065,
|
||||
"trace_id": "bfa70bd527ce23c595426c74bfee607e"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.038811917009298,
|
||||
"trace_id": "95bca29d2812c4cc930ac9fed93a1f7c"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.028251375013497,
|
||||
"trace_id": "b9e433fdd149f2922f0b3d78386e6d5a"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.018512499984354,
|
||||
"trace_id": "a11d9c704425e92132f9f5647d38bc1f"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.017416250018869,
|
||||
"trace_id": "0eb74ac09ce2aa9a36d676c2eafde20e"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.0125838330131955,
|
||||
"trace_id": "17f11595e2dc8fe7bbfb33e1bff4c637"
|
||||
},
|
||||
{
|
||||
"status": 503,
|
||||
"elapsed": 4.007467999996152,
|
||||
"trace_id": "ddfbc5519d3d9a746207a7d2c427364e"
|
||||
}
|
||||
],
|
||||
"before": "firing",
|
||||
"after": "resolved",
|
||||
"pool_waiting": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794239845.959,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"telemetry_outage": {
|
||||
"successful_messages": 2575,
|
||||
"seconds": 26.571,
|
||||
"collector_before": {
|
||||
"BlockIO": "0B / 0B",
|
||||
"CPUPerc": "1.58%",
|
||||
"Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
|
||||
"ID": "62ead4df7cf3",
|
||||
"MemPerc": "29.31%",
|
||||
"MemUsage": "75.03MiB / 256MiB",
|
||||
"Name": "mtusi-lab6-collector-1",
|
||||
"NetIO": "2.92MB / 1.2MB",
|
||||
"PIDs": "16"
|
||||
},
|
||||
"collector_after": {
|
||||
"BlockIO": "0B / 0B",
|
||||
"CPUPerc": "0.75%",
|
||||
"Container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
|
||||
"ID": "62ead4df7cf3",
|
||||
"MemPerc": "29.96%",
|
||||
"MemUsage": "76.7MiB / 256MiB",
|
||||
"Name": "mtusi-lab6-collector-1",
|
||||
"NetIO": "9MB / 1.56MB",
|
||||
"PIDs": "16"
|
||||
},
|
||||
"collector_metrics": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_queue_capacity",
|
||||
"data_type": "logs",
|
||||
"exporter": "otlphttp/loki",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_queue_capacity",
|
||||
"data_type": "traces",
|
||||
"exporter": "otlp/tempo",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"256"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_queue_size",
|
||||
"data_type": "logs",
|
||||
"exporter": "otlphttp/loki",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"23"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_queue_size",
|
||||
"data_type": "traces",
|
||||
"exporter": "otlp/tempo",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"202"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_send_failed_spans_total",
|
||||
"exporter": "otlp/tempo",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"264"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_exporter_sent_spans_total",
|
||||
"exporter": "otlp/tempo",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"5848"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "otelcol_receiver_accepted_spans_total",
|
||||
"instance": "collector:8888",
|
||||
"job": "collector",
|
||||
"otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper",
|
||||
"receiver": "otlp",
|
||||
"transport": "http"
|
||||
},
|
||||
"value": [
|
||||
1794239872.968,
|
||||
"32506"
|
||||
]
|
||||
}
|
||||
],
|
||||
"queue_limit_batches": 256,
|
||||
"collector_limit_mib": 256
|
||||
},
|
||||
"dashboard_queries": [
|
||||
{
|
||||
"panel": "Business RPS",
|
||||
"expression": "sum(rate(http_requests_total[1m]))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "5xx ratio",
|
||||
"expression": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "HTTP p50",
|
||||
"expression": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "HTTP p95",
|
||||
"expression": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "HTTP p99",
|
||||
"expression": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Jobs by outcome",
|
||||
"expression": "sum by(outcome)(rate(jobs_processed_total[1m]))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Job p95",
|
||||
"expression": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Accepted to ready p95",
|
||||
"expression": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Queue depth",
|
||||
"expression": "max(queue_depth)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Oldest unfinished job",
|
||||
"expression": "max(queue_oldest_age_seconds)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "DB pool used / waiting",
|
||||
"expression": "sum by(service_name)(db_pool_used)",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "DB pool waiting",
|
||||
"expression": "sum by(service_name)(db_pool_waiting)",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Worker busy / capacity",
|
||||
"expression": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Process RSS",
|
||||
"expression": "process_memory_bytes",
|
||||
"series": 4
|
||||
},
|
||||
{
|
||||
"panel": "Process CPU cores",
|
||||
"expression": "rate(process_cpu_seconds_total[1m])",
|
||||
"series": 4
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,680 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"name": "mtusi-lab6-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:43:08.631200429Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "0432eff5e9d84ea99431573f134f5c23",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"name": "mtusi-lab6-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:43:08.658375471Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "bfc39b720927442aa51a68b9fe837119",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
|
||||
"name": "mtusi-lab6-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:43:17.121717419Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "collector",
|
||||
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
|
||||
"name": "mtusi-lab6-collector-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:20:09.734253509Z",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.128.0",
|
||||
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "grafana",
|
||||
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
|
||||
"name": "mtusi-lab6-grafana-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.50566143Z",
|
||||
"image": "grafana/grafana:12.0.1",
|
||||
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3000"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_grafana_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
|
||||
"name": "mtusi-lab6-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:21:04.334408756Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "loki",
|
||||
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
|
||||
"name": "mtusi-lab6-loki-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:41:58.345305841Z",
|
||||
"image": "grafana/loki:3.5.1",
|
||||
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3100/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3100"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_loki_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9",
|
||||
"name": "mtusi-lab6-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:26:11.136784593Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
|
||||
"name": "mtusi-lab6-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:33:32.491039297Z",
|
||||
"image": "mtusi-lab6-minio:2025-04-22",
|
||||
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
|
||||
"name": "mtusi-lab6-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:26:11.058089009Z",
|
||||
"image": "mtusi-lab6-mc:2025-04-16",
|
||||
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
|
||||
"name": "mtusi-lab6-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:20:09.5754303Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "prometheus",
|
||||
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
|
||||
"name": "mtusi-lab6-prometheus-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.39409243Z",
|
||||
"image": "prom/prometheus:v3.4.1",
|
||||
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"9090/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "9090"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_prometheus_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
|
||||
"name": "mtusi-lab6-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:20:09.209044467Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49",
|
||||
"name": "mtusi-lab6-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:03.889331218Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "tempo",
|
||||
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
|
||||
"name": "mtusi-lab6-tempo-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:41:58.448806924Z",
|
||||
"image": "grafana/tempo:2.7.2",
|
||||
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3200"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_tempo_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c",
|
||||
"name": "mtusi-lab6-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:03.82336276Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"name": "mtusi-lab6-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.501501805Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "fcc51ba5c356403998a22d67f9b9be19",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"name": "mtusi-lab6-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.556862139Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
|
||||
"release": "lab6-v1"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "e0df69c5e459edb8a0e635a57674425f017aa3c0a61a8953ac0125d1a84113c6",
|
||||
"name": "mtusi-lab6-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.284677889Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "collector",
|
||||
"container": "62ead4df7cf3f43575854bf8d5f9b8853b25da9b3d39d6a7c8278e9d2544fbe3",
|
||||
"name": "mtusi-lab6-collector-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:20:09.734253509Z",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.128.0",
|
||||
"image_id": "sha256:1ab0baba0ee3695d823c46653d8a6e8894896e668ce8bd7ebe002e948d827bc7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "grafana",
|
||||
"container": "945105411fc4153c35af71d0305450956cb7254e3acfd0c3cb78b143d6c8971c",
|
||||
"name": "mtusi-lab6-grafana-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.50566143Z",
|
||||
"image": "grafana/grafana:12.0.1",
|
||||
"image_id": "sha256:06dc8d60e184705e5dc00e051a6d92342a44010d7d5e538d0a36339e85abb9b7",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3000"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_grafana_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
|
||||
"name": "mtusi-lab6-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.236679055Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "loki",
|
||||
"container": "89dabceedcb1940f75b5bb6f675be5200fe4029cba18b09eb436f46f0be71cb3",
|
||||
"name": "mtusi-lab6-loki-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:41:58.345305841Z",
|
||||
"image": "grafana/loki:3.5.1",
|
||||
"image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3100/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3100"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_loki_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5c11c076780bdae49862e42b23f8eedab776b26ae4d64c35b99366a6061e69f9",
|
||||
"name": "mtusi-lab6-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:13.574632917Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "fd8b3ff870f2c29b970c9ce6ea935aea8ec6ea0313301ca5852a507c1317354d",
|
||||
"name": "mtusi-lab6-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.159907764Z",
|
||||
"image": "mtusi-lab6-minio:2025-04-22",
|
||||
"image_id": "sha256:75e121abf175353cf9ca15afff1e46ba7c619f50ee59c5ea001c087970a56362",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "53ab65b441ea332946e7f5799e95210c2360051bb0ae55cd7dee8571ebf091a7",
|
||||
"name": "mtusi-lab6-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:13.645406292Z",
|
||||
"image": "mtusi-lab6-mc:2025-04-16",
|
||||
"image_id": "sha256:c265b33de56ecd2f8b1dfa0530f451224fe92ef0e9de633e41a1ded9d7f57e10",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "4565800d0bfe21bed96a1acbd34914a89282a6b40a4818ee278fbc85bffd02a7",
|
||||
"name": "mtusi-lab6-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.211149514Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "prometheus",
|
||||
"container": "b2bf1ff8abb55952217bedab8f75758d57436c428aca588cd863a2c8f19eadfb",
|
||||
"name": "mtusi-lab6-prometheus-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:23:49.39409243Z",
|
||||
"image": "prom/prometheus:v3.4.1",
|
||||
"image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"9090/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "9090"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_prometheus_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "7669c995c34318193e29e33592fa2f7ad9431ada63ffe904c44fa5e4667c1272",
|
||||
"name": "mtusi-lab6-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.321981847Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab6_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "c415a0a8ef936332f08f3ef4a5f0e0ba093093df5c96c1e2ee28f3d2e80c0e49",
|
||||
"name": "mtusi-lab6-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:11.757355583Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "tempo",
|
||||
"container": "7a1732267b593a8772764033a0f8b6e014732dd8f99bc6010817060f19dbbe4e",
|
||||
"name": "mtusi-lab6-tempo-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:41:58.448806924Z",
|
||||
"image": "grafana/tempo:2.7.2",
|
||||
"image_id": "sha256:4b0277a9b572a4b1fa43b01468e58911ec214dd69d033f2cd476596d14de9fe3",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"3200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "3200"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [
|
||||
"mtusi-lab6_tempo_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab6_admin",
|
||||
"mtusi-lab6_observe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "958e9185ae3e5991fe806ab171806b80d34084f98bf3ad21f69daa17da818e8c",
|
||||
"name": "mtusi-lab6-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:45:12.362175042Z",
|
||||
"image": "mtusi-lab6-api:lab6",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.107}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||
"stopped": {
|
||||
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"name": "mtusi-lab6-api-1",
|
||||
"instance": "fcc51ba5c356403998a22d67f9b9be19",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"stopped_at_utc": "2026-11-09T15:45:19.129171+00:00",
|
||||
"stop_seconds": 0.3545,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 1.107}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,15 @@
|
||||
[
|
||||
{
|
||||
"labels": {
|
||||
"alertname": "QueueLagGrowing",
|
||||
"severity": "warning"
|
||||
},
|
||||
"annotations": {
|
||||
"runbook": "Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains.",
|
||||
"summary": "Oldest unfinished attachment is older than 15 seconds"
|
||||
},
|
||||
"state": "firing",
|
||||
"activeAt": "2026-11-09T15:55:54.478517494Z",
|
||||
"value": "3.2563177e+01"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1,171 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"name": "mtusi-lab6-api-1",
|
||||
"instance": "f6c55a60b9d54fc49a966a29db53c7c9",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80"
|
||||
},
|
||||
{
|
||||
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"name": "mtusi-lab6-api-2",
|
||||
"instance": "201f4ef054b141e89207a80a27997663",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80"
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5",
|
||||
"name": "mtusi-lab6-api-3",
|
||||
"instance": "89984cccbfad40c8ae3896b5789dd706",
|
||||
"release": "lab6-68cfb17c",
|
||||
"image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f"
|
||||
},
|
||||
{
|
||||
"container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8",
|
||||
"name": "mtusi-lab6-api-4",
|
||||
"instance": "42711a4025dd43b5aa9c806b73e710ac",
|
||||
"release": "lab6-68cfb17c",
|
||||
"image_id": "sha256:4d61e78c9f57382d9cccae365bf2ecf81d771ee89fb0ea96c2e81eff0c46606f"
|
||||
}
|
||||
],
|
||||
"target_release": "lab6-68cfb17c",
|
||||
"lb_before": {
|
||||
"service": "lb",
|
||||
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
|
||||
"name": "mtusi-lab6-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.236679055Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_front"
|
||||
]
|
||||
},
|
||||
"lb_after": {
|
||||
"service": "lb",
|
||||
"container": "bcb788cc809228a4eae56d74e05dccc669cda07727de600cae2c5b70bdc74c4c",
|
||||
"name": "mtusi-lab6-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:45:11.236679055Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab6_front"
|
||||
]
|
||||
},
|
||||
"lb_not_restarted": true,
|
||||
"build_seconds": 23.6953,
|
||||
"migration_output": "Migrations up to date",
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"new_container": "455980c2c18beeb8dc9748cf91ef03f7ee50b040dfd1c8daa21354194723d9c5",
|
||||
"old_ip": "172.25.0.2",
|
||||
"new_ip": "172.25.0.5",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"89984cccbfad40c8ae3896b5789dd706",
|
||||
"f6c55a60b9d54fc49a966a29db53c7c9"
|
||||
],
|
||||
"observed_instances": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"89984cccbfad40c8ae3896b5789dd706"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.9622
|
||||
},
|
||||
{
|
||||
"old_container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"new_container": "b03c5a489cdcd77d35618bc60d982d91ceb871cefac32e1c97c178c00f9012b8",
|
||||
"old_ip": "172.25.0.3",
|
||||
"new_ip": "172.25.0.2",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"42711a4025dd43b5aa9c806b73e710ac",
|
||||
"89984cccbfad40c8ae3896b5789dd706"
|
||||
],
|
||||
"observed_instances": [
|
||||
"42711a4025dd43b5aa9c806b73e710ac",
|
||||
"89984cccbfad40c8ae3896b5789dd706"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 6.1869
|
||||
}
|
||||
],
|
||||
"during_replacement": {
|
||||
"seconds": 12.1492,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 6406,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 527.28,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.911,
|
||||
"p95": 11.297,
|
||||
"p99": 56.654
|
||||
},
|
||||
"instances": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"42711a4025dd43b5aa9c806b73e710ac",
|
||||
"89984cccbfad40c8ae3896b5789dd706",
|
||||
"f6c55a60b9d54fc49a966a29db53c7c9"
|
||||
]
|
||||
},
|
||||
"total": {
|
||||
"seconds": 42.993,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 24247,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 563.975,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.93,
|
||||
"p95": 9.719,
|
||||
"p99": 52.049
|
||||
},
|
||||
"instances": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"42711a4025dd43b5aa9c806b73e710ac",
|
||||
"89984cccbfad40c8ae3896b5789dd706",
|
||||
"f6c55a60b9d54fc49a966a29db53c7c9"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 24247,
|
||||
"history_messages": 24247,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"baseline": {
|
||||
"seconds": 3.0007,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1736,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 578.527,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.783,
|
||||
"p95": 9.151,
|
||||
"p99": 37.03
|
||||
},
|
||||
"instances": [
|
||||
"2af58291b4f34b53bcd4a8c81a6c4abb",
|
||||
"aee7f9540f7549b88fdbb8ee4290e6af"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"stopped": {
|
||||
"container": "035b9faf19ff90ee2aba7c1b9d7dfd0e4f7f8e969e291bcf9e8600afc35ada40",
|
||||
"name": "mtusi-lab6-api-1",
|
||||
"instance": "2af58291b4f34b53bcd4a8c81a6c4abb",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"stopped_at_utc": "2026-11-09T15:45:44.649698+00:00",
|
||||
"stop_seconds": 0.4897,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.005,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 2698,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 337.039,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 9.853,
|
||||
"p95": 23.34,
|
||||
"p99": 65.796
|
||||
},
|
||||
"instances": [
|
||||
"2af58291b4f34b53bcd4a8c81a6c4abb",
|
||||
"aee7f9540f7549b88fdbb8ee4290e6af"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.3292,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0295,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.3137,
|
||||
"seconds_from_stop_completion": 0.014
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
},
|
||||
{
|
||||
"stopped": {
|
||||
"container": "15135d556ff513409ae4c2a6e9145cd0cb42654845c7adc26869f66515b72d9b",
|
||||
"name": "mtusi-lab6-api-2",
|
||||
"instance": "aee7f9540f7549b88fdbb8ee4290e6af",
|
||||
"release": "lab6-v1",
|
||||
"image_id": "sha256:16bc5484f0c208749661846bac0d881605d657a71b3343714865ff465e05ca80",
|
||||
"stopped_at_utc": "2026-11-09T15:45:58.712702+00:00",
|
||||
"stop_seconds": 0.5425,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.005,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 2729,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 340.91,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 9.895,
|
||||
"p95": 22.682,
|
||||
"p99": 60.64
|
||||
},
|
||||
"instances": [
|
||||
"a535000080734d769e44ed18a64615ae",
|
||||
"aee7f9540f7549b88fdbb8ee4290e6af"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.1705,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0247,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.1615,
|
||||
"seconds_from_stop_completion": 0.0157
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 31.1691,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 12705,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 407.615,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 7.861,
|
||||
"p95": 21.097,
|
||||
"p99": 58.639
|
||||
},
|
||||
"instances": [
|
||||
"201f4ef054b141e89207a80a27997663",
|
||||
"2af58291b4f34b53bcd4a8c81a6c4abb",
|
||||
"a535000080734d769e44ed18a64615ae",
|
||||
"aee7f9540f7549b88fdbb8ee4290e6af"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 12705,
|
||||
"history_messages": 12705,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"each_original_container_tested": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,15 @@
|
||||
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
............................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 19 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
student-tests/test_jobs.py: 6 warnings
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
31 passed, 27 warnings in 13.44s
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"code_file_path": "",
|
||||
"code_function_name": "",
|
||||
"code_line_number": "0",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "INFO",
|
||||
"flags": "1",
|
||||
"observed_timestamp": "1794239727634612632",
|
||||
"scope_name": "messenger.events",
|
||||
"service_instance_id": "3d072627b1ab496c9308fc049bd79784",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "lab6-v1",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "be90e8be55e18b54",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.34.1",
|
||||
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1794239727634585856",
|
||||
"{\"event\":\"http_completed\",\"method\":\"POST\",\"request_id\":\"23dc38ec-dc6c-4592-9ba9-6744d664e150\",\"route\":\"/api/v1/chats/{chat_id}/attachments\",\"service_name\":\"messenger-api\",\"severity\":\"INFO\",\"span_id\":\"be90e8be55e18b54\",\"status_class\":\"2xx\",\"timestamp\":1794239727.6345801,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"code_file_path": "",
|
||||
"code_function_name": "",
|
||||
"code_line_number": "0",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "INFO",
|
||||
"flags": "1",
|
||||
"observed_timestamp": "1794239729394242216",
|
||||
"scope_name": "messenger.events",
|
||||
"service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "lab6-v1",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "f024a4b9cef096b9",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.34.1",
|
||||
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1794239729394201088",
|
||||
"{\"attempt\":1,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"f024a4b9cef096b9\",\"timestamp\":1794239729.3941958,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"code_file_path": "",
|
||||
"code_function_name": "",
|
||||
"code_line_number": "0",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "INFO",
|
||||
"flags": "1",
|
||||
"observed_timestamp": "1794239731438346217",
|
||||
"scope_name": "messenger.events",
|
||||
"service_instance_id": "1bb38b267fcc402dae8d85fb5e9d5198",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "lab6-v1",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "866c15dacbeadbc2",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.34.1",
|
||||
"trace_id": "69e61ec89e07ea9c4d93f81b188820fe"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1794239731438312704",
|
||||
"{\"attempt\":2,\"event\":\"job_attempt\",\"outcome\":\"retry\",\"service_name\":\"messenger-worker\",\"severity\":\"INFO\",\"span_id\":\"866c15dacbeadbc2\",\"timestamp\":1794239731.4383066,\"trace_id\":\"69e61ec89e07ea9c4d93f81b188820fe\"}"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 51852023,
|
||||
"linesProcessedPerSecond": 82276,
|
||||
"totalBytesProcessed": 24822333,
|
||||
"totalLinesProcessed": 39387,
|
||||
"execTime": 0.478715,
|
||||
"queueTime": 0.000184,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 3,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 3,
|
||||
"totalStructuredMetadataBytesProcessed": 11794082
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 15,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 3,
|
||||
"store": {
|
||||
"totalChunksRef": 2,
|
||||
"totalChunksDownloaded": 2,
|
||||
"chunksDownloadTime": 4766583,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 931797,
|
||||
"headChunkLines": 2188,
|
||||
"decompressedBytes": 23890536,
|
||||
"decompressedLines": 37199,
|
||||
"compressedBytes": 2915879,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 3,
|
||||
"headChunkStructuredMetadataBytes": 262392,
|
||||
"decompressedStructuredMetadataBytes": 11531690
|
||||
},
|
||||
"chunkRefsFetchTime": 385583,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0,
|
||||
"usedBloomFilters": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
# Official public images/binaries were removed upstream; build fixed release sources.
|
||||
FROM golang:1.24.2-bookworm AS minio-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
|
||||
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
|
||||
|
||||
FROM golang:1.24.2-bookworm AS mc-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
|
||||
|
||||
FROM python:3.13.5-slim AS minio
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
|
||||
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY infra/minio/server.sh /server.sh
|
||||
USER 10002
|
||||
ENTRYPOINT ["/bin/sh", "/server.sh"]
|
||||
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
|
||||
|
||||
FROM debian:bookworm-slim AS mc
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
],
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": [
|
||||
"originals/*",
|
||||
"staging/*",
|
||||
"variants/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments/originals/*",
|
||||
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||
"arn:aws:s3:::messenger-attachments/variants/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments >/dev/null
|
||||
mc anonymous set none local/messenger-attachments >/dev/null
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
|
||||
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
|
||||
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,62 @@
|
||||
worker_processes 1;
|
||||
error_log /dev/stderr warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
server_tokens off;
|
||||
limit_conn_zone $server_name zone=uploads:64k;
|
||||
|
||||
# Docker DNS; resolve updates the upstream when replicas get new IPs.
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
# All A records of api participate in the default round-robin balancing.
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name messenger;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
limit_conn uploads 16;
|
||||
client_max_body_size 11m;
|
||||
client_body_buffer_size 64k;
|
||||
client_body_timeout 10s;
|
||||
proxy_request_buffering on;
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
|
||||
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
|
||||
# must not be resent by Nginx after an ambiguous response.
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: HighApiErrorRate
|
||||
expr: sum(rate(http_requests_total{status_class="5xx"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001) > 0.05
|
||||
for: 15s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Business API 5xx exceeds 5% in 1 minute
|
||||
runbook: Check postgres.query error spans, DB pool and request logs. Restore DB and confirm resolved.
|
||||
- alert: QueueLagGrowing
|
||||
expr: max(queue_oldest_age_seconds) > 15
|
||||
for: 15s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: Oldest unfinished attachment is older than 15 seconds
|
||||
runbook: Check worker capacity/busy, job outcomes and consumer spans. Restore worker and confirm queue drains.
|
||||
@@ -0,0 +1,80 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
metric_expiration: 10s
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 20s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 20s
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
logs:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlphttp/loki
|
||||
traces:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp/tempo
|
||||
@@ -0,0 +1,436 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger \u2014 API and pipeline",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"lab6"
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Business RPS",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "5xx ratio",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_requests_total[1m])),0.001)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.5,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(http_server_duration_seconds_bucket[1m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Jobs by outcome",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(outcome)(rate(jobs_processed_total[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Job p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Accepted to ready p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_seconds_bucket[1m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(queue_depth)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Oldest unfinished job",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(queue_oldest_age_seconds)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool used / waiting",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name)(db_pool_used)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool waiting",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name)(db_pool_waiting)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Worker busy / capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Process RSS",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_memory_bytes",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU cores",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(process_cpu_seconds_total[1m])",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
access: proxy
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
url: http://loki:3100
|
||||
access: proxy
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
url: http://tempo:3200
|
||||
access: proxy
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
@@ -0,0 +1,28 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore: {store: inmemory}
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index: {prefix: index_, period: 24h}
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,9 @@
|
||||
global:
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 2s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs: [{targets: ['collector:9464']}]
|
||||
- job_name: collector
|
||||
static_configs: [{targets: ['collector:8888']}]
|
||||
@@ -0,0 +1,21 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
ingester:
|
||||
trace_idle_period: 2s
|
||||
max_block_duration: 10s
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal: {path: /var/tempo/wal}
|
||||
local: {path: /var/tempo/blocks}
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_history_idx ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Additive: old lab2 tables and messages remain valid.
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
|
||||
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position)
|
||||
);
|
||||
@@ -0,0 +1,19 @@
|
||||
ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1),
|
||||
attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0,
|
||||
lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text
|
||||
);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz
|
||||
);
|
||||
CREATE TABLE job_replays (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL
|
||||
);
|
||||
CREATE INDEX jobs_due ON attachment_jobs(next_at);
|
||||
@@ -0,0 +1,61 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.9.0
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
attrs==26.1.0
|
||||
boto3==1.38.27
|
||||
botocore==1.38.46
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
googleapis-common-protos==1.75.0
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httpx==0.28.1
|
||||
idna==3.20
|
||||
importlib_metadata==8.7.1
|
||||
iniconfig==2.3.1
|
||||
jmespath==1.1.0
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.13.0
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
opentelemetry-api==1.34.1
|
||||
opentelemetry-exporter-otlp-proto-common==1.34.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.34.1
|
||||
opentelemetry-proto==1.34.1
|
||||
opentelemetry-sdk==1.34.1
|
||||
opentelemetry-semantic-conventions==0.55b1
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pillow==11.3.0
|
||||
pluggy==1.7.0
|
||||
protobuf==5.29.6
|
||||
psycopg==3.2.9
|
||||
psycopg-binary==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
pycparser==3.12
|
||||
pydantic==2.11.5
|
||||
pydantic_core==2.33.2
|
||||
Pygments==2.21.0
|
||||
pytest==8.4.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyYAML==6.0.3
|
||||
redis==6.2.0
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
s3transfer==0.13.1
|
||||
six==1.17.0
|
||||
sniffio==1.3.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.3
|
||||
zipp==4.1.2
|
||||
@@ -0,0 +1,15 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.3
|
||||
psycopg[binary]==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.2.0
|
||||
argon2-cffi==25.1.0
|
||||
pytest==8.4.0
|
||||
httpx==0.28.1
|
||||
pydantic==2.11.5
|
||||
anyio==4.9.0
|
||||
boto3==1.38.27
|
||||
Pillow==11.3.0
|
||||
opentelemetry-api==1.34.1
|
||||
opentelemetry-sdk==1.34.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.34.1
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path('secrets')
|
||||
p = root / 'postgres_password'
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24))
|
||||
pw = p.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
|
||||
'grafana_password': secrets.token_hex(24),
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
'minio_root_user': 'lab4-admin',
|
||||
'minio_root_password': secrets.token_hex(24),
|
||||
's3_access_key_id': 'messenger-api',
|
||||
's3_secret_access_key': secrets.token_hex(24),
|
||||
}.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Bind-mounted Docker secrets must be readable by container uid 10001.
|
||||
# Parent directory remains 0700 on the host.
|
||||
for path in root.iterdir():
|
||||
path.chmod(0o444)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready (values hidden).')
|
||||
PY
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# A unique disposable Compose project; never touches normal project volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
project="mtusi-lab6-migration-check-$$"
|
||||
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
|
||||
echo 'Refusing to reuse a pre-existing scratch project' >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup() {
|
||||
docker compose -p "$project" down --volumes >/dev/null 2>&1
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
|
||||
docker compose -p "$project" run --rm --no-deps -e OTEL_ENABLED=0 api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
|
||||
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/cluster.py restart
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec python3 scripts/cluster.py stop-one
|
||||
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:', name)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
||||
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, concurrency=4):
|
||||
self.c = Client(timeout=6)
|
||||
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
||||
self.concurrency = concurrency
|
||||
self.done = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts = []
|
||||
self.confirmed = {}
|
||||
self.prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(self, number):
|
||||
client = Client(timeout=6)
|
||||
index = 0
|
||||
while not self.done.is_set():
|
||||
text = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
key = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.alice,self.chat,text,key=key)
|
||||
row = response['json']
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
||||
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
||||
with self.lock:
|
||||
self.confirmed[row['id']] = row
|
||||
index += 1
|
||||
except Failure:
|
||||
# Only idempotent message POST is retried, with its original body and key.
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
||||
time.sleep(0.02)
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
|
||||
def start(self):
|
||||
self.started = time.monotonic()
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
for future in self.futures:
|
||||
future.result(timeout=10)
|
||||
self.pool.shutdown()
|
||||
self.ended = time.monotonic()
|
||||
|
||||
def summary(self, start=None, end=None):
|
||||
start = self.started if start is None else start
|
||||
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
||||
with self.lock:
|
||||
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
||||
success = [a for a in attempts if a['ok']]
|
||||
duration = end-start
|
||||
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
||||
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
||||
'successful_rps':round(len(success)/duration,3),
|
||||
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
||||
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
||||
'instances':sorted({a['instance'] for a in success})}
|
||||
|
||||
def verify_data(self):
|
||||
rows = history(self.c,self.bob,self.chat)
|
||||
actual = {row['id']:row for row in rows}
|
||||
check(len(actual)==len(rows),'duplicated message ids')
|
||||
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
||||
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
||||
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
||||
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
||||
|
||||
|
||||
def concurrent_replay():
|
||||
cluster.wait_cluster()
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
def send(_):
|
||||
local = Client()
|
||||
barrier.wait()
|
||||
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send,range(8)))
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
||||
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
||||
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
||||
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
||||
'responses_201':8,'messages_created':1,'different_body_status':409})
|
||||
|
||||
|
||||
def verify_persistence():
|
||||
before = cluster.snapshot()
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
||||
after = cluster.snapshot()
|
||||
roles = ['api','postgres','redis','minio','broker','relay','worker','lb']
|
||||
previous = {item['container']:item for item in before}
|
||||
for item in after:
|
||||
if item['service'] in roles:
|
||||
old = previous[item['container']]
|
||||
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
||||
check(old['volumes']==item['volumes'],'volume identities changed')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
||||
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
||||
|
||||
|
||||
def public_failover():
|
||||
cluster.wait_cluster()
|
||||
try:
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
||||
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
||||
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
||||
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
||||
finally:
|
||||
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
||||
cluster.wait_cluster()
|
||||
|
||||
|
||||
def reject_new_connection(survivor, address):
|
||||
code = '''import json,socket,sys
|
||||
try:
|
||||
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
||||
except OSError as exc:
|
||||
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
||||
else:
|
||||
s.close()
|
||||
print(json.dumps({'accepted':True}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
||||
|
||||
|
||||
def failover_each():
|
||||
cluster.wait_cluster()
|
||||
originals = cluster.replicas()
|
||||
check(len(originals)==2,'two running replicas required')
|
||||
traffic = Traffic().start()
|
||||
phases = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
baseline_end = time.monotonic()
|
||||
baseline = traffic.summary(end=baseline_end)
|
||||
for original in originals:
|
||||
# Alias changes on restart, so resolve the current process of this exact container.
|
||||
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
||||
all_before = cluster.inspect([current['container']])[0]
|
||||
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
||||
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
||||
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
||||
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
||||
start = time.monotonic()
|
||||
try:
|
||||
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
||||
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
||||
stop_returned = time.monotonic()
|
||||
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
||||
read_deadline = start+10
|
||||
while True:
|
||||
try:
|
||||
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
||||
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
||||
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
||||
read_done = time.monotonic()
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
||||
time.sleep(0.1)
|
||||
check(read_done-start<=10,'read recovery exceeded10s')
|
||||
rejected = reject_new_connection(survivor,address)
|
||||
check(not rejected['accepted'],'new connection accepted by stopped API')
|
||||
time.sleep(max(0,8-(time.monotonic()-start)))
|
||||
end = time.monotonic()
|
||||
summary = traffic.summary(start,end)
|
||||
with traffic.lock:
|
||||
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
||||
recent = [a for a in successful if a['started']>stop_returned]
|
||||
recovery = min(a['ended'] for a in recent)-start if recent else None
|
||||
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
||||
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
||||
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
||||
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
||||
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
||||
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
||||
'seconds_from_action_start':round(read_done-start,4),
|
||||
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
||||
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
||||
finally:
|
||||
cluster.command(['docker','start',current['container']])
|
||||
cluster.wait_healthy(current['container'])
|
||||
cluster.wait_cluster()
|
||||
time.sleep(2)
|
||||
finally:
|
||||
traffic.stop()
|
||||
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
||||
|
||||
|
||||
def rolling_update():
|
||||
cluster.wait_cluster()
|
||||
before = cluster.replicas()
|
||||
check(len(before)==2,'rolling demonstration expects two replicas')
|
||||
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
target_release = 'lab6-'+uuid.uuid4().hex[:8]
|
||||
traffic = Traffic().start()
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
build_start = time.monotonic()
|
||||
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
||||
build_end = time.monotonic()
|
||||
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
||||
swap_start = time.monotonic()
|
||||
for old in before:
|
||||
step_start = time.monotonic()
|
||||
result = cluster.replace_one(old['container'])
|
||||
result['seconds'] = round(time.monotonic()-step_start,4)
|
||||
replacements.append(result)
|
||||
swap_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
finally:
|
||||
traffic.stop()
|
||||
after = cluster.replicas()
|
||||
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
||||
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
||||
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
||||
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
||||
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
||||
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
||||
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
||||
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
||||
|
||||
|
||||
def database_fixture(container, data):
|
||||
code = '''import json,sys,psycopg
|
||||
from psycopg import sql
|
||||
from messenger.db import setting
|
||||
p=json.load(sys.stdin)
|
||||
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
||||
if p['action']=='install':
|
||||
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
||||
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif p['action']=='remove':
|
||||
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
||||
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
else:
|
||||
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(response.stdout)
|
||||
|
||||
|
||||
def graceful_shutdown():
|
||||
cluster.wait_cluster()
|
||||
nodes = cluster.replicas()
|
||||
target,survivor = nodes[0],nodes[1]
|
||||
address = cluster.front_address(target['container'])
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
marker = 'drain_'+uuid.uuid4().hex
|
||||
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
||||
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
||||
'''
|
||||
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
||||
def in_flight():
|
||||
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(result.stdout)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(in_flight)
|
||||
try:
|
||||
deadline = time.monotonic()+3
|
||||
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
||||
time.sleep(0.05)
|
||||
signaled = time.monotonic()
|
||||
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
||||
deadline = signaled+1.5
|
||||
while True:
|
||||
rejected = reject_new_connection(survivor['container'],address)
|
||||
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
||||
if not rejected['accepted']:
|
||||
break
|
||||
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
||||
time.sleep(0.05)
|
||||
check(still_running,'connection refusal observed only after process exit, not during drain')
|
||||
drained = future.result(timeout=8)
|
||||
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
||||
check(drained['message']['text']==marker,'in-flight body changed')
|
||||
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
||||
elapsed = time.monotonic()-signaled
|
||||
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
||||
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
||||
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
||||
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
||||
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
||||
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
||||
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
||||
'confirmed_message_preserved':True}
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('graceful-shutdown.json',result)
|
||||
|
||||
|
||||
def infrastructure():
|
||||
state = cluster.snapshot()
|
||||
for item in state:
|
||||
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
||||
admin_ports={'grafana':'3000','prometheus':'9090','loki':'3100','tempo':'3200'}
|
||||
if item['service'] in admin_ports:
|
||||
for bindings in (item['published_ports'] or {}).values():
|
||||
check(all(b['HostIp']=='127.0.0.1' and b['HostPort']==admin_ports[item['service']] for b in bindings),'admin port is not localhost-only')
|
||||
elif item['service']!='lb':
|
||||
check(not item['published_ports'],'service exposed a port bypassing LB')
|
||||
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
||||
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
||||
save('infrastructure.json',{'result':'PASS','containers':state})
|
||||
|
||||
|
||||
def dependency_failures():
|
||||
c = Client(timeout=6)
|
||||
actor = c.user('dependencies')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result = {'result':'PASS'}
|
||||
try:
|
||||
cluster.compose('stop','-t','2','redis')
|
||||
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
||||
finally:
|
||||
cluster.compose('start','redis')
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
def status(path, token=None):
|
||||
headers = {'Authorization':'Bearer '+token} if token else {}
|
||||
request = urllib.request.Request(c.base+path,headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = urllib.request.urlopen(request,timeout=6)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.code, round(time.monotonic()-started,4)
|
||||
try:
|
||||
cluster.compose('stop','-t','2','postgres')
|
||||
code,elapsed = status('/api/v1/users/me',actor['token'])
|
||||
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
||||
anonymous,_ = status('/api/v1/users/me')
|
||||
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
||||
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
||||
check(all(code==200 for code in live),'API process liveness depends on DB')
|
||||
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
||||
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
||||
finally:
|
||||
cluster.compose('start','postgres')
|
||||
cluster.wait_cluster()
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result['same_session_after_recovery'] = True
|
||||
save('dependency-failures.json',result)
|
||||
|
||||
|
||||
def verify_nginx():
|
||||
state = cluster.snapshot()
|
||||
lb = next(item for item in state if item['service']=='lb')
|
||||
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
||||
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
||||
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
||||
c = Client()
|
||||
response = c.call('GET','/health/ready',schema='Health')
|
||||
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
||||
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
||||
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
||||
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
||||
args = parser.parse_args()
|
||||
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
||||
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit real fault injection; emits IDs/results, never session tokens."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
|
||||
|
||||
def compose(*args):
|
||||
if args and args[0]=='start': args=('up','-d','--no-deps',*args[1:])
|
||||
result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120)
|
||||
if result.returncode: raise RuntimeError(result.stderr)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def inside(body,**values):
|
||||
script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()'
|
||||
proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120)
|
||||
if proc.returncode: raise RuntimeError(proc.stderr)
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def state(aid):
|
||||
return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid)
|
||||
|
||||
|
||||
def wait_for(predicate,seconds=30):
|
||||
end=time.monotonic()+seconds
|
||||
while time.monotonic()<end:
|
||||
result=predicate()
|
||||
if result: return result
|
||||
time.sleep(.2)
|
||||
raise AssertionError('Condition timed out')
|
||||
|
||||
|
||||
def ready(c,actor,att):
|
||||
return c.wait_ready(actor,att)
|
||||
|
||||
|
||||
def main():
|
||||
c=Client();alice,bob,_,chat=c.bootstrap()
|
||||
evidence={'result':'INCOMPLETE'}
|
||||
trigger=False
|
||||
try:
|
||||
# Durable acceptance while both relay and worker are absent: real COMMIT/publish gap.
|
||||
compose('stop','worker','relay')
|
||||
att,raw=c.attachment(alice,chat)
|
||||
c.message(alice,chat,'',attachments=[att['id']])
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
|
||||
before=state(att['id']);check(before['dispatched_at'] is None,'outbox already published')
|
||||
compose('restart','api','broker')
|
||||
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0)
|
||||
c.call('GET','/api/v1/users/me',actor=alice)
|
||||
compose('start','relay','worker')
|
||||
meta=ready(c,bob,att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original changed')
|
||||
evidence['commit_publish_gap']={'accepted_status':202,'outbox_unpublished':True,'api_broker_restarted':True,'same_job_ready':state(att['id'])['job_id']==before['job_id']}
|
||||
# Publisher has already confirmed acceptance, broker restart preserves unread work.
|
||||
compose('stop','worker')
|
||||
att2,_=c.attachment(alice,chat)
|
||||
wait_for(lambda: state(att2['id'])['dispatched_at'])
|
||||
compose('restart','broker')
|
||||
compose('start','worker')
|
||||
ready(c,bob,att2)
|
||||
evidence['broker_persistence']={'unread_job_survived_restart':True}
|
||||
# Delay ready COMMIT in real PostgreSQL AFTER all S3 PUTs, kill before ack.
|
||||
compose('stop','worker')
|
||||
target,_=c.attachment(alice,chat)
|
||||
inside("with db.pool.connection() as c:\n c.execute(\"CREATE OR REPLACE FUNCTION lab5_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='\"+v['aid']+\"'::uuid AND NEW.status='ready' THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$\")\n c.execute('CREATE TRIGGER lab5_pause BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_pause_ready()')\nresult=True",aid=target['id'])
|
||||
trigger=True
|
||||
compose('start','worker')
|
||||
def at_commit():
|
||||
return inside("with db.pool.connection() as c:\n result=bool(c.execute(\"SELECT 1 FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'UPDATE attachments SET status=%'\").fetchone())")
|
||||
wait_for(at_commit)
|
||||
observed=inside("with db.pool.connection() as c:\n row=c.execute('SELECT object_key FROM attachments WHERE id=%s',(v['aid'],)).fetchone()\nobj=store.get(row['object_key']);result={'s3_effect_written':obj.length>0};obj.close()",aid=target['id'])
|
||||
compose('kill','-s','SIGKILL','worker')
|
||||
# unless-stopped can restart a killed worker; stop it to retain the fault boundary.
|
||||
compose('stop','worker')
|
||||
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True")
|
||||
trigger=False
|
||||
crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt')
|
||||
print('Waiting for real 90s lease expiry after killed worker.',flush=True)
|
||||
started=time.monotonic()
|
||||
wait_for(lambda: state(target['id'])['status']=='queued',110)
|
||||
compose('start','worker')
|
||||
ready(c,bob,target)
|
||||
after=state(target['id'])
|
||||
evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']}
|
||||
# Re-deliver the completed original job. Ready metadata is immutable.
|
||||
original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']
|
||||
duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id'])
|
||||
check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result')
|
||||
evidence['duplicate']={'ready_noop':True}
|
||||
# Retry exhaustion with real missing staging (transient storage failure), explicit replay.
|
||||
compose('stop','worker','relay')
|
||||
failed,failed_raw=c.attachment(alice,chat)
|
||||
st=state(failed['id'])
|
||||
inside("store.delete(v['key']);result=True",key=st['staging_key'])
|
||||
for attempt in range(5):
|
||||
inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id'])
|
||||
failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json']
|
||||
check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id'])
|
||||
compose('start','relay','worker')
|
||||
ready(c,bob,failed)
|
||||
evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409}
|
||||
evidence['result']='PASS'
|
||||
(ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n')
|
||||
print('PASS: jobs-faults.json')
|
||||
finally:
|
||||
if trigger:
|
||||
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True")
|
||||
compose('start','relay','worker')
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Query real backends and reproduce bounded incidents; no secret values saved."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
from verify_jobs import compose,inside,wait_for,state
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(urllib.request.Request(url,headers={'Accept':'application/json'}),timeout=8) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def query(expression):
|
||||
return get('http://localhost:9090/api/v1/query?'+urllib.parse.urlencode({'query':expression}))['data']['result']
|
||||
|
||||
|
||||
def alerts():
|
||||
return get('http://localhost:9090/api/v1/alerts')['data']['alerts']
|
||||
|
||||
|
||||
def alert_state(name,state):
|
||||
rows=[a for a in alerts() if a['labels']['alertname']==name]
|
||||
return any(a['state']==state for a in rows) if state=='firing' else not rows
|
||||
|
||||
|
||||
def collect_names(trace):
|
||||
names=[]
|
||||
def walk(value):
|
||||
if isinstance(value,dict):
|
||||
if 'spanId' in value and 'name' in value: names.append(value['name'])
|
||||
for item in value.values(): walk(item)
|
||||
elif isinstance(value,list):
|
||||
for item in value: walk(item)
|
||||
walk(trace)
|
||||
return names
|
||||
|
||||
|
||||
def save(name,data):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(data,indent=2)+'\n')
|
||||
|
||||
|
||||
def main():
|
||||
c=Client();alice,bob,_,chat=c.bootstrap()
|
||||
report={'result':'INCOMPLETE'}
|
||||
lock_process=None
|
||||
try:
|
||||
# Controlled retry in the same causal chain.
|
||||
compose('stop','worker')
|
||||
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=trace.png',202,
|
||||
actor=alice,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
att=response['json'];trace_id=response['headers']['X-Trace-Id']
|
||||
compose('stop','minio');compose('start','worker')
|
||||
wait_for(lambda: (lambda s:s['attempt']>=1 and s['status']=='queued')(state(att['id'])),30)
|
||||
time.sleep(2)
|
||||
compose('start','minio')
|
||||
# API re-enters LB after readiness recovers.
|
||||
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0,60)
|
||||
ready=c.wait_ready(bob,att)
|
||||
def chain():
|
||||
try:
|
||||
trace=get('http://localhost:3200/api/traces/'+trace_id)
|
||||
names=collect_names(trace)
|
||||
return trace if names.count('attachments.consume')>=2 and all(n in names for n in ['attachments.publish','attachments.process_objects','s3.put','postgres.query']) else None
|
||||
except Exception: return None
|
||||
trace=wait_for(chain,60)
|
||||
save('upload-trace.json',trace)
|
||||
logquery='{service_name=~"messenger-.*"} |= "'+trace_id+'"'
|
||||
def correlated_logs():
|
||||
try:
|
||||
result=get('http://localhost:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':logquery,'limit':100,'start':str(time.time_ns()-600_000_000_000)}))
|
||||
streams=result['data']['result']
|
||||
services={item['stream'].get('service_name') for item in streams}
|
||||
return result if {'messenger-api','messenger-worker'}<=services else None
|
||||
except Exception: return None
|
||||
|
||||
logs=wait_for(correlated_logs,60);save('upload-logs.json',logs)
|
||||
report['trace']={'trace_id':trace_id,'span_names':collect_names(trace),'retry_attempts':state(att['id'])['attempt'],'api_worker_logs_correlated':True}
|
||||
# Incident 1: worker stopped, queued age is measured by relay.
|
||||
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),40)
|
||||
compose('stop','worker')
|
||||
queued,_=c.attachment(alice,chat)
|
||||
wait_for(lambda: alert_state('QueueLagGrowing','firing'),70)
|
||||
save('queue-alert-firing.json',alerts())
|
||||
report['queue_incident']={'oldest_age':query('max(queue_oldest_age_seconds)'),'depth':query('max(queue_depth)'),'alert':'firing'}
|
||||
compose('start','worker');c.wait_ready(bob,queued)
|
||||
wait_for(lambda: alert_state('QueueLagGrowing','resolved'),30)
|
||||
save('queue-alert-resolved.json',alerts());report['queue_incident']['after_restore']='resolved'
|
||||
# Incident 2: sessions table lock affects business auth, readiness remains healthy.
|
||||
code='import os\nfrom messenger.db import Database\nd=Database();d.open();c=d.pool.getconn()\nc.execute(\'SET statement_timeout=0\')\nc.execute(\'SET idle_in_transaction_session_timeout=0\')\nc.execute("SET application_name=\'lab6-fault-lock\'")\nc.commit()\nc.execute(\'LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE\')\nprint(\'locked\',flush=True)\ntry:\n c.execute(\'SELECT pg_sleep(90)\')\nfinally:\n os._exit(0)\n'
|
||||
lock_process=subprocess.Popen(['docker','compose','exec','-T','api','python','-c',code],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
||||
check(lock_process.stdout.readline().strip()=='locked','DB lock was not acquired')
|
||||
failures=[]
|
||||
started=time.monotonic()
|
||||
while time.monotonic()-started<35:
|
||||
r=c.call('GET','/api/v1/users/me',503,actor=alice)
|
||||
failures.append({'status':r['status'],'elapsed':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id')})
|
||||
wait_for(lambda: alert_state('HighApiErrorRate','firing'),40)
|
||||
save('db-alert-firing.json',alerts())
|
||||
# Terminate exact lock holder via PG primary, not the API container.
|
||||
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
|
||||
lock_process.wait(timeout=10);lock_process=None
|
||||
for _ in range(300): c.call('GET','/api/v1/users/me',actor=alice)
|
||||
wait_for(lambda: alert_state('HighApiErrorRate','resolved'),75)
|
||||
save('db-alert-resolved.json',alerts())
|
||||
report['db_incident']={'failures':failures,'before':'firing','after':'resolved','pool_waiting':query('sum(db_pool_waiting)')}
|
||||
# Backend outage must not affect accepted messages or grow queues beyond configuration.
|
||||
before=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
|
||||
compose('stop','tempo','loki')
|
||||
count=0;start=time.monotonic()
|
||||
while time.monotonic()-start<25:
|
||||
c.message(alice,chat,'telemetry outage '+str(count),key=uuid.uuid4().hex);count+=1
|
||||
if count%20==0: time.sleep(.1)
|
||||
during=query('{__name__=~"otelcol_(receiver_accepted_spans|exporter_sent_spans|exporter_send_failed_spans|exporter_queue_size|exporter_queue_capacity|exporter_enqueue_failed_spans).*"}')
|
||||
after=json.loads(compose('stats','--no-stream','--format','{{json .}}','collector').strip())
|
||||
compose('start','tempo','loki')
|
||||
report['telemetry_outage']={'successful_messages':count,'seconds':round(time.monotonic()-start,3),'collector_before':before,'collector_after':after,'collector_metrics':during,'queue_limit_batches':256,'collector_limit_mib':256}
|
||||
# Every provisioned dashboard expression must return a successful Prometheus response.
|
||||
dashboard=json.loads((ROOT/'infra/observability/grafana/dashboards/messenger.json').read_text())
|
||||
expressions=[]
|
||||
for panel in dashboard['panels']:
|
||||
for target in panel['targets']:
|
||||
values=query(target['expr']);check(values,'No data for '+panel['title'])
|
||||
expressions.append({'panel':panel['title'],'expression':target['expr'],'series':len(values)})
|
||||
report['dashboard_queries']=expressions
|
||||
report['result']='PASS';save('observability.json',report)
|
||||
print('PASS: observability.json, traces/logs and firing/resolved alerts',flush=True)
|
||||
finally:
|
||||
if lock_process is not None:
|
||||
inside("with db.pool.connection() as c:\n rows=c.execute(\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE (application_name='lab6-fault-lock' OR query='SELECT pg_sleep(90)') AND pid<>pg_backend_pid()\").fetchall()\nresult={'terminated':len(rows)}")
|
||||
compose('start','worker','minio','tempo','loki')
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from . import telemetry
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from redis.exceptions import RedisError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .db import Database
|
||||
from .errors import ApiError, invalid, not_found, unauthorized
|
||||
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
|
||||
from .messages import store_message, validate_key, MESSAGE_FIELDS
|
||||
from .object_store import ObjectStore, unavailable
|
||||
from . import attachments
|
||||
from urllib.parse import quote
|
||||
from .pagination import decode_cursor, page, page_limit, public, stamp
|
||||
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
USER_FIELDS = 'id, username, display_name, created_at'
|
||||
|
||||
|
||||
def error_response(request, error):
|
||||
return JSONResponse({'error': {'code': error.code, 'message': error.message,
|
||||
'request_id': request.state.request_id}},
|
||||
status_code=error.status, headers=error.headers)
|
||||
|
||||
|
||||
def authenticate(request: Request):
|
||||
token = bearer(request.headers.get('Authorization', ''))
|
||||
token_hash = digest(token)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
|
||||
(token_hash,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
request.state.token_hash = token_hash
|
||||
return row['user_id']
|
||||
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
|
||||
|
||||
def chat_public(conn, row):
|
||||
item = public(row)
|
||||
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
|
||||
return item
|
||||
|
||||
|
||||
def create_app():
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.db = Database()
|
||||
app.state.storage = ObjectStore()
|
||||
app.state.upload_slots = asyncio.Semaphore(6)
|
||||
app.state.db.open()
|
||||
try:
|
||||
telemetry.init("messenger-api",app.state.db)
|
||||
yield
|
||||
finally:
|
||||
app.state.storage.close()
|
||||
app.state.db.close()
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.state.release = os.environ.get('APP_RELEASE', 'lab6-v1')
|
||||
|
||||
async def request_context(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
path = request.url.path
|
||||
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
|
||||
if request.method == 'POST' and json_endpoint:
|
||||
media = request.headers.get('Content-Type', '').split(';')
|
||||
unsupported = media[0].strip().lower() != 'application/json'
|
||||
for parameter in media[1:]:
|
||||
name, _, value = parameter.strip().partition('=')
|
||||
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
|
||||
unsupported = True
|
||||
if unsupported:
|
||||
if path != '/api/v1/users':
|
||||
await run_in_threadpool(authenticate, request)
|
||||
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
|
||||
try:
|
||||
json.loads((await request.body()).decode('utf-8'))
|
||||
except (UnicodeError, json.JSONDecodeError):
|
||||
raise invalid() from None
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
|
||||
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
|
||||
except Exception as exc:
|
||||
# Log exception class only: credentials, SQL and message content stay out of logs.
|
||||
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
|
||||
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = app.state.instance_id
|
||||
response.headers['X-App-Release'] = app.state.release
|
||||
return response
|
||||
|
||||
@app.middleware('http')
|
||||
async def traced_request(request: Request,call_next):
|
||||
route=telemetry.route_name(request.url.path)
|
||||
started=time.monotonic()
|
||||
with telemetry.span(request.method+' '+route,kind=telemetry.SpanKind.SERVER,
|
||||
context=telemetry.propagate.extract(dict(request.headers)),
|
||||
attributes={'http.request.method':request.method,'http.route':route}) as active:
|
||||
response=await request_context(request,call_next)
|
||||
active.set_attribute('http.response.status_code',response.status_code)
|
||||
if response.status_code>=500: active.set_status(telemetry.StatusCode.ERROR)
|
||||
telemetry.http_done(request.method,route,response.status_code,time.monotonic()-started,request.state.request_id)
|
||||
response.headers['X-Trace-Id']=format(active.get_span_context().trace_id,'032x')
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, exc):
|
||||
return error_response(request, exc)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error_response(request, invalid())
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error_response(request, not_found() if exc.status_code == 404 else invalid())
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready(request: Request):
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
|
||||
conn.execute('SELECT status,variants FROM attachments LIMIT 0')
|
||||
conn.execute('SELECT 1 FROM job_outbox LIMIT 0')
|
||||
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
|
||||
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
|
||||
request.app.state.storage.check()
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
def register(body: CreateUser, request: Request):
|
||||
password_hash = HASHER.hash(body.password)
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
|
||||
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
return public(row)
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def create_session(request: Request):
|
||||
# Run hashing and synchronous database calls outside the event loop.
|
||||
username, password = basic_credentials(request.headers.get('Authorization', ''))
|
||||
if await request.body():
|
||||
raise invalid()
|
||||
return await run_in_threadpool(login, request, username, password)
|
||||
|
||||
def login(request, username, password):
|
||||
db = request.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
|
||||
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
|
||||
if not row or not valid:
|
||||
raise unauthorized(basic=True)
|
||||
row.pop('password_hash')
|
||||
token, expires_at = new_session(db.session_ttl)
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
|
||||
(digest(token), row['id'], expires_at))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def me(request: Request, user_id=Depends(authenticate)):
|
||||
db = request.app.state.db
|
||||
key = 'user:' + str(user_id)
|
||||
try:
|
||||
cached = db.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (RedisError, ValueError):
|
||||
pass
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
result = public(row)
|
||||
try:
|
||||
db.cache.set(key, json.dumps(result), ex=60)
|
||||
except RedisError:
|
||||
pass
|
||||
return result
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def logout(request: Request, user_id=Depends(authenticate)):
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
|
||||
if str(user_id) not in body.member_ids:
|
||||
raise invalid()
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
|
||||
(uuid4(), body.title)).fetchone()
|
||||
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
|
||||
[(row['id'], uuid_value(member)) for member in body.member_ids])
|
||||
result = chat_public(conn, row)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats(request: Request, user_id=Depends(authenticate)):
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'chats:' + str(user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
|
||||
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
|
||||
result = page(scope, rows, limit)
|
||||
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
|
||||
return chat_public(conn, row)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
key = validate_key(request.headers.get('Idempotency-Key', ''))
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
|
||||
return public(row)
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'messages:' + str(cid)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
|
||||
params = (cid, *after, limit+1) if after else (cid, limit+1)
|
||||
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
|
||||
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
|
||||
return page(scope, rows, limit)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
|
||||
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
|
||||
cid=uuid_value(chat_id)
|
||||
def access():
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn,cid,user_id)
|
||||
await run_in_threadpool(access)
|
||||
return await attachments.upload(request,user_id,cid,filename)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
return attachments.metadata(row)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
if variant not in ('original','thumbnail','crop','optimized'):
|
||||
raise invalid()
|
||||
if row['status']!='ready':
|
||||
raise ApiError(409,'conflict','Вложение ещё не готово')
|
||||
item=row
|
||||
key=row['object_key']
|
||||
if variant!='original':
|
||||
item=next((v for v in row['variants'] if v['name']==variant),None)
|
||||
if item is None:
|
||||
raise not_found()
|
||||
key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant
|
||||
opened=request.app.state.storage.get(key)
|
||||
if opened.length!=item['size_bytes']:
|
||||
opened.close()
|
||||
raise unavailable()
|
||||
return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={
|
||||
'Content-Length':str(item['size_bytes']),
|
||||
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Bounded binary upload, image validation and DB/S3 coordination."""
|
||||
import asyncio
|
||||
from . import telemetry
|
||||
import hashlib
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
import warnings
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .pagination import public
|
||||
|
||||
MAX_BYTES=10485760
|
||||
MAX_PIXELS=40000000
|
||||
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
|
||||
PngImagePlugin.MAX_TEXT_CHUNK=1048576
|
||||
PngImagePlugin.MAX_TEXT_MEMORY=1048576
|
||||
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
|
||||
DECODER=threading.BoundedSemaphore(1)
|
||||
LOG=logging.getLogger('messenger')
|
||||
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants'
|
||||
|
||||
|
||||
def object_lock(conn,key,wait=True):
|
||||
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
||||
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
|
||||
return True if wait else row['locked']
|
||||
|
||||
|
||||
def filename_value(value):
|
||||
if not value or not value.strip() or len(value)>255:
|
||||
raise invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise invalid() from None
|
||||
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def metadata(row):
|
||||
result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')})
|
||||
return result
|
||||
|
||||
|
||||
def validate_image(file,mime):
|
||||
if not DECODER.acquire(timeout=3):
|
||||
raise ApiError(503,'unavailable','Обработка изображений занята')
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
|
||||
if actual is None or actual!=mime:
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
|
||||
if image.width*image.height>MAX_PIXELS:
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
|
||||
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
|
||||
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
|
||||
finally:
|
||||
DECODER.release()
|
||||
|
||||
|
||||
def compensate(db,store,key):
|
||||
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,key,wait=False):
|
||||
return
|
||||
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone()
|
||||
if not exists:
|
||||
store.delete(key)
|
||||
except Exception as exc:
|
||||
# A periodic, grace-protected janitor will retry after dependency recovery.
|
||||
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
|
||||
|
||||
|
||||
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
|
||||
aid=uuid4()
|
||||
key='staging/'+aid.hex
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
object_lock(conn,key)
|
||||
store.put(key,file,length,mime)
|
||||
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) '
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS,
|
||||
(aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone()
|
||||
job=uuid4()
|
||||
ctx=telemetry.carrier()
|
||||
conn.execute('INSERT INTO attachment_jobs(id,attachment_id,traceparent,tracestate) VALUES (%s,%s,%s,%s)',(job,aid,ctx.get('traceparent'),ctx.get('tracestate')))
|
||||
conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,))
|
||||
return metadata(row)
|
||||
except psycopg.Error:
|
||||
compensate(db,store,key)
|
||||
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
|
||||
except BaseException:
|
||||
compensate(db,store,key)
|
||||
raise
|
||||
|
||||
|
||||
async def upload(request,user_id,chat_id,filename):
|
||||
filename=filename_value(filename)
|
||||
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
|
||||
if mime not in ('image/png','image/jpeg'):
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
|
||||
declared=request.headers.get('Content-Length')
|
||||
if declared is not None:
|
||||
try:
|
||||
length=int(declared)
|
||||
if length<0:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid() from None
|
||||
if length>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
try:
|
||||
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
|
||||
except TimeoutError:
|
||||
raise ApiError(503,'unavailable','Слишком много загрузок') from None
|
||||
try:
|
||||
with tempfile.TemporaryFile(mode='w+b') as file:
|
||||
total=0
|
||||
digest=hashlib.sha256()
|
||||
async def read_body():
|
||||
nonlocal total
|
||||
async for chunk in request.stream():
|
||||
total+=len(chunk)
|
||||
if total>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(file.write,chunk)
|
||||
try:
|
||||
await asyncio.wait_for(read_body(),timeout=30)
|
||||
except TimeoutError:
|
||||
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
|
||||
except ClientDisconnect:
|
||||
raise invalid() from None
|
||||
await run_in_threadpool(validate_image,file,mime)
|
||||
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
|
||||
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
def lookup(conn,attachment_id,user_id):
|
||||
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
|
||||
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
|
||||
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
class ObjectResponse(StreamingResponse):
|
||||
def __init__(self,opened,**kwargs):
|
||||
self.opened=opened
|
||||
super().__init__(opened.chunks(),**kwargs)
|
||||
|
||||
async def __call__(self,scope,receive,send):
|
||||
try:
|
||||
await super().__call__(scope,receive,send)
|
||||
finally:
|
||||
self.opened.close()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
|
||||
import argparse
|
||||
from datetime import datetime,timezone,timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from .attachments import object_lock
|
||||
from .db import Database
|
||||
from .object_store import ObjectStore
|
||||
|
||||
|
||||
def sweep(db,store,grace_seconds=300,apply=False):
|
||||
if grace_seconds<60:
|
||||
raise ValueError('Grace must be at least60seconds')
|
||||
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
|
||||
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
|
||||
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
|
||||
for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)):
|
||||
for obj in page.get('Contents',[]):
|
||||
result['listed']+=1
|
||||
if obj['LastModified']>=cutoff:
|
||||
result['fresh']+=1
|
||||
continue
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,obj['Key'],wait=False):
|
||||
result['locked']+=1
|
||||
continue
|
||||
if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone():
|
||||
result['tracked']+=1
|
||||
continue
|
||||
result['orphans']+=1
|
||||
if apply:
|
||||
store.delete(obj['Key'])
|
||||
result['deleted']+=1
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--grace-seconds',type=int,default=300)
|
||||
parser.add_argument('--apply',action='store_true')
|
||||
parser.add_argument('--dry-run',action='store_true')
|
||||
parser.add_argument('--loop',action='store_true')
|
||||
parser.add_argument('--interval',type=int,default=60)
|
||||
args=parser.parse_args()
|
||||
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
|
||||
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
|
||||
db=Database()
|
||||
store=ObjectStore()
|
||||
db.open()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
result=sweep(db,store,args.grace_seconds,args.apply)
|
||||
print(json.dumps(result),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
|
||||
if not args.loop:
|
||||
raise
|
||||
finally:
|
||||
Path('/tmp/cleanup-heartbeat').touch()
|
||||
if not args.loop:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Bounded dependencies; PostgreSQL is always the source of truth."""
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from . import telemetry
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
def setting(name, default=None):
|
||||
filename = os.environ.get(name + '_FILE')
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
|
||||
if value is None:
|
||||
raise RuntimeError(f'Missing configuration: {name}')
|
||||
return value
|
||||
|
||||
|
||||
class TracedConnection(psycopg.Connection):
|
||||
def execute(self,query,*args,**kwargs):
|
||||
# Trace actual query wait, never a pooled lease containing S3/image work.
|
||||
operation=str(query).split(None,1)[0].upper() if str(query).strip() else 'OTHER'
|
||||
if operation not in ('SELECT','INSERT','UPDATE','DELETE'): operation='OTHER'
|
||||
with telemetry.span('postgres.query',attributes={'db.system':'postgresql','db.operation.name':operation}):
|
||||
try:
|
||||
return super().execute(query,*args,**kwargs)
|
||||
except Exception as exc:
|
||||
telemetry.event('db_operation_failed',error_type=type(exc).__name__)
|
||||
raise
|
||||
|
||||
def commit(self):
|
||||
with telemetry.span('postgres.commit',attributes={'db.system':'postgresql'}):
|
||||
return super().commit()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.session_ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
self.pool = ConnectionPool(
|
||||
setting('DATABASE_URL'), connection_class=TracedConnection, min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
|
||||
timeout=2, max_waiting=16, open=False,
|
||||
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
|
||||
)
|
||||
self.cache = Redis.from_url(
|
||||
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=30)
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
@@ -0,0 +1,19 @@
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid():
|
||||
return ApiError(400, 'invalid_request', 'Неверный запрос')
|
||||
|
||||
|
||||
def unauthorized(basic=False):
|
||||
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
|
||||
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
|
||||
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, 'not_found', 'Объект не найден')
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Transactional outbox, fenced processing, bounded retries and operator replay."""
|
||||
from contextlib import nullcontext
|
||||
import signal
|
||||
import time
|
||||
from datetime import datetime,timezone
|
||||
from . import telemetry
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
from redis import Redis
|
||||
from .db import setting
|
||||
from .attachments import object_lock
|
||||
|
||||
LOG=logging.getLogger('messenger')
|
||||
STREAM='attachment-jobs'
|
||||
GROUP='image-workers'
|
||||
MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5'))
|
||||
LEASE_SECONDS=90
|
||||
|
||||
|
||||
def broker_client():
|
||||
return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True,
|
||||
socket_connect_timeout=1,socket_timeout=3,max_connections=4)
|
||||
|
||||
|
||||
def job_lock(conn,job_id):
|
||||
return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
|
||||
('job:'+str(job_id),)).fetchone()['ok']
|
||||
|
||||
|
||||
def unlock(conn,job_id):
|
||||
conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),))
|
||||
|
||||
|
||||
def release_lock(conn,job_id):
|
||||
# Session locks survive ROLLBACK. A failed transaction must be cleared before unlock.
|
||||
try:
|
||||
conn.rollback()
|
||||
unlock(conn,job_id)
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.close() # never return a session with an uncertain held lock to the pool
|
||||
raise
|
||||
|
||||
|
||||
def relay_tick(db,broker):
|
||||
# Recovery uses the same session lock as the processor. A slow live worker is never fenced out.
|
||||
with db.pool.connection() as conn:
|
||||
expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE a.status='processing' AND j.lease_until<clock_timestamp() LIMIT 50").fetchall()
|
||||
for row in expired:
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,row['id']):
|
||||
continue
|
||||
try:
|
||||
conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id="
|
||||
"(SELECT attachment_id FROM attachment_jobs WHERE id=%s AND lease_until<clock_timestamp()) "
|
||||
"AND status='processing'",(row['id'],))
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(row['id'],))
|
||||
conn.execute('UPDATE attachment_jobs SET next_at=clock_timestamp(),lease_until=NULL WHERE id=%s',(row['id'],))
|
||||
conn.commit()
|
||||
finally:
|
||||
release_lock(conn,row['id'])
|
||||
with db.pool.connection() as conn:
|
||||
rows=conn.execute("SELECT j.*,o.job_id FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"JOIN attachments a ON a.id=j.attachment_id WHERE o.dispatched_at IS NULL AND a.status='queued' "
|
||||
"AND j.next_at<=clock_timestamp() ORDER BY j.next_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED").fetchall()
|
||||
for row in rows:
|
||||
payload={'schema_version':1,'type':'attachment.process.v1','job_id':str(row['id']),
|
||||
'attachment_id':str(row['attachment_id']),'pipeline_version':row['pipeline_version'],
|
||||
'attempt':row['attempt']+1,'created_at':row['created_at'].isoformat()}
|
||||
for key in ('traceparent','tracestate'):
|
||||
if row[key]: payload[key]=row[key]
|
||||
with telemetry.span('attachments.publish',kind=telemetry.SpanKind.PRODUCER,context=telemetry.propagate.extract(payload)):
|
||||
payload.update(telemetry.carrier())
|
||||
broker.xadd(STREAM,{'payload':json.dumps(payload)})
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=clock_timestamp() WHERE job_id=%s',(row['id'],))
|
||||
return len(rows)
|
||||
|
||||
|
||||
def process_objects_local(store,row,directory=None):
|
||||
with (nullcontext(directory) if directory else tempfile.TemporaryDirectory(prefix='image-job-')) as directory:
|
||||
source=Path(directory)/'input'
|
||||
opened=store.get(row['staging_key'])
|
||||
try:
|
||||
with source.open('wb') as file:
|
||||
total=0
|
||||
for chunk in opened.chunks():
|
||||
total+=len(chunk)
|
||||
if total>row['size_bytes'] or total>10485760: raise ValueError('invalid_image')
|
||||
file.write(chunk)
|
||||
finally:
|
||||
opened.close()
|
||||
raw_hash=hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']:
|
||||
raise ValueError('invalid_image')
|
||||
completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory],
|
||||
capture_output=True,timeout=20)
|
||||
if completed.returncode:
|
||||
raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit')
|
||||
variants=json.loads(completed.stdout)
|
||||
with source.open('rb') as file:
|
||||
store.put(row['object_key'],file,row['size_bytes'],row['content_type'])
|
||||
for name,item in variants.items():
|
||||
with (Path(directory)/name).open('rb') as file:
|
||||
store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type'])
|
||||
return list(variants.values())
|
||||
|
||||
|
||||
def process_objects(store,row):
|
||||
# A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread.
|
||||
timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45'))
|
||||
if not 1<=timeout<=45: raise ValueError('processing_limit')
|
||||
row={**row,'telemetry_context':telemetry.carrier()}
|
||||
with tempfile.TemporaryDirectory(prefix='image-job-') as directory:
|
||||
process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory],
|
||||
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
|
||||
text=True,start_new_session=True)
|
||||
try:
|
||||
output,_=process.communicate(json.dumps(row,default=str),timeout=timeout)
|
||||
except BaseException:
|
||||
# Terminate processor AND decoder before lock release or another attempt.
|
||||
try: os.killpg(process.pid,signal.SIGKILL)
|
||||
except ProcessLookupError: pass
|
||||
process.communicate()
|
||||
raise
|
||||
if process.returncode==65: raise ValueError('invalid_image')
|
||||
if process.returncode==66: raise ValueError('processing_limit')
|
||||
if process.returncode: raise RuntimeError('storage_unavailable')
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def work(db,store,job_id,delivery_attempt=None):
|
||||
job_id=UUID(str(job_id))
|
||||
# Dedicated pooled session holds the lock, with short committed transactions between effects.
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,job_id):
|
||||
return False
|
||||
conn.commit()
|
||||
try:
|
||||
row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due '
|
||||
'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone()
|
||||
conn.commit()
|
||||
if not row or row['status'] in ('ready','failed'):
|
||||
return True
|
||||
if row['status']!='queued' or not row['due']:
|
||||
return False
|
||||
if delivery_attempt is not None and delivery_attempt!=row['attempt']+1:
|
||||
return True
|
||||
attempt=row['attempt']+1
|
||||
if attempt>MAX_ATTEMPTS:
|
||||
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.commit()
|
||||
return True
|
||||
conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s",
|
||||
(attempt,LEASE_SECONDS,job_id))
|
||||
conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.commit()
|
||||
started=time.monotonic()
|
||||
try:
|
||||
variants=process_objects(store,row)
|
||||
except Exception as exc:
|
||||
permanent=isinstance(exc,ValueError)
|
||||
final=permanent or attempt>=MAX_ATTEMPTS
|
||||
code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable')
|
||||
if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'):
|
||||
code='processing_failed'
|
||||
delay=min(30,2**(attempt-1))*random.uniform(.5,1.5)
|
||||
conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
|
||||
('failed' if final else 'queued',code if final else None,row['id']))
|
||||
conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id))
|
||||
if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
|
||||
conn.commit()
|
||||
telemetry.job_done('failed' if final else 'retry',time.monotonic()-started,attempt)
|
||||
LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt)
|
||||
return True
|
||||
# Same live lock and attempt fence protect publication; no new attempt can publish concurrently.
|
||||
current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone()
|
||||
if current['attempt']!=attempt:
|
||||
conn.rollback()
|
||||
return False
|
||||
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(Jsonb(variants),row['id']))
|
||||
conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,))
|
||||
conn.commit()
|
||||
telemetry.job_done('ready',time.monotonic()-started,attempt,(datetime.now(timezone.utc)-row['created_at']).total_seconds())
|
||||
LOG.info('job_completed attempt=%s',attempt)
|
||||
clean_staging(db,store,row['id'])
|
||||
return True
|
||||
finally:
|
||||
release_lock(conn,job_id)
|
||||
|
||||
|
||||
def clean_staging(db,store,attachment_id=None):
|
||||
with db.pool.connection() as conn:
|
||||
rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL "
|
||||
"AND (status='ready' OR (status='failed' AND updated_at<clock_timestamp()-interval '24 hours')) "
|
||||
+('AND id=%s ' if attachment_id else '')+'LIMIT 50', (attachment_id,) if attachment_id else ()).fetchall()
|
||||
for row in rows:
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,row['staging_key'],wait=False): continue
|
||||
current=conn.execute("SELECT status FROM attachments WHERE id=%s FOR UPDATE",(row['id'],)).fetchone()
|
||||
if current['status'] not in ('ready','failed'): continue
|
||||
store.delete(row['staging_key'])
|
||||
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s',(row['id'],))
|
||||
|
||||
|
||||
def replay(db,job_id,reason):
|
||||
if not reason.strip() or len(reason)>200: raise ValueError('Short audit reason required')
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,job_id): raise ValueError('Job active')
|
||||
try:
|
||||
row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone()
|
||||
if not row or row['status']!='failed' or not row['staging_key']:
|
||||
raise ValueError('Replay requires failed job with retained staging')
|
||||
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,))
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
|
||||
conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason))
|
||||
conn.commit()
|
||||
finally:
|
||||
release_lock(conn,job_id)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""One PostgreSQL transaction owns both a message and its retry result."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from uuid import uuid4
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def validate_key(key):
|
||||
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
raise invalid()
|
||||
return key
|
||||
|
||||
|
||||
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
|
||||
|
||||
|
||||
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
|
||||
attachment_ids=[] if attachment_ids is None else attachment_ids
|
||||
payload={'text':text}
|
||||
if attachment_ids:
|
||||
payload['attachment_ids']=attachment_ids
|
||||
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
params = (user_id, chat_id, key)
|
||||
# A competing INSERT waits for the other transaction, then the next statement sees its result.
|
||||
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
|
||||
(*params, fingerprint))
|
||||
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
|
||||
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
|
||||
if record['message_id'] is not None and record['active']:
|
||||
if record['request_hash'] != fingerprint:
|
||||
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
|
||||
(record['message_id'],)).fetchone()
|
||||
ids=[uuid_value(item) for item in attachment_ids]
|
||||
if ids:
|
||||
allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
|
||||
if len(allowed)!=len(ids):
|
||||
raise not_found()
|
||||
if any(a['status']=='failed' for a in allowed):
|
||||
raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой')
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
|
||||
if ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
|
||||
[(row['id'],aid,i) for i,aid in enumerate(ids)])
|
||||
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
|
||||
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from .db import setting
|
||||
|
||||
|
||||
def main():
|
||||
directory = Path(__file__).resolve().parents[2] / 'migrations'
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(directory.glob('*.sql')):
|
||||
sql = path.read_text()
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
|
||||
if found:
|
||||
if found[0] != checksum:
|
||||
raise RuntimeError('Applied migration checksum changed: ' + path.name)
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
|
||||
print('Applied', path.name)
|
||||
print('Migrations up to date')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from .errors import invalid
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
try:
|
||||
# Require the UUID string representation, not integer/coercible values.
|
||||
if not isinstance(value, str) or len(value) != 36:
|
||||
raise ValueError
|
||||
return UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', strict=True)
|
||||
|
||||
@field_validator('*')
|
||||
@classmethod
|
||||
def valid_strings(cls, value, info):
|
||||
if isinstance(value, str):
|
||||
if not value.strip() and info.field_name != 'text':
|
||||
raise ValueError('Blank string')
|
||||
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
|
||||
if '\x00' in value:
|
||||
raise ValueError('NUL is not supported')
|
||||
value.encode('utf-8')
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('username')
|
||||
@classmethod
|
||||
def username_ascii(cls, value):
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
raise ValueError('Invalid username')
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def members(cls, value):
|
||||
ids = [uuid_value(item) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('Duplicate member')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def attachment_values(cls,value):
|
||||
ids=[uuid_value(item) for item in value]
|
||||
if len(ids)!=len(set(ids)):
|
||||
raise ValueError('Duplicate attachment')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
@model_validator(mode='after')
|
||||
def nonempty_message(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('Message must contain text or attachment')
|
||||
return self
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from . import telemetry
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from .db import setting
|
||||
from .errors import ApiError
|
||||
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
def unavailable():
|
||||
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
|
||||
|
||||
|
||||
class OpenObject:
|
||||
def __init__(self,body,release,length):
|
||||
self.body=body
|
||||
self.release=release
|
||||
self.length=length
|
||||
self.closed=False
|
||||
self.close_lock=threading.Lock()
|
||||
|
||||
def chunks(self):
|
||||
try:
|
||||
while True:
|
||||
data=self.body.read(65536)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
with self.close_lock:
|
||||
if self.closed:
|
||||
return
|
||||
self.closed=True
|
||||
try:
|
||||
self.body.close()
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.bucket=setting('S3_BUCKET','messenger-attachments')
|
||||
self.gate=threading.BoundedSemaphore(8)
|
||||
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
|
||||
region_name=setting('S3_REGION','us-east-1'),
|
||||
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
|
||||
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
|
||||
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
|
||||
|
||||
@contextmanager
|
||||
def operation(self):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
yield
|
||||
except (BotoCoreError,ClientError):
|
||||
raise unavailable() from None
|
||||
finally:
|
||||
self.gate.release()
|
||||
|
||||
def check(self):
|
||||
with self.operation():
|
||||
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
|
||||
|
||||
def put(self,key,body,length,mime):
|
||||
with self.operation():
|
||||
body.seek(0)
|
||||
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
|
||||
|
||||
def delete(self,key):
|
||||
with self.operation():
|
||||
self.client.delete_object(Bucket=self.bucket,Key=key)
|
||||
|
||||
def get(self,key):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
result=self.client.get_object(Bucket=self.bucket,Key=key)
|
||||
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
|
||||
except (BotoCoreError,ClientError):
|
||||
self.gate.release()
|
||||
raise unavailable() from None
|
||||
except BaseException:
|
||||
self.gate.release()
|
||||
raise
|
||||
|
||||
def pages(self,prefix=PREFIX):
|
||||
token=None
|
||||
while True:
|
||||
params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000}
|
||||
if token:
|
||||
params['ContinuationToken']=token
|
||||
with self.operation():
|
||||
result=self.client.list_objects_v2(**params)
|
||||
yield result
|
||||
if not result.get('IsTruncated'):
|
||||
break
|
||||
token=result['NextContinuationToken']
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
|
||||
def traced_s3(function):
|
||||
@wraps(function)
|
||||
def wrapped(*args,**kwargs):
|
||||
with telemetry.span('s3.'+function.__name__,attributes={'rpc.system':'aws-api','rpc.service':'S3'}):
|
||||
return function(*args,**kwargs)
|
||||
return wrapped
|
||||
|
||||
|
||||
for _operation in ('put','get','delete','check'):
|
||||
setattr(ObjectStore,_operation,traced_s3(getattr(ObjectStore,_operation)))
|
||||
@@ -0,0 +1,56 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
from .errors import invalid
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def encode_cursor(scope, row):
|
||||
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(scope, value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
|
||||
raise ValueError
|
||||
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
|
||||
raise ValueError
|
||||
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
|
||||
if ts.tzinfo is None:
|
||||
raise ValueError
|
||||
return ts, uuid_value(data[2])
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
def page_limit(value):
|
||||
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
|
||||
raise invalid()
|
||||
return int(value)
|
||||
|
||||
|
||||
def public(row):
|
||||
def value(item):
|
||||
if isinstance(item,datetime):
|
||||
return stamp(item)
|
||||
if isinstance(item,UUID):
|
||||
return str(item)
|
||||
if isinstance(item,list):
|
||||
return [value(part) for part in item]
|
||||
return item
|
||||
return {key:value(item) for key,item in row.items()}
|
||||
|
||||
|
||||
def page(scope, rows, limit):
|
||||
return {'items': [public(row) for row in rows[:limit]],
|
||||
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Bounded image decoder subprocess. Original bytes are never re-encoded."""
|
||||
import hashlib
|
||||
import json
|
||||
import resource
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageOps, PngImagePlugin
|
||||
|
||||
|
||||
def fitted(image, maximum):
|
||||
scale=min(1,maximum/image.width,maximum/image.height)
|
||||
size=(max(1,int(image.width*scale)),max(1,int(image.height*scale)))
|
||||
return image.resize(size,Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def process_file(source, directory):
|
||||
Image.MAX_IMAGE_PIXELS=40000000
|
||||
PngImagePlugin.MAX_TEXT_CHUNK=1048576
|
||||
PngImagePlugin.MAX_TEXT_MEMORY=1048576
|
||||
with Image.open(source) as original:
|
||||
if original.format not in ('PNG','JPEG') or original.width*original.height>40000000:
|
||||
raise ValueError('invalid_image')
|
||||
image=ImageOps.exif_transpose(original)
|
||||
# Alpha is composited on white; output carries no source metadata.
|
||||
rgba=image.convert('RGBA')
|
||||
white=Image.new('RGBA',rgba.size,'white')
|
||||
white.alpha_composite(rgba)
|
||||
image=white.convert('RGB')
|
||||
image.info.clear()
|
||||
outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'),
|
||||
'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'),
|
||||
'optimized':(fitted(image,1280),'WEBP','image/webp')}
|
||||
manifest={}
|
||||
for name,(variant,fmt,mime) in outputs.items():
|
||||
path=Path(directory)/name
|
||||
variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'')
|
||||
raw=path.read_bytes()
|
||||
manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height,
|
||||
'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}
|
||||
return manifest
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU,(15,15))
|
||||
try:
|
||||
print(json.dumps(process_file(sys.argv[1],sys.argv[2])))
|
||||
except (OSError,ValueError,MemoryError,Image.DecompressionBombError):
|
||||
sys.exit(65)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds."""
|
||||
import os
|
||||
import signal
|
||||
import json
|
||||
from . import telemetry
|
||||
import sys
|
||||
from uuid import UUID
|
||||
from .jobs import process_objects_local
|
||||
from .object_store import ObjectStore
|
||||
|
||||
|
||||
def main():
|
||||
# Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline.
|
||||
signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL))
|
||||
signal.alarm(45)
|
||||
row=json.load(sys.stdin)
|
||||
row['id']=UUID(row['id'])
|
||||
telemetry.init('messenger-worker',child=True)
|
||||
store=ObjectStore()
|
||||
try:
|
||||
with telemetry.span('attachments.process_objects',context=telemetry.propagate.extract(row.get('telemetry_context',{}))):
|
||||
print(json.dumps(process_objects_local(store,row,sys.argv[1])))
|
||||
except ValueError as exc:
|
||||
sys.exit(66 if str(exc)=='processing_limit' else 65)
|
||||
except Exception:
|
||||
# Do not export credentials, storage keys or SDK response details.
|
||||
sys.exit(75)
|
||||
finally:
|
||||
store.close()
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,49 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, InvalidHashError
|
||||
from .errors import unauthorized
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
|
||||
|
||||
|
||||
def digest(token):
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, encoded = header.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
return username, password
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized(basic=True) from None
|
||||
|
||||
|
||||
def check_password(encoded, password):
|
||||
try:
|
||||
return HASHER.verify(encoded, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def bearer(header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError
|
||||
return token
|
||||
except ValueError:
|
||||
raise unauthorized() from None
|
||||
|
||||
|
||||
def new_session(ttl):
|
||||
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Explicit low-cardinality telemetry; no bodies, headers, storage keys or SQL params."""
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import trace,metrics,propagate
|
||||
from opentelemetry.trace import SpanKind,StatusCode
|
||||
|
||||
HTTP_ROUTES={'/api/v1/users','/api/v1/users/me','/api/v1/auth/sessions','/api/v1/auth/sessions/current',
|
||||
'/api/v1/chats','/health/live','/health/ready'}
|
||||
ALLOWED={'event','request_id','route','method','status_class','outcome','attempt','error_type'}
|
||||
INITIALIZED=False
|
||||
PROVIDERS=[]
|
||||
HANDLER=None
|
||||
SERVICE='messenger-api'
|
||||
BUSY=0
|
||||
HTTP_COUNT=HTTP_TIME=JOB_COUNT=JOB_TIME=READY_TIME=None
|
||||
|
||||
|
||||
def route_name(path):
|
||||
if path in HTTP_ROUTES: return path
|
||||
if re.fullmatch(r'/api/v1/chats/[^/]+(?:/(?:messages|attachments))?',path):
|
||||
return re.sub(r'(?<=/chats/)[^/]+','{chat_id}',path)
|
||||
if re.fullmatch(r'/api/v1/attachments/[^/]+(?:/content)?',path):
|
||||
return re.sub(r'(?<=/attachments/)[^/]+','{attachment_id}',path)
|
||||
return 'unmatched'
|
||||
|
||||
|
||||
def safe_fields(fields):
|
||||
return {key:value for key,value in fields.items() if key in ALLOWED and isinstance(value,(str,int,float,bool))}
|
||||
|
||||
|
||||
def tracer(): return trace.get_tracer('messenger','1.0')
|
||||
|
||||
|
||||
def carrier():
|
||||
data={};propagate.inject(data);return data
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name,kind=SpanKind.INTERNAL,context=None,attributes=None):
|
||||
with tracer().start_as_current_span(name,kind=kind,context=context,attributes=attributes or {},
|
||||
record_exception=False,set_status_on_exception=False) as active:
|
||||
try: yield active
|
||||
except BaseException:
|
||||
active.set_status(StatusCode.ERROR)
|
||||
raise
|
||||
|
||||
|
||||
def event(name,**fields):
|
||||
if HANDLER is None: return
|
||||
context=trace.get_current_span().get_span_context()
|
||||
data=safe_fields({'event':name,**fields})
|
||||
data.update(timestamp=time.time(),severity='INFO',service_name=SERVICE,
|
||||
trace_id=format(context.trace_id,'032x'),span_id=format(context.span_id,'016x'))
|
||||
record=logging.LogRecord('messenger.events',logging.INFO,'',0,data,(),None)
|
||||
HANDLER.emit(record)
|
||||
print(json.dumps(data),flush=True)
|
||||
|
||||
|
||||
def init(service,db=None,child=False):
|
||||
global INITIALIZED,HANDLER,SERVICE,HTTP_COUNT,HTTP_TIME,JOB_COUNT,JOB_TIME,READY_TIME
|
||||
if INITIALIZED or os.environ.get('OTEL_ENABLED')!='1': return
|
||||
INITIALIZED=True;SERVICE=service
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import View,ExplicitBucketHistogramAggregation
|
||||
from opentelemetry.sdk._logs import LoggerProvider,LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
resource=Resource.create({'service.name':service,'service.version':'lab6-v1',
|
||||
'service.instance.id':os.environ.setdefault('TELEMETRY_INSTANCE_ID',uuid4().hex),
|
||||
'deployment.environment.name':'lab'})
|
||||
endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT','http://collector:4318').rstrip('/')
|
||||
tp=TracerProvider(resource=resource)
|
||||
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint+'/v1/traces',timeout=1),
|
||||
max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000))
|
||||
trace.set_tracer_provider(tp);PROVIDERS.append(tp)
|
||||
lp=LoggerProvider(resource=resource)
|
||||
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint+'/v1/logs',timeout=1),
|
||||
max_queue_size=256,max_export_batch_size=32,schedule_delay_millis=500,export_timeout_millis=1000))
|
||||
HANDLER=LoggingHandler(level=logging.INFO,logger_provider=lp);PROVIDERS.append(lp)
|
||||
if child: return
|
||||
reader=PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint+'/v1/metrics',timeout=1),
|
||||
export_interval_millis=2000,export_timeout_millis=1500)
|
||||
mp=MeterProvider(resource=resource,metric_readers=[reader],views=[
|
||||
View(instrument_name='*_duration',aggregation=ExplicitBucketHistogramAggregation([.005,.01,.025,.05,.1,.25,.5,1,2,5,10,30,60,120])),
|
||||
View(instrument_name='attachment_ready',aggregation=ExplicitBucketHistogramAggregation([.1,.5,1,2,5,10,30,60,120,300]))])
|
||||
metrics.set_meter_provider(mp);PROVIDERS.append(mp)
|
||||
meter=metrics.get_meter('messenger','1.0')
|
||||
HTTP_COUNT=meter.create_counter('http_requests')
|
||||
HTTP_TIME=meter.create_histogram('http_server_duration',unit='s')
|
||||
JOB_COUNT=meter.create_counter('jobs_processed')
|
||||
JOB_TIME=meter.create_histogram('job_processing_duration',unit='s')
|
||||
READY_TIME=meter.create_histogram('attachment_ready',unit='s')
|
||||
def process_values(_):
|
||||
rss=int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE')
|
||||
return [Observation(rss)]
|
||||
meter.create_observable_gauge('process_memory',unit='By',callbacks=[process_values])
|
||||
meter.create_observable_counter('process_cpu',unit='s',callbacks=[lambda _:[Observation(time.process_time())]])
|
||||
meter.create_observable_gauge('worker_busy',callbacks=[lambda _:[Observation(BUSY)]])
|
||||
meter.create_observable_gauge('worker_capacity',callbacks=[lambda _:[Observation(1 if service=='messenger-worker' else 0)]])
|
||||
if db:
|
||||
def pool_value(name):
|
||||
return lambda _:[Observation(db.pool.get_stats().get(name,0))]
|
||||
meter.create_observable_gauge('db_pool_used',callbacks=[lambda _:[Observation(db.pool.get_stats().get('pool_size',0)-db.pool.get_stats().get('pool_available',0))]])
|
||||
meter.create_observable_gauge('db_pool_waiting',callbacks=[pool_value('requests_waiting')])
|
||||
def queue_values(field):
|
||||
def callback(_):
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
row=conn.execute("SELECT count(*) AS depth,coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0)::float AS age FROM attachments WHERE status IN ('queued','processing')").fetchone()
|
||||
return [Observation(row[field],{'queue':'attachments'})]
|
||||
except Exception: return []
|
||||
return callback
|
||||
if service=='messenger-relay':
|
||||
meter.create_observable_gauge('queue_depth',callbacks=[queue_values('depth')])
|
||||
meter.create_observable_gauge('queue_oldest_age',unit='s',callbacks=[queue_values('age')])
|
||||
|
||||
|
||||
def http_done(method,route,status,duration,request_id):
|
||||
fields={'method':method,'route':route,'status_class':str(status//100)+'xx'}
|
||||
if HTTP_COUNT and not route.startswith('/health/'):
|
||||
HTTP_COUNT.add(1,fields);HTTP_TIME.record(duration,fields)
|
||||
event('http_completed',**fields,request_id=request_id)
|
||||
|
||||
|
||||
def job_done(outcome,duration,attempt,accepted_age=None):
|
||||
current=trace.get_current_span()
|
||||
current.set_attribute('job.type','attachment.process.v1')
|
||||
current.set_attribute('job.outcome',outcome)
|
||||
current.set_attribute('job.attempt',attempt)
|
||||
if outcome in ('retry','failed'): current.set_status(StatusCode.ERROR)
|
||||
attrs={'job_type':'attachment.process.v1','outcome':outcome}
|
||||
if JOB_COUNT:
|
||||
JOB_COUNT.add(1,attrs);JOB_TIME.record(duration,attrs)
|
||||
if accepted_age is not None and outcome=='ready': READY_TIME.record(accepted_age,{'job_type':'attachment.process.v1'})
|
||||
event('job_attempt',outcome=outcome,attempt=attempt)
|
||||
|
||||
|
||||
def shutdown():
|
||||
for provider in reversed(PROVIDERS):
|
||||
try: provider.shutdown()
|
||||
except Exception: pass
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Separate relay/consumer processes; durable Redis Streams consumer group."""
|
||||
import argparse
|
||||
from . import telemetry
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
from uuid import uuid4
|
||||
from redis.exceptions import ResponseError
|
||||
from .cleanup import sweep
|
||||
from .db import Database
|
||||
from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP
|
||||
from .object_store import ObjectStore
|
||||
|
||||
LOG=logging.getLogger('messenger')
|
||||
STOP=False
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('role',choices=['worker','relay','replay'])
|
||||
parser.add_argument('--job-id')
|
||||
parser.add_argument('--reason',default='operator retry')
|
||||
args=parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
def stop(*_):
|
||||
global STOP
|
||||
STOP=True
|
||||
signal.signal(signal.SIGTERM,stop)
|
||||
signal.signal(signal.SIGINT,stop)
|
||||
db=Database();db.open();store=ObjectStore();broker=broker_client()
|
||||
telemetry.init('messenger-worker' if args.role=='worker' else 'messenger-relay',db)
|
||||
consumer=uuid4().hex
|
||||
last_cleanup=0
|
||||
try:
|
||||
if args.role=='replay':
|
||||
replay(db,args.job_id,args.reason)
|
||||
print('Replay queued with audit')
|
||||
return
|
||||
if args.role=='worker':
|
||||
try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
|
||||
except ResponseError as exc:
|
||||
if 'BUSYGROUP' not in str(exc): raise
|
||||
while not STOP:
|
||||
try:
|
||||
if args.role=='relay':
|
||||
relay_tick(db,broker)
|
||||
clean_staging(db,store)
|
||||
if time.monotonic()-last_cleanup>60:
|
||||
sweep(db,store,300,True)
|
||||
last_cleanup=time.monotonic()
|
||||
time.sleep(.3)
|
||||
else:
|
||||
claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=5000,start_id='0-0',count=1)
|
||||
messages=claimed[1]
|
||||
if not messages:
|
||||
batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
|
||||
messages=batches[0][1] if batches else []
|
||||
for message_id,fields in messages:
|
||||
payload=json.loads(fields['payload'])
|
||||
with telemetry.span('attachments.consume',kind=telemetry.SpanKind.CONSUMER,
|
||||
context=telemetry.propagate.extract(payload)):
|
||||
telemetry.BUSY=1
|
||||
try:
|
||||
if work(db,store,payload['job_id'],payload['attempt']):
|
||||
broker.xack(STREAM,GROUP,message_id)
|
||||
broker.xdel(STREAM,message_id)
|
||||
finally:
|
||||
telemetry.BUSY=0
|
||||
except Exception as exc:
|
||||
LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
broker.close();store.close();db.close();telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from messenger.app import create_app
|
||||
|
||||
PASSWORD = 'Тестовый:пароль🔐' * 8
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with TestClient(create_app()) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def user(c):
|
||||
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
|
||||
r = c.post('/api/v1/users', json=body)
|
||||
assert r.status_code == 201, r.text
|
||||
return r.json(), body
|
||||
|
||||
|
||||
def login(c, body):
|
||||
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
|
||||
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
|
||||
assert r.status_code == 201, r.text
|
||||
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
|
||||
|
||||
|
||||
def test_auth_unicode_logout_and_cache(client):
|
||||
u, body = user(client)
|
||||
h, session = login(client, body)
|
||||
assert session['user'] == u
|
||||
assert len(session['token']) >= 43
|
||||
key = 'user:' + u['id']
|
||||
client.app.state.db.cache.delete(key)
|
||||
for _ in range(2):
|
||||
assert client.get('/api/v1/users/me', headers=h).json() == u
|
||||
cached = json.loads(client.app.state.db.cache.get(key))
|
||||
assert cached == u
|
||||
assert 0 < client.app.state.db.cache.ttl(key) <= 60
|
||||
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
|
||||
r = client.get('/api/v1/users/me', headers=h)
|
||||
assert r.status_code == 401
|
||||
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
|
||||
assert r.json()['error']['request_id'] == r.headers['x-request-id']
|
||||
|
||||
|
||||
def test_session_expiry(monkeypatch):
|
||||
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
|
||||
with TestClient(create_app()) as client:
|
||||
_, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 200
|
||||
time.sleep(2.1)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 401
|
||||
|
||||
|
||||
def test_registration_race(client):
|
||||
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
|
||||
with ThreadPoolExecutor(max_workers=6) as executor:
|
||||
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
|
||||
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
|
||||
|
||||
|
||||
def test_chat_rollback_after_insert(client):
|
||||
u, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
title = 'rollback_' + uuid.uuid4().hex
|
||||
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
|
||||
assert r.status_code == 400
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
|
||||
|
||||
|
||||
def test_cursor_access_and_strict_validation(client):
|
||||
a, ab = user(client)
|
||||
b, bb = user(client)
|
||||
ah, _ = login(client, ab)
|
||||
bh, _ = login(client, bb)
|
||||
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
|
||||
path = '/api/v1/chats/' + chats[0]['id']
|
||||
for text in ['Один 👋', 'Два', 'Три']:
|
||||
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
|
||||
p = client.get(path+'/messages?limit=2', headers=ah).json()
|
||||
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
|
||||
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
|
||||
assert [m['text'] for m in tail['items']] == ['Три']
|
||||
assert tail['next_cursor'] is None
|
||||
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
|
||||
assert r.status_code == 400
|
||||
assert client.get(path, headers=bh).status_code == 404
|
||||
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
|
||||
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
|
||||
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
|
||||
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
|
||||
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
|
||||
assert client.post('/api/v1/users',json=payload).status_code == 400
|
||||
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
|
||||
|
||||
|
||||
def test_session_storage_and_migrations(client):
|
||||
from messenger.migrate import main as migrate
|
||||
u, body = user(client)
|
||||
_, session = login(client, body)
|
||||
db = client.app.state.db
|
||||
def snapshot():
|
||||
with db.pool.connection() as conn:
|
||||
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
|
||||
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
|
||||
before = snapshot()
|
||||
migrate()
|
||||
migrate()
|
||||
assert snapshot() == before
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
|
||||
assert row['token_hash'] != session['token']
|
||||
assert len(row['token_hash']) == 64
|
||||
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
|
||||
assert stored.startswith('$argon2id$')
|
||||
assert stored != body['password']
|
||||
|
||||
|
||||
def test_json_media_type_and_utf8(client):
|
||||
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
|
||||
raw = json.dumps(body).encode()
|
||||
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
|
||||
r = client.post('/api/v1/users',content=raw,headers=headers)
|
||||
assert r.status_code == 415
|
||||
assert r.json()['error']['code'] == 'unsupported_media_type'
|
||||
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
|
||||
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
|
||||
assert r.status_code == 400, encoding
|
||||
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
|
||||
assert r.status_code == 401
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Real PostgreSQL + private S3 integration; synthetic images only."""
|
||||
import hashlib
|
||||
import io
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from PIL import Image
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def image_bytes(kind='PNG'):
|
||||
stream = io.BytesIO()
|
||||
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def actors(c):
|
||||
a,ab=user(c)
|
||||
b,bb=user(c)
|
||||
e,eb=user(c)
|
||||
ah,_=login(c,ab)
|
||||
bh,_=login(c,bb)
|
||||
eh,_=login(c,eb)
|
||||
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
|
||||
return ah,bh,eh,chat
|
||||
|
||||
|
||||
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
|
||||
raw=image_bytes() if raw is None else raw
|
||||
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
|
||||
headers={**headers,'Content-Type':mime})
|
||||
|
||||
|
||||
def finish(client,attachment_id):
|
||||
from messenger.jobs import work
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id']
|
||||
work(client.app.state.db,client.app.state.storage,job)
|
||||
import time
|
||||
for _ in range(100):
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status']
|
||||
if status=='ready': return
|
||||
time.sleep(.1)
|
||||
raise AssertionError('Job did not become ready: '+status)
|
||||
|
||||
|
||||
def test_original_png_jpeg_and_message(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
|
||||
raw=image_bytes(kind)
|
||||
r=upload(client,ah,chat,raw,mime)
|
||||
assert r.status_code==202
|
||||
att=r.json()
|
||||
assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None
|
||||
assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409
|
||||
finish(client,att['id'])
|
||||
att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json()
|
||||
assert att['status']=='ready' and len(att['variants'])==3
|
||||
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
|
||||
assert 'object_key' not in att and 'bucket' not in att
|
||||
path='/api/v1/attachments/'+att['id']
|
||||
assert client.get(path,headers=bh).json()==att
|
||||
content=client.get(path+'/content',headers=bh)
|
||||
assert content.status_code==200 and content.content==raw
|
||||
assert content.headers['Content-Type']==mime
|
||||
for suffix in ['', '/content']:
|
||||
assert client.get(path+suffix,headers=eh).status_code==404
|
||||
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
|
||||
json={'text':'','attachment_ids':[att['id']]})
|
||||
assert response.status_code==201
|
||||
assert response.json()['attachment_ids']==[att['id']]
|
||||
|
||||
|
||||
def test_invalid_media_corruption_limits_and_filename(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
|
||||
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
|
||||
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
|
||||
for raw,mime,expected in cases:
|
||||
response=upload(client,ah,chat,raw,mime)
|
||||
assert response.status_code==expected
|
||||
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
|
||||
# Valid PNG header advertises >40Mp; reject before attempting decompression.
|
||||
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
|
||||
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
|
||||
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
|
||||
assert upload(client,ah,chat,bomb).status_code==413
|
||||
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
|
||||
assert upload(client,ah,chat,filename=filename).status_code==400
|
||||
|
||||
|
||||
def test_foreign_attachment_binding_and_normalized_replay(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
|
||||
path='/api/v1/chats/'+other['id']+'/messages'
|
||||
for aid in [att['id'],str(uuid.uuid4())]:
|
||||
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
|
||||
assert response.status_code==404
|
||||
assert upload(client,eh,chat).status_code==404
|
||||
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
first=client.post(path,headers=h,json={'text':'same'})
|
||||
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
|
||||
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
|
||||
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
|
||||
assert conflict.status_code==409
|
||||
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
|
||||
assert duplicates.status_code==400
|
||||
|
||||
|
||||
def test_compensate_s3_when_metadata_insert_fails(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
db=client.app.state.db
|
||||
store=client.app.state.storage
|
||||
before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
|
||||
function='fail_attachment_'+uuid.uuid4().hex
|
||||
trigger='fail_attachment_'+uuid.uuid4().hex
|
||||
filename='fault_'+uuid.uuid4().hex+'.png'
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
response=upload(client,ah,chat,filename=filename)
|
||||
assert response.status_code==503
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
|
||||
after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
|
||||
assert after==before
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
|
||||
|
||||
def test_compensation_preserves_committed_object(client):
|
||||
from messenger.attachments import compensate
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
# A retry after an unknown COMMIT result must never delete an already tracked object.
|
||||
compensate(client.app.state.db,client.app.state.storage,key)
|
||||
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
|
||||
assert content.status_code==200 and content.content==image_bytes()
|
||||
|
||||
|
||||
def test_stream_close_even_if_headers_cannot_be_sent(client):
|
||||
import asyncio
|
||||
import pytest
|
||||
from messenger.attachments import ObjectResponse
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
store=client.app.state.storage
|
||||
opened=store.get(key)
|
||||
response=ObjectResponse(opened,media_type='image/png')
|
||||
async def fail_send(message):
|
||||
raise OSError('synthetic disconnected transport')
|
||||
async def receive():
|
||||
return {'type':'http.disconnect'}
|
||||
with pytest.raises(Exception):
|
||||
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
|
||||
assert opened.closed
|
||||
acquired=0
|
||||
try:
|
||||
for _ in range(8):
|
||||
assert store.gate.acquire(blocking=False)
|
||||
acquired+=1
|
||||
finally:
|
||||
for _ in range(acquired):
|
||||
store.gate.release()
|
||||
|
||||
|
||||
def test_chunked_upload_is_limited_without_content_length(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
chunks=iter([b'x'*1048576]*10+[b'x'])
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
|
||||
content=chunks,headers={**ah,'Content-Type':'image/png'})
|
||||
assert response.status_code==413
|
||||
assert response.json()['error']['code']=='payload_too_large'
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def setup_chat(c, extra=None):
|
||||
u, body = user(c)
|
||||
headers, _ = login(c, body)
|
||||
members = [u['id']] + (extra or [])
|
||||
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
|
||||
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
|
||||
|
||||
|
||||
def test_instance_alias_on_success_and_errors(client):
|
||||
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
|
||||
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
|
||||
ids = [r.headers.get('X-Instance-Id') for r in responses]
|
||||
assert ids[0] and len(set(ids)) == 1
|
||||
assert all(r.headers.get('X-Request-Id') for r in responses)
|
||||
|
||||
|
||||
def test_key_required_replay_and_conflict(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
|
||||
for key in ['', 'a b', 'bad/', 'x'*129]:
|
||||
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
|
||||
h = {**headers, 'Idempotency-Key':'valid:._-123'}
|
||||
first = client.post(path, headers=h, json={'text':'same'})
|
||||
assert first.status_code == 201
|
||||
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
|
||||
assert replay.status_code == 201 and replay.json() == first.json()
|
||||
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
|
||||
assert len(client.get(path,headers=headers).json()['items']) == 1
|
||||
|
||||
|
||||
def test_concurrent_key_produces_one_message(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
|
||||
assert [r.status_code for r in results] == [201]*16
|
||||
assert all(r.json() == results[0].json() for r in results)
|
||||
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
|
||||
|
||||
|
||||
def test_scope_and_replay_rechecks_authorization(client):
|
||||
b, bb = user(client)
|
||||
bh, _ = login(client, bb)
|
||||
a, ah, path = setup_chat(client, [b['id']])
|
||||
key = 'same-in-independent-scopes'
|
||||
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
|
||||
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
|
||||
other_path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
|
||||
assert third.status_code == 201 and third.json()['id'] != first['id']
|
||||
outsider, ob = user(client)
|
||||
oh, _ = login(client, ob)
|
||||
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
|
||||
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
|
||||
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
|
||||
assert 86390 <= ttl <= 86400
|
||||
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
|
||||
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
assert len(client.get(path,headers=h).json()['items']) == 2
|
||||
|
||||
|
||||
def test_rollback_side_effect_and_key_on_database_error(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'fault_' + uuid.uuid4().hex
|
||||
function = 'fault_' + uuid.uuid4().hex
|
||||
trigger = 'trigger_' + uuid.uuid4().hex
|
||||
db = client.app.state.db
|
||||
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert failed.status_code == 500
|
||||
assert failed.headers['X-Instance-Id']
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert retried.status_code == 201
|
||||
assert client.get(path,headers=h).json()['items'] == [retried.json()]
|
||||
|
||||
|
||||
def test_ready_requires_compatible_schema(client):
|
||||
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
|
||||
try:
|
||||
r = client.get('/health/ready')
|
||||
assert r.status_code == 503
|
||||
assert r.json()['error']['code'] == 'unavailable'
|
||||
assert client.get('/health/live').status_code == 200
|
||||
finally:
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
|
||||
@@ -0,0 +1,63 @@
|
||||
import io
|
||||
import pytest
|
||||
from messenger.jobs import work, replay
|
||||
from test_attachments import actors,upload,image_bytes,finish
|
||||
from test_api import client
|
||||
|
||||
|
||||
def job_for(client,aid):
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id']
|
||||
|
||||
|
||||
def test_duplicate_ready_is_noop_and_not_replayable(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
path='/api/v1/attachments/'+att['id']
|
||||
before=client.get(path,headers=ah).json()
|
||||
job=job_for(client,att['id'])
|
||||
for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1)
|
||||
assert client.get(path,headers=ah).json()==before
|
||||
with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready')
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
|
||||
assert row['attempt']==1
|
||||
assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None
|
||||
|
||||
|
||||
def test_old_delivery_does_not_consume_attempt(client):
|
||||
# Direct invocation avoids consumer timing; it exercises the same real DB/S3 code.
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
job=job_for(client,att['id'])
|
||||
assert work(client.app.state.db,client.app.state.storage,job,999)
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
|
||||
assert row['attempt'] in (0,1) # independent live consumer may already finish this job
|
||||
finish(client,att['id'])
|
||||
|
||||
|
||||
def test_replay_sql_failure_releases_session_lock(client):
|
||||
import psycopg
|
||||
from messenger.db import setting
|
||||
from psycopg.rows import dict_row
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
job=job_for(client,att['id'])
|
||||
db=client.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id']))
|
||||
conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$")
|
||||
conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()')
|
||||
try:
|
||||
with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock')
|
||||
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other:
|
||||
acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok']
|
||||
if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),))
|
||||
assert acquired, 'Aborted replay leaked a session advisory lock into the pool'
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays')
|
||||
conn.execute('DROP FUNCTION lab5_fail_audit()')
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Real image transformations, no mock storage."""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def test_worker_variants_and_original(tmp_path):
|
||||
source=tmp_path/'input'
|
||||
original=Image.new('RGBA',(320,200),(30,90,150,70))
|
||||
original.save(source,format='PNG')
|
||||
before=hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
|
||||
assert result.returncode==0, result.stderr
|
||||
manifest=json.loads(result.stdout)
|
||||
assert hashlib.sha256(source.read_bytes()).hexdigest()==before
|
||||
for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]:
|
||||
item=manifest[kind]
|
||||
with Image.open(tmp_path/kind) as image:
|
||||
assert image.format==fmt and image.size==size
|
||||
assert not image.getexif()
|
||||
raw=(tmp_path/kind).read_bytes()
|
||||
assert item['sha256']==hashlib.sha256(raw).hexdigest()
|
||||
assert item['size_bytes']==len(raw)
|
||||
|
||||
|
||||
def test_no_upscale_thumbnail_and_exif(tmp_path):
|
||||
source=tmp_path/'input'
|
||||
image=Image.new('RGB',(80,40),'red')
|
||||
exif=Image.Exif();exif[274]=6
|
||||
image.save(source,format='JPEG',exif=exif)
|
||||
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
|
||||
assert result.returncode==0, result.stderr
|
||||
with Image.open(tmp_path/'thumbnail') as output:
|
||||
assert output.size==(40,80)
|
||||
assert not output.getexif()
|
||||
|
||||
|
||||
def test_deadline_covers_slow_s3(monkeypatch):
|
||||
import contextlib
|
||||
import http.server
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import pytest
|
||||
from messenger.jobs import process_objects
|
||||
class Slow(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Length','100')
|
||||
self.end_headers()
|
||||
try:
|
||||
for _ in range(100):
|
||||
self.wfile.write(b'x');self.wfile.flush();time.sleep(.05)
|
||||
except (BrokenPipeError,ConnectionResetError): pass
|
||||
def log_message(self,*args): pass
|
||||
server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow)
|
||||
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
|
||||
monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False)
|
||||
monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port))
|
||||
monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2')
|
||||
row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow',
|
||||
'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'}
|
||||
started=time.monotonic()
|
||||
try:
|
||||
with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row)
|
||||
assert time.monotonic()-started<4
|
||||
finally:
|
||||
server.shutdown();server.server_close()
|
||||
@@ -0,0 +1,51 @@
|
||||
from test_api import client
|
||||
from messenger.telemetry import route_name, safe_fields
|
||||
|
||||
|
||||
def test_route_labels_never_include_ids_or_query():
|
||||
assert route_name('/api/v1/chats/a87e23bc-430c-4862-9da8-1de1286931aa/messages')=='/api/v1/chats/{chat_id}/messages'
|
||||
assert route_name('/api/v1/attachments/a87e23bc-430c-4862-9da8-1de1286931aa/content')=='/api/v1/attachments/{attachment_id}/content'
|
||||
assert route_name('/unknown/private-name')=='unmatched'
|
||||
|
||||
|
||||
def test_log_attributes_allowlist_drops_sensitive_values():
|
||||
result=safe_fields({'event':'http_completed','request_id':'test','route':'/api/v1/users',
|
||||
'Authorization':'Bearer SECRET','password':'SECRET','body':'SECRET','filename':'private'})
|
||||
assert result=={'event':'http_completed','request_id':'test','route':'/api/v1/users'}
|
||||
|
||||
|
||||
def test_retry_consumer_span_marks_failure():
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
from messenger import telemetry
|
||||
exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
with provider.get_tracer('test').start_as_current_span('attachments.consume'):
|
||||
telemetry.job_done('retry',.1,2)
|
||||
span=exporter.get_finished_spans()[0]
|
||||
assert span.attributes['job.outcome']=='retry'
|
||||
assert span.attributes['job.attempt']==2
|
||||
assert span.status.status_code==StatusCode.ERROR
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
def test_db_span_covers_query_not_image_work(client,monkeypatch):
|
||||
import time
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from messenger import telemetry
|
||||
exporter=InMemorySpanExporter();provider=TracerProvider();provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
monkeypatch.setattr(telemetry,'tracer',lambda:provider.get_tracer('test'))
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
time.sleep(.15) # non-DB work with a leased connection
|
||||
conn.execute('SELECT pg_sleep(0.02)')
|
||||
spans=exporter.get_finished_spans()
|
||||
assert any(s.name=='postgres.query' for s in spans)
|
||||
assert not any(s.name=='postgres.transaction' for s in spans)
|
||||
for span in spans:
|
||||
if span.name=='postgres.query':
|
||||
assert (span.end_time-span.start_time)/1e9<.15
|
||||
assert 'db.statement' not in span.attributes
|
||||
provider.shutdown()
|
||||
Reference in New Issue
Block a user