156 changed files with 18393 additions and 35 deletions
+45 -5
View File
@@ -6,25 +6,65 @@ APP_ENV=lab
SESSION_TTL_SECONDS=3600 SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values. # Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey: # Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2):
SESSION_STORE=postgres SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2 API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400 IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000 # SeaweedFS S3 (since lab4)
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_BUCKET=messenger-attachments S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760 MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000 MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox);
# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher,
# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor.
BROKER_URL_FILE=
WORKER_CONCURRENCY=2 WORKER_CONCURRENCY=2
WORKER_PREFETCH=1
JOB_MAX_ATTEMPTS=5 JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60 JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=15
JOB_BACKOFF_BASE_SECONDS=2
JOB_BACKOFF_MAX_SECONDS=60
WORKER_MEMORY_LIMIT_MB=512
STAGING_RETENTION_SECONDS=86400 STAGING_RETENTION_SECONDS=86400
PIPELINE_VERSION=1
# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the
# training stand by default; set FAULT_INJECTION=0 here to disable.
# Pools and timeouts (same as lab3)
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_WAIT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
REDIS_POOL_MAX=20
REDIS_TIMEOUT_MS=500
SHUTDOWN_DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=10
# lab4: S3 client and uploads
S3_CONNECT_TIMEOUT_SECONDS=1
S3_READ_TIMEOUT_SECONDS=5
S3_MAX_ATTEMPTS=2
S3_POOL_MAX=20
UPLOAD_TIMEOUT_SECONDS=60
UPLOAD_IDLE_TIMEOUT_SECONDS=10
DECODE_CONCURRENCY=2
# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval
ATTACHMENT_GRACE_SECONDS=3600
JANITOR_INTERVAL_SECONDS=300
# lab6: OpenTelemetry. Standard SDK variables; compose.yaml sets the same values for every app service
# and a distinct OTEL_SERVICE_NAME (messenger-api / messenger-worker / messenger-dispatcher / messenger-janitor).
OTEL_SERVICE_NAME=messenger-api OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# true -> SDK off in all app processes (logs still go to stderr as JSON)
OTEL_SDK_DISABLED=false
+11
View File
@@ -0,0 +1,11 @@
*.sh text eol=lf
*.sql text eol=lf
*.conf text eol=lf
*.cfg text eol=lf
*.example text eol=lf
*.cmd text eol=crlf
*.yaml text eol=lf
*.yml text eol=lf
*.json text eol=lf
*.py text eol=lf
*.md text eol=lf
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 6 ## Лабораторная 6
Автор / группа: TODO Автор / группа: Бабиченко Максим, БВТ2303
Целевая оценка (3/4/5): TODO Целевая оценка (3/4/5): 4
Трек 5, если заявлен: TODO Трек 5, если заявлен: не заявлен
Что реализовано и что перенесено: TODO Что реализовано и что перенесено: перенесено всё решение lab5 (API, outbox, dispatcher, worker, janitor, тесты lab1–5). Добавлены: OpenTelemetry SDK во всех процессах (traces/metrics/logs по OTLP), W3C trace context API → outbox → RabbitMQ → worker (parent-child, replay — link), OTel Collector (memory_limiter, batch, ограниченные очереди), Prometheus (OTLP receiver, recording/alert rules + unit-тесты), Loki, Tempo, Grafana с provisioning источников и dashboard из Git, SLI/SLO, 6 алертов с runbooks, сценарии инцидентов.
Команды запуска из чистого clone: TODO Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python scripts/init_secrets.py`, `docker compose up -d --build --wait`, `python student-tests/collect_evidence.py ...`).
Результаты проверок и ссылка на REPORT.md: TODO Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», `evidence/summary.md`, `evidence/incident-*.json`; прогон без Docker — `evidence/native-sandbox/`.
Ограничения / известные ошибки: TODO Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (одна копия каждого хранилища, буфер Collector в памяти, без TLS до lab7).
Источники и использование ИИ: TODO Источники и использование ИИ: REPORT.md, шапка (документация OTel/Prometheus/Loki/Tempo/Grafana; ИИ-ассистент Claude).
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены. - [x] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод. - [ ] make check и make test выполнены; приложен реальный вывод (Docker-прогон: `evidence/`).
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. - [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/incident-*.json`).
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. - [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. - [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+31
View File
@@ -0,0 +1,31 @@
FROM python:3.11.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt
COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \
pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py ./
COPY migrations/ ./migrations/
RUN useradd --create-home --shell /usr/sbin/nologin appuser \
&& mkdir -p /faults && chown appuser:appuser /faults
# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser
# копируется в пустой named volume при первом монтировании.
USER appuser
ARG APP_VERSION=dev
ENV APP_VERSION=${APP_VERSION}
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)"
CMD ["python", "main.py"]
+347 -16
View File
@@ -1,37 +1,368 @@
# Отчёт — лабораторная 6 # Отчёт — лабораторная 6
- Автор, группа: TODO - Автор, группа: Бабиченко Максим, БВТ2303
- Целевая оценка; выбранный трек 5, если нужен: TODO - Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран
- Commit/PR: TODO после публикации - Commit/PR: <<ссылка на MR в Gitea после публикации>>
- Использованные источники, библиотеки и помощь ИИ: TODO - Использованные источники, библиотеки и помощь ИИ: всё из lab5 (FastAPI, psycopg 3, pika, boto3, Pillow, RabbitMQ 4.1, SeaweedFS 3.80, PostgreSQL 17, Redis 7.4, HAProxy 3.2). Новое: OpenTelemetry Python SDK 1.45.2 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) и `opentelemetry-instrumentation-psycopg` 0.66b2; образы `otel/opentelemetry-collector-contrib:0.164.0`, `prom/prometheus:v3.15.1`, `grafana/loki:3.7.9`, `grafana/tempo:3.1.1`, `grafana/grafana:12.4.14`, `busybox:1.37.0` (bootstrap прав volumes). Документация: OTel Collector configuration (memory_limiter, batch, exporterhelper sending_queue/retry_on_failure, internal telemetry), OTel semantic conventions (HTTP, database, messaging, `otel.sdk.*`), W3C Trace Context, Prometheus OTLP receiver и histograms/`histogram_quantile`, promtool unit testing, Loki OTLP ingestion и structured metadata, Tempo 3 single binary, Grafana provisioning (datasources, dashboards, derived fields, traces-to-logs). При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите.
## Запуск из чистого clone ## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. Проверяю на Windows (PowerShell, Docker Desktop). Python на хосте нужен только для тестов (стандартная библиотека). Перед запуском остановить стенд lab5 (`docker compose down` в его папке): заняты порты 8080/8404. Имя проекта Compose — `messenger-lab6`, volumes отдельные от lab5.
```powershell
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab6.git
cd mtusi2026lab6
git checkout solution/bvt2303-babichenko
$env:PYTHONUTF8 = "1"
Copy-Item .env.example .env
python scripts/init_secrets.py # как в lab5 + secrets/grafana_admin_password (папка secrets/ в .gitignore)
docker compose --env-file .env config --quiet
docker compose up -d --build --wait # postgres, s3, rabbitmq, otel-collector, prometheus, loki, tempo ->
# migrate, s3-init, telemetry-init -> api x2, janitor, dispatcher, worker,
# grafana -> lb
docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy
python tests/smoke.py --base-url http://127.0.0.1:8080 # make test при включённой телеметрии
# доказательства lab6 (~35 мин) в evidence/ и таблица замеров в этот отчёт:
python student-tests/collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history `
incident-worker incident-db incident-telemetry incident-collector load
```
| Что | Где |
| --- | --- |
| Grafana (dashboard «Messenger — lab6 observability», Explore, Alerting) | http://127.0.0.1:3000, логин `admin`, пароль — `secrets/grafana_admin_password` |
| Prometheus: алерты pending/firing, запросы | http://127.0.0.1:9090/alerts, http://127.0.0.1:9090/graph |
| Loki / Tempo HTTP API (для тестов) | http://127.0.0.1:3100, http://127.0.0.1:3200 |
| Трасса конкретного ответа API | заголовок ответа `X-Trace-Id` (или свой `traceparent` в запросе) → Grafana → Explore → Tempo |
Отдельные проверки (make на Windows нет):
| Команда | Что проверяет |
| --- | --- |
| `python scripts/check_contract.py` | make check |
| `python student-tests/check_configs.py` | конфиги Collector/Prometheus/Loki/Tempo их же инструментами, unit-тесты алертов (promtool), provisioning Grafana через её API |
| `python student-tests/test_telemetry.py [request upload-chain retry-chain replay-link history]` | запрос ↔ лог ↔ трасса, цепочка upload → worker, retry, replay с link, сохранность истории |
| `python student-tests/check_dashboard.py` | каждый запрос dashboard и правил выполняется на живых данных, запрещённых меток нет |
| `python student-tests/incidents.py [worker-stop db-slow telemetry-down collector-down]` | инциденты: firing → диагностика → resolved |
| `docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml` | unit-тесты порогов/окон/`for` алертов |
| `python scripts/build_dashboard.py` | пересобрать `infra/grafana/dashboards/messenger.json` из описания панелей |
Bootstrap: как в lab5 (`migrate`, `s3-init`, definitions RabbitMQ) плюс `telemetry-init` (одноразовый `chown 10001` volumes Loki и Tempo: оба процесса работают не от root). Provisioning Grafana — файлы `infra/grafana/provisioning` и `infra/grafana/dashboards`, монтируются только для чтения: источники данных и dashboard появляются при старте, ручного набора нет. Всё идемпотентно: повторный `up` ничего не пересоздаёт и не дублирует.
Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py`. Выключить телеметрию целиком (для сравнения): `OTEL_SDK_DISABLED=true` в `.env` и `docker compose up -d`.
## Архитектура и решения ## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. ```text
+--------------------------- сеть telemetry (internal) ----------------------------+
Клиент -> lb -> api x2 ---|-- OTLP/HTTP :4318 --> otel-collector --+-- traces OTLP gRPC --> tempo (tempodata) |
| | (traces, metrics, memory_limiter |-- logs OTLP HTTP --> loki (lokidata) |
| | logs) batch, queues +-- metrics OTLP HTTP --> prometheus(promdata)|
PostgreSQL <-+ outbox | ^ pull :8888 (метрики Collector)|
(источник истины) | dispatcher, worker, janitor -- OTLP --^ | |
| grafana -------+--> prometheus, loki, tempo |
+----------------------------------------------------------------------------------+
W3C traceparent: API (спан create, outbox) -> dispatcher (спан publish, AMQP headers + поле сообщения)
-> worker (спан process, consumer) -> S3/PostgreSQL; retry/replay -> новая строка outbox
127.0.0.1: 8080 (lb), 8404 (stats), 3000 (grafana), 9090 (prometheus), 3100 (loki), 3200 (tempo)
```
Перенесено из lab5 без изменения бизнес-логики и контракта: всё приложение (API, outbox, dispatcher, worker, janitor, варианты изображений, повторы, DLQ, replay), миграции, тесты lab1–5, hooks. Бизнес-API не менялся; единственное расширение — необязательный заголовок ответа `X-Trace-Id` (trace_id запроса) и приём входящего `traceparent` (обратно совместимо, клиент может не знать о них).
Что добавлено в lab6:
- `telemetry.py` — настройка OpenTelemetry SDK для всех процессов: ресурс, три провайдера (traces/metrics/logs) с OTLP/HTTP exporter, ограниченные очереди, сэмплер, JSON-логи в stderr + OTLP, метрики процесса/контейнера, W3C propagator, помощники спанов.
- Инструментирование: HTTP server-спаны и метрики в middleware `main.py`; спаны PostgreSQL (`opentelemetry-instrumentation-psycopg`, текст SQL с плейсхолдерами, без значений), S3 (`storage.py`), Redis и Argon2id (`auth.py`); messaging-спаны и перенос контекста через outbox/RabbitMQ (`pipeline.py`, `dispatcher.py`, `broker.py`, `worker.py`); метрики очереди/outbox (dispatcher), пула БД (`db.py`), занятости worker, времени до ready; структурные события логов (`event=...`).
- Конфигурации в Git: `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml`, `infra/prometheus/rules/{recording,alerts}.yml`, `infra/prometheus/tests/alerts_test.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, `infra/grafana/provisioning/**`, `infra/grafana/dashboards/messenger.json` (генерирует `scripts/build_dashboard.py`), `docs/runbooks.md`.
- `compose.yaml`: сервисы `otel-collector`, `telemetry-init`, `prometheus`, `loki`, `tempo`, `grafana`; сети `telemetry` (internal) и `ops` (только чтобы опубликовать UI на 127.0.0.1); volumes `promdata`, `lokidata`, `tempodata`, `grafanadata`; секрет `grafana_admin_password`. Restart hook перезапускает и хранилища телеметрии.
- Тесты: `student-tests/{obs.py, test_telemetry.py, check_dashboard.py, check_configs.py, incidents.py}`, обновлены `check_stack.py`, `run_persistence.py`, `collect_evidence.py`.
### Роли и назначение хранилищ
| Роль | Сервис | Версия | Зачем | Health | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Collector | `otel-collector` | otelcol-contrib 0.164.0 | приём OTLP, защита памяти, пакетирование, буфер на время недоступности хранилища, маршрутизация в 3 хранилища. **Не хранит** | метрики `otelcol_*` (scrape) — образ distroless, без shell | нет (очередь в памяти) | telemetry; 4317/4318/8888 не публикуются | нет |
| Метрики | `prometheus` | 3.15.1 | числовые ряды: «что, сколько, когда», percentiles из histogram, SLI, alert rules, состояние алертов | `/-/ready` | `promdata` (7 дней, ≤ 1 GB) | telemetry, ops; 127.0.0.1:9090 | нет |
| Логи | `loki` | 3.7.9 | события с контекстом: «почему» (error_type, request_id, trace_id) | — (`/ready` проверяют тесты) | `lokidata` (7 дней) | telemetry, ops; 127.0.0.1:3100 | нет |
| Трассы | `tempo` | 3.1.1 | причинная цепочка запроса через процессы и время каждого шага: «где» | — (`/ready`) | `tempodata` (7 дней) | telemetry, ops; 127.0.0.1:3200 | нет |
| UI | `grafana` | 12.4.14 | запросы к трём хранилищам, dashboard, связки лог ↔ трасса ↔ метрика; сама сигналы не хранит | `/api/health` | `grafanadata` (сессии/настройки UI) | telemetry, ops; 127.0.0.1:3000 | `grafana_admin_password` |
Почему так: Grafana сама не хранит сигналы, а каждому сигналу нужно своё хранилище с подходящей моделью — TSDB для рядов (Prometheus умеет OTLP напрямую, поэтому без remote-write и без exporter-а в Collector), индекс по меткам + сжатые строки для логов (Loki принимает OTLP и хранит атрибуты записи как structured metadata, поэтому `trace_id`/`request_id` не становятся метками индекса), блоки трасс по trace_id (Tempo 3 single binary без Kafka). Collector отделяет приложение от хранилищ: приложение знает один адрес, а смена/недоступность хранилища его не касается.
### Сигналы: контракт → реальные имена
Имена OTel переводит в Prometheus `translation_strategy: UnderscoreEscapingWithSuffixes` (зафиксировано в `prometheus.yml`): точки → `_`, суффикс единицы (`_seconds`, `_bytes`) и `_total` для счётчиков; `service.name` → метка `job`, `service.instance.id` → `instance`, остальные ресурсные атрибуты — в `target_info`.
| Сигнал контракта (уровень) | Инструмент OTel (единица) | В хранилище | Метки / поля |
| --- | --- | --- | --- |
| HTTP requests (3) | `http.server.request.duration` histogram, его `_count` | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`, `http_response_status_class` (`2xx`…) |
| HTTP duration (3) | то же, секунды; buckets 5 мс … 10 с | `http_server_request_duration_seconds_bucket` | те же + `le` |
| Jobs processed / failed (3) | `messenger.job.duration` histogram, `_count` (одна метрика, без дубля) | `messenger_job_duration_seconds_count` | `job_type="attachment.process.v1"`, `outcome` = ready / retry / failed / stale / lease_lost / db_error |
| Job processing duration (3) | `messenger.job.duration` (s), buckets 0,1 … 120 с | `messenger_job_duration_seconds_bucket` | те же |
| Доставки/публикации (доп.) | `messenger.messages.consumed`, `messenger.messages.published` counters | `messenger_messages_consumed_total`, `..._published_total` | `messaging_destination_name`, `outcome` (processed/noop/dead_lettered/requeued; ok/error) |
| Логи API и worker (3) | Python `logging` → OTel Logs API | Loki: метки `service_name`, `service_instance_id`, `deployment_environment_name`; structured metadata `event`, `severity_text`, `trace_id`, `span_id`, `request_id`, `job_id`, `error_type`… | в stderr та же запись одной JSON-строкой (`ts`, `severity`, `service.name`, `event`, `trace_id`, `span_id`) |
| Трассы API и worker (3) | server / producer / consumer / client / internal спаны | Tempo | `http.route`, `http.response.status_code`, `messaging.*`, `messenger.job.*`, `db.statement`, `aws.s3.key`, `error.type` |
| Queue depth / oldest job age (4) | `messenger.queue.depth` gauge; `messenger.queue.oldest_age` (s); доп. `messenger.jobs.backlog`, `messenger.queue.consumers`, `messenger.outbox.pending`, `messenger.outbox.oldest_age` (s) | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds`, … | `messaging_destination_name` (очередь/DLQ), `status` |
| DB pool used/waiting (4) | `db.client.connection.count` (`state`=used/idle), `.max`, `.pending_requests`, `.timeouts`, `db.client.connection.wait_time` (s, histogram) | `db_client_connection_*` | `db_client_connection_pool_name="messenger-api"`, `db_client_connection_state` |
| Worker busy/capacity (4) | `messenger.worker.busy`, `messenger.worker.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | `messaging_destination_name` |
| Process / container CPU, RAM (4) | `process.cpu.time` (s), `process.memory.usage` (By); `container.cpu.time`, `container.cpu.limit`, `container.memory.usage`, `container.memory.limit` (cgroup v2, только в Docker) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `container_*` | `cpu_mode` |
| End-to-end time to ready (4) | `messenger.attachment.time_to_ready` (s), buckets 0,5 … 600 с; от COMMIT задания в API до COMMIT ready, по часам PostgreSQL | `messenger_attachment_time_to_ready_seconds_*` | `job_type` |
| Сквозная связь upload → publish → consume → S3/DB (4) | W3C `traceparent`/`tracestate`: parent-child, replay — link | Tempo | см. ниже |
| Потери телеметрии (4) | SDK: `otel.sdk.processor.span.processed{error.type}`, `otel.sdk.exporter.span.exported{error.type}`; Collector: `otelcol_exporter_{sent,send_failed,enqueue_failed}_*`, `otelcol_exporter_queue_size/capacity`, `otelcol_receiver_refused_*` | Prometheus | `exporter`, `error_type` |
Ресурс (одинаков для всех трёх сигналов процесса): `service.name` = `messenger-api` / `messenger-worker` / `messenger-dispatcher` / `messenger-janitor` (и `messenger-ops` у `ops.py`) — задаётся в коде процесса, поэтому общий `.env` не делает janitor «api»; `service.version` = `APP_VERSION`; `service.instance.id` = hostname контейнера (тот же, что `X-Instance-Id`); `deployment.environment.name=lab` (`OTEL_RESOURCE_ATTRIBUTES`).
**Что не попадает в телеметрию.** Метки метрик — только перечисленные выше (метод, шаблон маршрута, класс/код статуса, тип/исход задания, очередь, состояние соединения, режим CPU); `check_dashboard.py` проверяет, что у метрик приложения нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, URL, имени файла, текста и значений-UUID. В спанах и логах есть технические ID синтетических данных (`job_id`, `attachment_id`, `request_id`, ключ S3 вида `attachments/<uuid>/v1/crop.jpg`) — без них нельзя пройти от загрузки к результату worker. Не пишутся: пароли, `Authorization`, токен сессии и его хеш (спаны Redis без ключа), текст сообщения, имя файла, байты изображения, значения параметров SQL (`capture_parameters=False`), signed URL (их нет). `test_telemetry.py request` ищет текст сообщения, пароль и токен в Loki и атрибутах спанов — 0 совпадений. В Collector дополнительно `attributes/scrub` удаляет любой атрибут, чьё имя похоже на password/authorization/cookie/secret/token/api key.
### Перенос контекста через очередь
```text
[api] POST /api/v1/chats/{chat_id}/attachments SERVER
├─ PostgreSQL INSERT attachments, S3 PutObject staging/<hex>, UPDATE ... state='queued'
└─ create attachments PRODUCER ← строка outbox в ТОЙ ЖЕ транзакции; traceparent этого спана
│ пишется в attachment_jobs.traceparent и в payload outbox
└─ [dispatcher] publish attachments PRODUCER ← родитель = traceparent из outbox; свой контекст
│ кладёт в AMQP header traceparent и в поле сообщения
└─ [worker] process attachments.process.v1 CONSUMER ← родитель = header (запасной — поле тела)
├─ claim (PostgreSQL), S3 GetObject, imaging, S3 PutObject x4, commit result (PostgreSQL)
├─ (ошибка) create attachments PRODUCER → publish → process (попытка 2) — та же трасса
└─ cleanup staging (S3 DeleteObject, UPDATE) — после ack, в той же трассе
```
Выбранная семантика: **parent-child** вдоль причинной цепочки (публикация и обработка — продолжение запроса пользователя, а задержка между ними видна как разрыв во времени). **Link** — когда это новое решение, а не продолжение: `ops.py replay` создаёт новую трассу (корень `ops replay`), спан `create` которой ссылается на исходную загрузку. Восстановление истёкшей аренды dispatcher-ом и сверка «queued без сообщения» создают спаны `recover expired lease` / `republish lost message` с родителем из `attachment_jobs.traceparent`. В lab5 `traceparent` был случайным; теперь это контекст реального спана.
Сэмплирование: `ParentBased(AlwaysOn)`, но клиентский спан без родителя не записывается — это фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, проверка соединений пула). Всё, что относится к запросу или заданию, сохраняется полностью (100%). Проверки здоровья (`/health/*`, HAProxy раз в секунду) — только в метриках, без трасс.
### SLI / SLO и окно
| SLO | SLI (числитель / знаменатель) | Цель | Окно | Recording rule |
| --- | --- | --- | --- | --- |
| SLO-1 доступность API | бизнес-запросы API без 5xx / все бизнес-запросы API (все маршруты кроме `/health/*`). 4xx — ошибки клиента, не входят в числитель | ≥ 99% | скользящий 1 ч (учебное окно; в эксплуатации то же выражение с 28d) | `messenger:slo_availability:ratio_1h`, бюджет `messenger:slo_availability:error_budget_remaining_1h` |
| SLO-2 задержка записи | успешные `POST /api/v1/chats/{chat_id}/messages` быстрее 250 мс (bucket `le="0.25"`) / все успешные такие запросы | ≥ 95% | 1 ч | `messenger:slo_latency_messages:ratio_1h` |
| SLO-3 фоновая обработка | вложения ready не позже 30 с от приёма (bucket `le="30"` гистограммы time_to_ready) / все ставшие ready | ≥ 95% | 1 ч | `messenger:slo_time_to_ready:ratio_1h` |
Проверяемость: все SLI — отношения счётчиков/buckets гистограмм, просуммированных по репликам (`sum(increase(...[1h]))`), поэтому SLO вычисляется одинаково на любом наборе реплик и видно на dashboard (строка «Обзор и SLO»).
### Dashboard: запросы и единицы
Dashboard — `infra/grafana/dashboards/messenger.json` (импортируется provisioning-ом, папка Messenger; 34 панели в 6 строках: обзор/SLO и алерты, API RED, фоновая обработка, насыщение, конвейер телеметрии, логи и трассы). Ключевые запросы (`$__rate_interval` подставляет Grafana; `API` = `job="messenger-api", http_route!~"/health/.*"`):
| Панель | PromQL | Единица |
| --- | --- | --- |
| RPS по маршрутам / по классу статуса | `sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{API}[$__rate_interval]))` | req/s |
| Доля 5xx | `(sum(rate(..._count{API, http_response_status_class="5xx"}[…])) or on() vector(0)) / sum(rate(..._count{API}[…]))` | доля (0–1) |
| p50 / p95 / p99 API | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{API}[$__rate_interval])))` | с |
| p95 по маршрутам | `histogram_quantile(0.95, sum by (le, http_route) (rate(..._bucket{API}[…])))` | с |
| Попытки заданий по исходу | `sum by (outcome) (rate(messenger_job_duration_seconds_count[…]))` | 1/с |
| Длительность попытки / время до ready p50/p95/p99 | `histogram_quantile(q, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="ready"}[…])))`, то же для `messenger_attachment_time_to_ready_seconds_bucket` | с |
| Глубина очереди и backlog | `max by (messaging_destination_name) (messenger_queue_depth)`, `max by (status) (messenger_jobs_backlog)` | сообщений / заданий |
| Возраст старейшего задания и outbox | `max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})`, `max(messenger_outbox_oldest_age_seconds)` | с |
| Пул БД | `sum by (instance) (db_client_connection_count{db_client_connection_state="used"})`, `max(db_client_connection_max)`, `sum by (instance) (db_client_connection_pending_requests)`; p95 ожидания `histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[…])))` | соединений / с |
| Насыщение worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля |
| CPU / RAM | `sum by (job, instance) (rate(process_cpu_time_seconds_total[…]))` (ядер), `rate(container_cpu_time_seconds_total) / container_cpu_limit` (доля лимита), `process_memory_usage_bytes`, `container_memory_usage_bytes` | ядра / доля / байты |
| Collector и SDK | `rate(otelcol_receiver_accepted_spans[…])`, `rate(otelcol_exporter_send_failed_spans[…])`, `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`, `otelcol_process_memory_rss`, `rate(otel_sdk_exporter_span_exported_total{error_type!=""}[…])` | 1/с, доля, байты |
| Логи / трассы | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`; Tempo TraceQL `{ status = error }` | — |
Percentile только из buckets гистограммы, просуммированных по всем репликам **до** `histogram_quantile` (`sum by (le)`); среднее и усреднение p95 реплик не используются. Точность — ширина bucket: например, при запросах по 5,0 с (statement_timeout) интерполяция внутри bucket 5–7,5 с даёт p95 ≈ 7,4 с; это ограничение метода, видно на инциденте 2.
### Алерты
`infra/prometheus/rules/alerts.yml`; состояние — Prometheus `/alerts` и `ALERTS{}` (видно и resolved по истории), Grafana показывает их как data source rules. Внешней доставки нет (по заданию).
| Алерт | Условие | Окно | Порог | for | Runbook |
| --- | --- | --- | --- | --- | --- |
| MessengerHighErrorRate | доля 5xx бизнес-запросов API и трафик > 0,2 запроса/с | rate 1m | > 5% (в 5 раз выше бюджета SLO-1) | 1m | `docs/runbooks.md#messengerhigherrorrate` |
| MessengerQueueLagGrowing | возраст старейшего готового задания | мгновенное значение (обновление 5–10 с) | > 30 с (SLO-3) | 1m | `#messengerqueuelaggrowing` |
| MessengerWorkerNoConsumers | consumers очереди = 0 | — | 0 | 1m | `#messengerworkernoconsumers` |
| MessengerDbPoolSaturated | запросы ждут соединение из пула | — | > 0 | 1m | `#messengerdbpoolsaturated` |
| TelemetryExportFailing | потери в Collector (send/enqueue failed) **или** очередь exporter заполнена > 5% | rate 2m | > 0 / > 5% | 1m | `#telemetryexportfailing` |
| TelemetryCollectorDown | `up{job="otel-collector"} == 0` | — | — | 1m | `#telemetrycollectordown` |
Пороги, окна и `for` проверены unit-тестами `infra/prometheus/tests/alerts_test.yml` (promtool: не срабатывает раньше `for`, не срабатывает при пустом трафике и на ошибках `/health`, resolved после восстановления) и живыми инцидентами ниже.
### Collector и SDK: память и недоступность хранилища
- SDK (каждый процесс): `BatchSpanProcessor` и `BatchLogRecordProcessor` с очередью 2048 (`OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_QUEUE_SIZE`), экспорт в фоновом потоке, таймаут 5 с (`OTEL_EXPORTER_OTLP_TIMEOUT`); метрики — агрегаты фиксированного размера раз в 10 с. Переполнение и непринятые пакеты считаются самим SDK (`OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true`). Ошибки экспорта пишутся в лог не чаще раза в 30 с и не отправляются тем же экспортом (иначе петля).
- Collector: первым в каждом pipeline `memory_limiter` (heap Go: жёсткий предел 200 MiB, мягкий 160 MiB — receiver отвечает клиентам ошибкой, SDK отбрасывает у себя), затем `batch` (2 с / 1024–2048 элементов). У каждого exporter своя `sending_queue` в памяти (Tempo, Loki — 200 пакетов, Prometheus — 100) и `retry_on_failure` (1 → 10 с, `max_elapsed_time` 120 с). Окно повторов считается для пакета с первой попытки; ожидание в очереди в него не входит, поэтому буфер на время недоступности — это очередь. Пакет отбрасывается, когда истекли его 120 с повторов (`otelcol_exporter_send_failed_*`) или очередь полна (`otelcol_exporter_enqueue_failed_*`). Pipeline независимы: недоступный Tempo не мешает логам и метрикам. Очередь не персистентная: рестарт Collector теряет буфер (осознанный компромисс учебного стенда, `file_storage` не нужен).
- Контейнер Collector 384 MiB: RSS = heap (≤ 200 MiB) + runtime Go + отображённый в память бинарник contrib (~150 MiB вытесняемых страниц); поэтому тест сравнивает с лимитом именно heap (`otelcol_process_runtime_heap_alloc_bytes`), а RSS/память контейнера приводит для сведения.
### Соответствие `.env.example`
| Переменная | Что делает в этом решении |
| --- | --- |
| всё из lab5 | как в lab5 (см. REPORT lab5) |
| OTEL_SERVICE_NAME | `messenger-api` для api; compose задаёт свой каждому сервису; фактически `service.name` берётся из кода процесса |
| OTEL_EXPORTER_OTLP_ENDPOINT / _PROTOCOL | `http://otel-collector:4318`, `http/protobuf` (SDK добавляет `/v1/traces`, `/v1/metrics`, `/v1/logs`) |
| OTEL_RESOURCE_ATTRIBUTES | `deployment.environment.name=lab` |
| OTEL_SDK_DISABLED | `true` — SDK выключен во всех процессах (логи остаются JSON в stderr); для сравнения telemetry off/on |
| (только compose) OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_METRIC_EXPORT_INTERVAL, OTEL_BSP_MAX_QUEUE_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED | 5 с, 10000 мс, 2048, 2048, `true` |
## Доказательства критериев ## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | `python student-tests/collect_evidence.py …` пишет вывод каждого шага в `evidence/<шаг>.txt` (дата, commit, код выхода), итог — `evidence/summary.md`, JSON инцидентов — `evidence/incident-*.json`, а таблицы замеров — в блок «Замеры (Docker)» ниже. Те же сценарии прогнаны в нативной песочнице (те же версии Collector/Prometheus/Loki/Tempo, процессы без Docker; см. «Где проверено») — `evidence/native-sandbox/`.
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. | Критерий README | Команда/сценарий | Наблюдаемый результат (Docker-стенд, `evidence/`; то же в нативной песочнице — `evidence/native-sandbox/`) | Файл доказательства |
| --- | --- | --- | --- |
| 3: минимум lab5 работает, `make test` при включённой телеметрии | `tests/smoke.py` | PASS через балансировщик с SDK во всех процессах | smoke.txt |
| 3: Collector, 3 хранилища и Grafana в одном воспроизводимом стенде; конфиги/provisioning в Git; persistent volumes | `check_stack.py`, `check_configs.py`, `test_telemetry.py history` | конфиги валидны инструментами тех же версий; unit-тесты алертов SUCCESS; после restart prometheus/loki/tempo прежние трасса, логи и точка метрики находятся | stack.txt, configs.txt, telemetry-history.txt |
| 3: API и worker отдают OTLP; RPS/errors/duration API, jobs ok/failed и duration worker; логи и реальные трассы | `check_dashboard.py`, `test_telemetry.py` | все 54 PromQL-выражения dashboard и 16 выражений правил выполняются без ошибок, ключевые сигналы с данными; трассы API, dispatcher, worker | dashboard.txt, telemetry.txt |
| 3: импортируемый dashboard; низкая кардинальность; trace/span id в логах; нет содержимого/секретов | `check_dashboard.py`, `check_configs.py` (Grafana API), `test_telemetry.py request` | метки только из белого списка, `http_route` — шаблоны; dashboard provisioned=true; текст, пароль, токен: 0 совпадений в Loki и спанах | dashboard.txt, configs.txt, telemetry.txt |
| 3: контролируемый запрос → trace + лог; job → worker trace + лог | `test_telemetry.py request upload-chain` | свой `traceparent` → server-спан с дочерними PostgreSQL/Redis → лог `event=http.request` с тем же trace_id/span_id и request_id = `X-Request-Id`; логи worker `job.claimed`/`job.ready` с trace_id загрузки | telemetry.txt |
| 3: назначение каждого хранилища | раздел «Роли и назначение хранилищ» | — | REPORT.md |
| 4: W3C через API → broker → worker, producer/consumer, цепочка до S3/DB, включая retry | `test_telemetry.py upload-chain retry-chain replay-link` | одна трасса: server → create (producer) → publish (producer, dispatcher) → process (consumer, worker) → 4×S3 PutObject + commit; retry: попытка 1 ERROR `storage_unavailable` → create → publish → попытка 2 ready; replay — новая трасса с link на исходную | telemetry.txt |
| 4: p50/p95/p99 из histogram, доля ошибок, queue depth/oldest age, DB pool, saturation worker; запросы и единицы | dashboard + раздел «Dashboard: запросы и единицы» | все панели выполняются на живых данных | dashboard.txt |
| 4: SLI/SLO и окно; alert rules high error rate и queue lag; проверены отказом (firing/resolved) | `incidents.py worker-stop db-slow`, `promtool test rules` | см. таблицы инцидентов: firing и resolved по каждому алерту | incident-worker.txt, incident-db.txt, configs.txt |
| 4: memory limiter/batch, ограниченная буферизация; остановка хранилища телеметрии не ломает сообщения и не растит RAM | `incidents.py telemetry-down collector-down` | API 100% успешных ответов, heap Collector ограничен, потери посчитаны | incident-telemetry.txt, incident-collector.txt |
| 4: разбор двух инцидентов и шаги диагностики | раздел «Эксперименты и отказы» | — | REPORT.md, docs/runbooks.md |
| Выбранный трек 5 | — | не выбран | — |
### Где проверено
**Нативная песочница (выполнено).** Среда, где писалось решение, не имеет доступа к Docker Hub, поэтому весь стенд был поднят теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API за балансировщиком, dispatcher, worker, janitor (Python 3.11, те же `requirements.txt`) и **те же версии** otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из `infra/` (заменены только адреса `tempo:4317` → `127.0.0.1:14317` и т.п.). Тесты запускались без изменений через тонкую подмену команды `docker compose` (stop/start/exec на процессы). Результаты — `evidence/native-sandbox/`, цифры — в разделе ниже. Grafana в песочнице не запускалась (нет бинарника без Docker Hub): её provisioning проверяет `check_configs.py` через HTTP API Grafana на Docker-стенде, а все запросы dashboard выполнены против живых Prometheus/Loki/Tempo (`check_dashboard.py`).
**Docker-стенд (Windows 10, Docker Desktop 28.1.1, Compose 2.35.1; VM 16 CPU / 15,2 GiB) — выполнено 2026-11-01.** Стенд поднят из чистого состояния (`init_secrets.py`, `docker compose pull`, `up -d --build --wait`) без правок кода и конфигов: `migrate`, `s3-init`, `telemetry-init` — Exited (0), остальные running/healthy; Loki и Tempo пишут в volumes от uid 10001 без ошибок прав. Полный прогон `collect_evidence.py` — все шаги PASS: lab6 (check, stack, smoke, configs, telemetry, dashboard, telemetry-history, 4 инцидента, load) и регрессия lab1–5 (`--from jobs-variants`, 22 шага, включая failover и persistence; restart hook вместе с ожиданием готовности — 60,9 с при лимите 90 с). Вывод — `evidence/*.txt`, сводка — `evidence/summary.md`, таблицы — блок «Замеры (Docker)» ниже. Grafana проверена через её API: 3 источника данных healthy, dashboard provisioned, трасса по `X-Trace-Id` открывается через источник Tempo, лог с тем же `trace_id` — через Loki (связки tracesToLogs и derived field TraceID).
## Эксперименты и отказы ## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. Гипотезы:
1. По сохранённым метрикам, логам и трассам можно найти причину инцидента, не заходя в контейнеры: остановленный worker отличается от медленной БД, а медленная БД — от медленного обработчика.
2. Недоступность любого звена телеметрии (хранилище, Collector) не влияет на ответы API, память Collector и процессов приложения ограничена, а потери телеметрии считаются.
Методика: `student-tests/incidents.py` — нагрузка (closed loop, POST сообщений), контролируемый отказ, опрос `/api/v1/alerts` каждые 5 с до firing, «диагностика» только запросами к Prometheus/Loki/Tempo, устранение отказа, ожидание resolved; хронология в выводе, итог в `evidence/incident-*.json` (без токенов и текстов).
### Нативная песочница: результаты
Окружение: Linux x86_64, 2 vCPU, 8 GiB RAM на всё (стенд, хранилища телеметрии и клиент нагрузки); PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, Python 3.11; otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 (те же версии и конфиги, что в compose.yaml). Дата прогона 2026-11-01. Сводка шагов (все PASS, включая регрессию lab1–5: варианты изображений, outbox, повторы/DLQ/replay, конкуренция worker, вложения, идемпотентность, гонки) — `evidence/native-sandbox/summary.md`.
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
| --- | --- | --- | --- | --- |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) |
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) |
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) |
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0.
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок.
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
| --- | --- | --- | --- |
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с |
| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) |
Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной).
### Инцидент 1: остановлен worker («задания не обрабатываются»)
Воздействие: `docker compose stop worker`, затем 3 загрузки вложений (API отвечает 202 queued — приём от worker не зависит).
Шаги диагностики (как в `docs/runbooks.md`):
1. **Алерт** `MessengerWorkerNoConsumers` (consumers = 0), затем `MessengerQueueLagGrowing` (возраст старейшего задания > 30 с). Dashboard «Возраст старейшего задания» растёт линейно — задания не забирают, а не «их стало много».
2. **Публикация или обработка?** `messenger_outbox_oldest_age_seconds` = 0 и `messenger_messages_published_total{outcome="ok"}` растёт → dispatcher и RabbitMQ работают, сообщения в брокере (`messenger_queue_depth` = числу загрузок).
3. **Кто не забирает?** `messenger_queue_consumers` = 0, метрики `messenger_worker_*` перестали обновляться (worker не экспортирует), последний лог worker в Loki — `event=service.stopped` (штатная остановка, а не падение — тогда был бы `job.handler_crashed`/обрыв без stopped).
4. **Трасса любой из загрузок** (trace_id из лога API `event=http.request` или `X-Trace-Id`): есть `publish attachments`, нет `process attachments.process.v1` — сообщение доставлено в очередь, но не взято.
5. Причина: worker остановлен. Действие: `docker compose start worker`. Проверка: вложения ready через 2–5 с (Docker 4,7 с, песочница 2,0 с), backlog → 0, оба алерта resolved (Docker — через 29,7 с после start).
### Инцидент 2: медленная БД (блокировка таблицы `messages`) vs медленный обработчик
Воздействие: под нагрузкой 8 потоков `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` в отдельной транзакции на 210 с (PostgreSQL жив, `/health/ready` 200, но каждая запись ждёт блокировку и через `statement_timeout` 5 с отменяется → API 503).
Шаги диагностики:
1. **Алерт** `MessengerHighErrorRate`. Срабатывает не сразу: окно rate 1m заполнено быстрыми запросами до инцидента (~115–130/с), а во время блокировки closed-loop нагрузка даёт ~1,6 запроса/с (каждый висит 5 с), поэтому доля > 5% появляется только через 50–80 с, плюс `for: 1m` (firing через 140,8 с от LOCK в Docker, 120,1 с в песочнице).
2. **Какие маршруты и коды**: только `POST /api/v1/chats/{chat_id}/messages`, код 503; чтения других таблиц (`GET /api/v1/users/me`, `GET /api/v1/chats`) во время блокировки 5–20 мс (Docker 17,2 мс, песочница 4,7 мс) → не «всё медленно», а конкретная таблица/операция.
3. **БД или обработчик — по трассе**: TraceQL `{ resource.service.name = "messenger-api" && status = error }` → server-спан ≈ 5005–5015 мс, из них дочерний `INSERT` (PostgreSQL) ≈ 5001 мс со статусом ERROR — 99,8–99,9% времени запрос ждал БД. Для контраста медленный **обработчик**: `POST /api/v1/users` — server 20–35 мс, из них `argon2id hash` 16–29 мс (CPU в процессе API), все спаны БД < 1 мс. По метрикам то же различие: при медленной БД растут `db_client_connection_count{state="used"}` (соединения заняты ожиданием) и p95 конкретного маршрута, при медленном обработчике пул свободен, а растёт CPU процесса API (`process_cpu_time_seconds_total`).
4. **Почему — по логам**: `{service_name="messenger-api"} | event="db.unavailable"` с `error_type=QueryCanceled` (statement_timeout), а не `OperationalError` (БД недоступна) или `PoolTimeout` (нет соединений).
5. **Подтверждение в PostgreSQL**: `pg_stat_activity` показывает сессии с `wait_event_type=Lock` и держателя блокировки (`LOCK TABLE ...`). Действие: завершить транзакцию-держателя. Проверка: 5xx → 0, алерт resolved через 30–50 с после снятия блокировки (Docker 50,1 с, песочница 30,0 с: окно rate 1m заполняется успешными запросами, затем ближайшая оценка правила).
### Инцидент 3: недоступна телеметрия (Tempo + Loki; затем сам Collector)
Ожидаем: API не замечает отказа; Collector копит пакеты в ограниченной очереди, heap не выходит за `memory_limiter`; `TelemetryExportFailing` firing → resolved; после восстановления очередь досылается, потери посчитаны. Без Collector: SDK каждого процесса отбрасывает лишнее (очередь 2048), память API не растёт, потери видны в `otel_sdk_exporter_span_exported_total{error_type!=""}` после восстановления.
Что показал эксперимент (числа — в таблицах выше):
- Tempo + Loki остановлены на 150 с под нагрузкой ~60 записей/с (Docker; в песочнице ~68/с): все ответы API 201 (9555 запросов), p95 не изменился (31 мс до и во время отказа). Очереди exporter Tempo/Loki росли (метрики продолжали идти в Prometheus — pipeline независимы), heap Collector вырос с 35,5 до max 86,9 MiB (песочница 44 → 93 MiB) — далеко от предела 200 MiB, память контейнера Collector не выше 107 MiB; `TelemetryExportFailing` перешёл в firing через 73 с (песочница ~100 с) — по заполнению очереди, ещё до первых потерь. После start очереди досланы за 60 с (песочница ~80 с), алерт resolved через 166 с после start (песочница ~155 с: окно rate 2m по счётчику потерь — последние пакеты с истёкшим окном повторов отброшены около момента восстановления, плюс `for`); все 13 пробных трасс, отправленных во время отказа, нашлись в Tempo (песочница 14 из 15). Потеряно и посчитано Collector-ом 4096 спанов и 235 записей логов (песочница 4096 и 269) — пакеты, у которых истекло окно повторов.
- Collector остановлен на 90 с: все ответы API 201 (5852 запроса), память двух реплик API 295 → max 314 MiB (+18 MiB; песочница +12 MiB) и после восстановления не растёт. Экспорт в SDK получает ошибку соединения, пакет отбрасывается (буфера на диске нет). В Docker остановленный контейнер исчезает из DNS сети, и каждая попытка экспорта висит до таймаута 5 с. Экспорт становится медленнее, чем API создаёт спаны, поэтому ограниченная очередь процессора (2048) переполняется и лишнее отбрасывается сразу: за отказ потеряно и посчитано 8252 спана `processor_queue_full` и 10752 `exporter_failed` (`otel_sdk_processor_span_processed_total`, `otel_sdk_exporter_span_exported_total` с `error_type`). В песочнице соединение отклонялось мгновенно: 24,6 тыс. `exporter_failed`, очередь не переполнялась. В обоих случаях память ограничена очередью и ответы API не замедлились (p95 31–32 мс); после start экспорт возобновился без перезапуска приложения (10287 спанов за минуту).
- Вывод: отказ любого звена телеметрии не затрагивает пользователей и не растит память без предела; потери ограничены временем отказа и видны в метриках. Короткую недоступность хранилища (до заполнения очереди Collector, здесь минуты) Collector переживает почти без потерь, недоступность самого Collector — нет.
<!-- measurements:begin -->
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
| --- | --- | --- | --- | --- |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) |
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) |
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) |
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0.
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок.
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
| --- | --- | --- | --- |
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) |
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
| --- | --- | --- | --- | --- | --- |
| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 |
| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 |
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 |
| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 |
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 |
| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 |
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 |
| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 |
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| TODO | TODO | TODO | TODO | | публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет |
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет |
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия |
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready |
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены |
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
<!-- measurements:end -->
## Ограничения и следующие шаги ## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. - Выдерживается: недоступность любого хранилища телеметрии и самого Collector — API и обработка вложений продолжают работать; память ограничена очередями SDK (2048 на сигнал и процесс) и Collector (memory_limiter, очереди фиксированного размера). Переживают рестарт хранилищ (volumes): метрики, логи, трассы, настройки Grafana.
- Теряется (и считается): телеметрия, отправленная, пока Collector недоступен (SDK не буферизует на диск); пакеты, у которых истекло окно повторов Collector или очередь переполнена; буфер Collector при его рестарте. Метрики приложения за время недоступности Collector — пропуск на графиках (cumulative-счётчики после восстановления продолжают с верного итога).
- Один экземпляр каждого хранилища и Collector на одном Docker-host — нет HA телеметрии; при отказе Prometheus алерты не вычисляются (нужен внешний «мета-мониторинг» — вне учебного стенда).
- Метрики очереди считает один dispatcher: если он упал, значения `messenger_queue_*` устаревают (видно по отсутствию свежих точек); алерт `absent()` на них — следующий шаг.
- Учебные упрощения: внутренние соединения OTLP и UI без TLS (lab7), Prometheus/Loki/Tempo опубликованы на 127.0.0.1 для тестов и защиты; сэмплирование трасс 100% (для учебной нагрузки; стоимость и баланс сэмплирования — трек 5.1, не заявлен); burn-rate алерты с несколькими окнами — трек 5.2, не заявлен.
- Подтверждено измерением: алерты срабатывают и снимаются на контролируемых отказах; диагностика отличает остановленный worker, медленную БД и медленный обработчик; отказ телеметрии не влияет на API. Гипотеза без измерения: накладные расходы телеметрии на задержку и CPU (сравнение `OTEL_SDK_DISABLED=true/false` при одинаковой нагрузке) — трек 5.1.
## Ответы на вопросы защиты (кратко)
- **Лог, метрика и trace об одном инциденте.** Метрика отвечает «что и когда, насколько массово» (доля 5xx выросла в 14:02, только на POST messages) — дёшево, агрегатно, для алертов. Трасса — «где именно» в одном запросе (5 с из 5,004 с — `INSERT` в PostgreSQL), через все процессы. Лог — «почему» с деталями события (`QueryCanceled`, request_id, job_id). Связаны через `trace_id`: из алерта → error-трасса → её логи.
- **Почему `chat_id` в labels может исчерпать хранилище.** Каждая уникальная комбинация меток — отдельный временной ряд со своим индексом и чанками в памяти. Чатов — неограниченное и растущее число, умноженное на маршруты, коды и buckets гистограммы → миллионы рядов, рост RAM/диска TSDB и медленные запросы. Поэтому в метрики — только ограниченные множества, а ID — в трассы/логи.
- **Как очередь переносит причинную связь между процессами.** Producer записывает W3C `traceparent` (trace_id + id своего спана) в сообщение — здесь в строку outbox, затем dispatcher в AMQP header и поле тела; consumer извлекает его и начинает свой спан с этим родителем (или с link). Так trace_id один для HTTP-запроса и фоновой обработки, хотя между ними секунды и разные процессы.
- **Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик.** Среднее прячет хвост: 99 запросов по 10 мс и один по 5 с дают среднее 60 мс, а пользователь видит 5 с. Перцентиль — не аддитивная величина: p99 двух реплик с разной нагрузкой нельзя усреднить (реплика с 10 запросами весила бы как реплика с 10 000). Правильно — сложить buckets гистограмм всех реплик (`sum by (le)`) и уже из суммы вычислить `histogram_quantile`.
+80
View File
@@ -0,0 +1,80 @@
"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202.
Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого
объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение:
1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503.
2. PUT исходника в staging/<hex> — упало: удаляем объект (PUT мог пройти)
и строку, клиенту 503. Если и это не вышло —
uploading-строку уберёт janitor после grace.
3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT
— упало/неизвестно: клиенту 503, строка
остаётся uploading -> janitor удалит объект
и строку. Если COMMIT всё же прошёл, задание
есть и будет обработано (повтор клиента даст
второе вложение — в контракте нет ключа).
4. 202 Attachment(status=queued) — только после шага 3.
Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный
брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно.
"""
import logging
from uuid import UUID, uuid4
import db
import settings
import storage
from errors import unavailable
from uploads import ImageInfo, Upload
log = logging.getLogger("messenger.attachments")
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
attachment_id = uuid4()
key = storage.new_key(uuid4().hex)
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
upload.size, upload.sha256, image.width, image.height, key)
try:
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
except storage.StorageUnavailable:
compensate(attachment_id, key)
raise unavailable("File storage is temporarily unavailable") from None
row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID)
if row is None:
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
log.warning("Upload %s was reclaimed before it was queued", attachment_id)
raise unavailable()
log.info("Attachment %s accepted into staging and queued", attachment_id)
return {**row, "variants": []}
def compensate(attachment_id: UUID, key: str) -> None:
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
try:
storage.delete(key, quick=True)
db.delete_upload_intent(attachment_id)
log.info("Upload %s rolled back after S3 failure", attachment_id)
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
def public(row: dict) -> dict:
"""Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей)."""
return {
"id": row["id"],
"chat_id": row["chat_id"],
"owner_id": row["owner_id"],
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["state"],
"variants": [
{key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for v in row.get("variants", [])
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
+150
View File
@@ -0,0 +1,150 @@
import base64
import binascii
import hashlib
import logging
import secrets
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from uuid import UUID
import redis
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from redis.backoff import NoBackoff
from redis.retry import Retry
import settings
import telemetry
from errors import BASIC_CHALLENGE, unauthorized, unavailable
log = logging.getLogger("messenger.auth")
hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1)
DUMMY_HASH = hasher.hash("there-is-no-such-user")
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
def hash_password(password: str) -> str:
# Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан.
with telemetry.internal_span("argon2id hash"):
return hasher.hash(password)
def password_matches(password_hash: str | None, password: str) -> bool:
with telemetry.internal_span("argon2id verify"):
try:
hasher.verify(password_hash or DUMMY_HASH, password)
except (VerificationError, InvalidHashError):
return False
return password_hash is not None
def parse_basic(header: str | None) -> tuple[str, str] | None:
"""'Basic base64(username:password)' -> (username, password) или None.
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
"""
scheme, _, value = (header or "").strip().partition(" ")
if scheme.lower() != "basic" or not value or len(value) > 1024:
return None
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None
username, colon, password = decoded.partition(":")
if not colon or not username or not username.isascii():
return None
return username, password
def parse_bearer(header: str | None) -> str | None:
scheme, _, value = (header or "").strip().partition(" ")
value = value.strip()
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
return None
return value
KEY_PREFIX = "session:"
# Атрибуты спанов Redis: система и операция. Ключ (session:<SHA-256 токена>) и значения не пишутся.
REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"}
_redis = redis.Redis(
connection_pool=redis.BlockingConnectionPool.from_url(
settings.REDIS_URL or "redis://localhost:6379/0",
max_connections=settings.REDIS_POOL_MAX,
timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
)
@dataclass(frozen=True)
class Session:
user_id: UUID
key: str
def session_key(token: str) -> str:
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
def create_session(user_id: UUID) -> tuple[str, datetime]:
"""Новая сессия: случайный токен и абсолютное время истечения."""
token = secrets.token_urlsafe(32)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
try:
with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}):
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
except redis.RedisError as exc:
log.warning("Redis is unavailable on login: %s", type(exc).__name__,
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
raise unavailable() from None
return token, expires_at
def check_bearer(header: str | None) -> Session:
token = parse_bearer(header)
if token is None:
raise unauthorized("Bearer token is required")
key = session_key(token)
try:
with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}):
user_id = _redis.get(key)
except redis.RedisError as exc:
log.warning("Redis is unavailable: %s", type(exc).__name__,
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
raise unavailable() from None
if user_id is None:
raise unauthorized("Session is invalid or expired")
return Session(UUID(user_id), key)
def delete_session(session: Session) -> None:
try:
with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}):
_redis.delete(session.key)
except redis.RedisError:
raise unavailable() from None
def ping_redis() -> None:
try:
_redis.ping()
except redis.RedisError:
raise unavailable() from None
def login_failed():
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
def close() -> None:
_redis.close()
_redis.connection_pool.disconnect()
+104
View File
@@ -0,0 +1,104 @@
"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms.
Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует
scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure:
они только проверяют её пассивным declare и не могут создать/удалить очередь.
exchange attachments (direct, durable)
--attachment.process.v1--> queue attachments.process.v1 (quorum, durable)
x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once),
x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря)
exchange attachments.dlx (direct, durable)
--attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py
через basic_get + nack не расходует delivery-limit)
Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до
подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются.
"""
import json
import logging
import pika
from pika.exceptions import AMQPError
import pipeline
import settings
log = logging.getLogger("messenger.broker")
logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал
BrokerError = (AMQPError, OSError)
def parameters(url: str | None = None) -> pika.URLParameters:
params = pika.URLParameters(url or settings.BROKER_URL)
params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS
params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно
params.socket_timeout = 5
params.connection_attempts = 1
params.client_properties = {"connection_name": settings.INSTANCE_ID}
return params
def connect(url: str | None = None) -> pika.BlockingConnection:
return pika.BlockingConnection(parameters(url))
def check_topology(channel) -> dict:
"""Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях."""
channel.exchange_declare(pipeline.EXCHANGE, passive=True)
main = channel.queue_declare(pipeline.QUEUE, passive=True).method
dlq = channel.queue_declare(pipeline.DLQ, passive=True).method
return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count}
def properties(message: dict) -> pika.BasicProperties:
# W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы).
headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message}
return pika.BasicProperties(
delivery_mode=pika.DeliveryMode.Persistent,
content_type="application/json",
message_id=f"{message['job_id']}:{message['attempt']}",
type=message["type"],
headers=headers or None,
)
class Publisher:
"""Канал в режиме confirm: basic_publish возвращает управление только после ack брокера.
mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError.
Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях
строка outbox остаётся неопубликованной и будет отправлена позже.
"""
def __init__(self):
self.connection = None
self.channel = None
def open(self):
if self.channel is None or self.channel.is_closed or self.connection.is_closed:
self.close()
self.connection = connect()
self.channel = self.connection.channel()
self.channel.confirm_delivery()
check_topology(self.channel)
log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"})
return self.channel
def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY):
body = json.dumps(message, separators=(",", ":")).encode("utf-8")
self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True)
def pump(self):
if self.connection is not None and self.connection.is_open:
self.connection.process_data_events(time_limit=0) # heartbeat в паузах
def close(self):
try:
if self.connection is not None and self.connection.is_open:
self.connection.close()
except BrokerError:
pass
self.connection = None
self.channel = None
+494
View File
@@ -0,0 +1,494 @@
name: messenger-lab6
x-app-build: &app-build
context: .
args:
APP_VERSION: ${APP_VERSION:-dev}
# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py.
x-pipeline-env: &pipeline-env
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
# Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults,
# доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0.
FAULT_INJECTION: ${FAULT_INJECTION:-1}
FAULTS_DIR: /faults
# lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry).
# Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется
# в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком.
x-otel-env: &otel-env
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс
OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются
OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
x-otel-depends: &otel-depends
# Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется.
condition: service_started
required: false
x-alive-check: &alive-check
# Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад.
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
interval: 10s
timeout: 3s
retries: 3
start_period: 20s
start_interval: 1s
x-s3-api-secrets: &s3-api-secrets
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
services:
migrate:
build: *app-build
command: ["python", "migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
s3:
image: chrislusf/seaweedfs:3.80
entrypoint: ["/usr/bin/weed"]
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
secrets: [s3_config]
volumes: ["s3data:/data"]
networks: [data]
healthcheck:
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
interval: 5s
timeout: 3s
retries: 24
start_period: 15s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
s3-init:
build: *app-build
command: ["python", "s3_init.py"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
s3: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
api:
build: *app-build
env_file: .env
environment:
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL
OTEL_SERVICE_NAME: messenger-api
<<: [*s3-api-secrets, *otel-env]
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [edge, data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
redis: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "1.00", memory: 512M}
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
janitor:
build: *app-build
command: ["python", "janitor.py"]
env_file: .env
environment:
CACHE_URL_FILE: "" # janitor не ходит в Redis
BROKER_URL_FILE: "" # и в брокер
OTEL_SERVICE_NAME: messenger-janitor
<<: [*s3-api-secrets, *otel-env]
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 192M}
# Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права —
# в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте.
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
rabbitmq:
image: rabbitmq:4.1.4-alpine
hostname: rabbitmq
volumes:
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
- rabbitmqdata:/var/lib/rabbitmq
secrets: [rabbitmq_definitions]
networks: [data]
healthcheck:
# Узел запущен, и vhost с очередями загружен (а не только «порт открыт»).
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
interval: 5s
timeout: 10s
retries: 24
start_period: 20s
start_interval: 2s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает
# просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет.
dispatcher:
build: *app-build
command: ["python", "dispatcher.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url_dispatcher
OTEL_SERVICE_NAME: messenger-dispatcher
<<: [*pipeline-env, *otel-env]
secrets: [database_url, broker_url_dispatcher]
volumes: ["faults:/faults"]
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
# Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем
# процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS.
# Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker.
worker:
build: *app-build
command: ["python", "worker.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url_worker
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
OTEL_SERVICE_NAME: messenger-worker
<<: [*pipeline-env, *s3-api-secrets, *otel-env]
secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key]
volumes: ["faults:/faults"]
tmpfs:
- /tmp:size=256m,mode=1777
networks: [data, telemetry]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
otel-collector: *otel-depends
healthcheck: *alive-check
restart: unless-stopped
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят;
# если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова.
stop_grace_period: 70s
deploy:
resources:
limits: {cpus: "2.00", memory: 1536M}
lb:
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8404:8404"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks: [edge]
depends_on:
api: {condition: service_healthy}
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
postgres:
image: postgres:17.11-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
secrets: [postgres_password]
volumes: ["pgdata:/var/lib/postgresql/data"]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
redis:
image: redis:7.4.11-alpine
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
volumes:
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
- redisdata:/data
secrets: [redis_auth]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
# ---------------- lab6: наблюдаемость ----------------
# OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter,
# отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry).
# Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus.
otel-collector:
image: otel/opentelemetry-collector-contrib:0.164.0
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
networks: [telemetry]
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
# heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib.
limits: {cpus: "0.50", memory: 384M}
# Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root).
telemetry-init:
image: busybox:1.37.0
user: root
command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"]
volumes:
- lokidata:/loki
- tempodata:/tempo
network_mode: none
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 32M}
# Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules.
# 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите.
prometheus:
image: prom/prometheus:v3.15.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml
- promdata:/prometheus
ports:
- "127.0.0.1:9090:9090"
networks: [telemetry, ops]
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 24
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов.
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
ports:
- "127.0.0.1:3100:3100"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов.
tempo:
image: grafana/tempo:3.1.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
ports:
- "127.0.0.1:3200:3200"
networks: [telemetry, ops]
depends_on:
telemetry-init: {condition: service_completed_successfully}
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git
# (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password.
grafana:
image: grafana/grafana:12.4.14
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
networks: [telemetry, ops]
depends_on:
prometheus: {condition: service_healthy}
loki: {condition: service_started}
tempo: {condition: service_started}
healthcheck:
# wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих.
test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 24
start_period: 20s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 384M}
networks:
edge: {}
data:
internal: true
# lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется.
telemetry:
internal: true
# Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты).
ops: {}
volumes:
pgdata: {}
redisdata: {}
s3data: {}
rabbitmqdata: {}
faults: {}
promdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
redis_auth: {file: ./secrets/redis_auth}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
s3_config: {file: ./secrets/s3_config.json}
broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher}
broker_url_worker: {file: ./secrets/broker_url_worker}
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
grafana_admin_password: {file: ./secrets/grafana_admin_password}
+452
View File
@@ -0,0 +1,452 @@
import logging
import time
from contextlib import contextmanager
from datetime import datetime
from uuid import UUID, uuid4
import psycopg
from psycopg import errors as pg_errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import pipeline
import settings
import telemetry
from errors import bad_request, conflict, not_found, unavailable
log = logging.getLogger("messenger.db")
def check_connection(conn) -> None:
"""Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса."""
from opentelemetry.instrumentation.utils import suppress_instrumentation
with suppress_instrumentation():
ConnectionPool.check_connection(conn)
pool = ConnectionPool(
conninfo=settings.DATABASE_URL,
min_size=settings.DB_POOL_MIN,
max_size=settings.DB_POOL_MAX,
timeout=settings.DB_POOL_WAIT_SECONDS,
kwargs={
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
"row_factory": dict_row,
"options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
},
check=check_connection,
open=False,
)
# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»).
POOL_NAME = {"db.client.connection.pool.name": "messenger-api"}
POOL_WAIT = telemetry.meter.create_histogram(
"db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool")
def _pool_stats():
try:
return pool.get_stats()
except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет
return None
def _observe_connections(_options):
from opentelemetry.metrics import Observation
stats = _pool_stats()
if not stats:
return []
size, available = stats.get("pool_size", 0), stats.get("pool_available", 0)
return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}),
Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})]
def _observe_stat(key):
def callback(_options):
from opentelemetry.metrics import Observation
stats = _pool_stats()
return [Observation(stats.get(key, 0), POOL_NAME)] if stats else []
return callback
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.count", [_observe_connections], unit="{connection}",
description="Connections in the pool by state (used/idle)")
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}",
description="Maximum pool size (DB_POOL_MAX)")
telemetry.meter.create_observable_up_down_counter(
"db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}",
description="Requests waiting for a free connection now")
telemetry.meter.create_observable_counter(
"db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}",
description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS")
@contextmanager
def connection(wait_seconds: float | None = None):
try:
started = time.perf_counter()
try:
with pool.connection(timeout=wait_seconds) as conn:
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
yield conn
except PoolTimeout:
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
raise
except (psycopg.OperationalError, PoolTimeout) as exc:
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__,
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
raise unavailable() from None
except psycopg.DataError as exc:
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__,
extra={"event": "db.rejected_value", "error.type": type(exc).__name__})
raise bad_request("Invalid request data") from None
TEXT_FIELDS = ("display_name", "title", "text", "filename")
def to_db(value: str) -> bytes:
return value.encode("utf-8")
def from_db(row: dict | None) -> dict | None:
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
if row is None:
return None
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
def ping() -> None:
with connection(wait_seconds=1) as conn:
conn.execute("SELECT 1")
MIGRATIONS_LOCK = 20260202
def migration_files() -> list:
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
def run_migrations() -> list[str]:
files = migration_files()
applied = []
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
try:
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now())""")
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
for path in files:
if path.name in done:
continue
with conn.transaction():
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
applied.append(path.name)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
return applied
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
applied = run_migrations()
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
return
except psycopg.OperationalError:
if time.monotonic() > deadline:
raise
log.warning("PostgreSQL is not ready yet, retry in 1 s")
time.sleep(1)
def missing_migrations() -> list[str]:
expected = [path.name for path in migration_files()]
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
try:
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
except pg_errors.UndefinedTable:
return expected
return [name for name in expected if name not in done]
def wait_for_schema(max_wait_seconds: int) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
missing = missing_migrations()
except psycopg.OperationalError:
missing = None
if missing == []:
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
return
if time.monotonic() > deadline:
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
else "missing " + ", ".join(missing))
time.sleep(1)
USER_COLUMNS = "id, username, display_name, created_at"
def create_user(username: str, display_name: str, password_hash: str) -> dict:
try:
with connection() as conn:
return from_db(conn.execute(
f"""INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
(uuid4(), username, to_db(display_name), password_hash),
).fetchone())
except pg_errors.UniqueViolation:
raise conflict("Username already exists") from None
def get_user(user_id: UUID) -> dict | None:
with connection() as conn:
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
def get_user_with_password(username: str) -> dict | None:
with connection() as conn:
return from_db(conn.execute(
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
).fetchone())
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
def create_chat(title: str, member_ids: list[UUID]) -> dict:
chat_id = uuid4()
members = sorted(member_ids)
try:
with connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, to_db(title)),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member_id) for member_id in members],
)
except pg_errors.ForeignKeyViolation:
raise bad_request("Unknown member") from None
return {**from_db(row), "member_ids": members}
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
with connection() as conn:
chat = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
WHERE c.id = %s""",
(user_id, chat_id),
).fetchone()
if chat is None:
raise not_found("Chat not found")
return from_db(chat)
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
where, params = "", [user_id]
if after is not None:
where = "WHERE (c.created_at, c.id) > (%s, %s)"
params += list(after)
with connection() as conn:
rows = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
{where}
ORDER BY c.created_at, c.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
).fetchone() is not None
def check_member(chat_id: UUID, user_id: UUID) -> None:
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
idempotency_key: str, request_hash: bytes) -> dict:
try:
with connection() as conn:
with conn.transaction():
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
raise not_found("Chat not found")
if attachment_ids:
# Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409).
# FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние.
found = conn.execute(
"""SELECT id, state FROM attachments
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""",
(attachment_ids, chat_id),
).fetchall()
if len(found) != len(attachment_ids):
raise not_found("Attachment not found")
failed = any(row["state"] == "failed" for row in found)
else:
failed = False
saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s"""
if failed:
# Повтор уже созданного сообщения возвращает прежний 201, даже если вложение
# с тех пор стало failed. Новое сообщение с failed-вложением — 409.
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
if saved is None:
raise conflict("Failed attachment cannot be attached")
created = None
else:
created = conn.execute(
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
RETURNING id, chat_id, sender_id, text, created_at""",
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
).fetchone()
if created is None:
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
if created is not None:
if attachment_ids:
with conn.cursor() as cur:
cur.executemany(
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES (%s, %s, %s, %s)""",
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
)
return {**from_db(created), "attachment_ids": list(attachment_ids)}
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
raise unavailable()
if bytes(saved.pop("request_hash")) != request_hash:
raise conflict("Idempotency-Key was already used with a different request")
return from_db(saved)
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
where, params = "m.chat_id = %s", [chat_id]
if after is not None:
where += " AND (m.created_at, m.id) > (%s, %s)"
params += list(after)
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
rows = conn.execute(
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
WHERE {where}
ORDER BY m.created_at, m.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
# ---------- вложения ----------
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, "
"pipeline_version, created_at, updated_at, object_key")
VISIBLE_STATES = ("queued", "processing", "ready", "failed")
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None:
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging.
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД).
"""
try:
with connection() as conn:
conn.execute(
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, source_key, state)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
width, height, source_key),
)
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None:
"""Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией.
После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL,
сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace).
"""
with connection() as conn:
with conn.transaction():
row = conn.execute(
f"""UPDATE attachments SET state = 'queued', updated_at = now()
WHERE id = %s AND state = 'uploading'
RETURNING {ATTACHMENT_COLUMNS}""",
(attachment_id,),
).fetchone()
if row is None:
return None
pipeline.enqueue_new(conn, attachment_id, actor)
return from_db(row)
def delete_upload_intent(attachment_id: UUID) -> None:
with connection(wait_seconds=1) as conn:
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
"""Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404.
Возвращает строку и варианты текущей версии pipeline (они есть только у ready).
"""
with connection() as conn:
row = conn.execute(
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
FROM attachments a
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
WHERE a.id = %s AND a.state = ANY(%s)""",
(user_id, attachment_id, list(VISIBLE_STATES)),
).fetchone()
if row is None:
raise not_found("Attachment not found")
variants = []
if row["state"] == "ready":
variants = conn.execute(
"""SELECT name, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s
ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""",
(attachment_id, row["pipeline_version"]),
).fetchall()
return {**from_db(row), "variants": variants}
+260
View File
@@ -0,0 +1,260 @@
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
Цикл:
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
"""
import logging
import signal
import threading
import time
from pathlib import Path
import psycopg
from opentelemetry.trace import SpanKind
from psycopg.rows import dict_row
import settings
import telemetry
telemetry.setup("messenger-dispatcher")
import broker # noqa: E402
import faults # noqa: E402
import pipeline # noqa: E402
log = logging.getLogger("messenger.dispatcher")
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
# только читает последний снимок (не ходит в БД из потока экспорта).
STATS_SECONDS = 5
STATS = {"at": 0.0, "db": None, "broker": None}
PUBLISHED = telemetry.meter.create_counter(
"messenger.messages.published", unit="{message}",
description="Outbox rows published to the broker (outcome: ok/error)")
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
def _fresh(kind):
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
def _observe(kind, build):
def callback(_options):
from opentelemetry.metrics import Observation
data = _fresh(kind)
return [Observation(value, attrs) for value, attrs in build(data)] if data else []
return callback
telemetry.meter.create_observable_gauge(
"messenger.queue.depth", [_observe("broker", lambda d: [
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
telemetry.meter.create_observable_gauge(
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
unit="{consumer}", description="Consumers subscribed to the queue")
telemetry.meter.create_observable_gauge(
"messenger.jobs.backlog", [_observe("db", lambda d: [
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
(d["processing"], {"status": "processing"})])],
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
telemetry.meter.create_observable_gauge(
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
telemetry.meter.create_observable_gauge(
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
unit="{message}", description="Outbox rows due but not yet published")
telemetry.meter.create_observable_gauge(
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
def collect_stats(conn, publisher) -> None:
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
if time.monotonic() - STATS["at"] < STATS_SECONDS:
return
db_stats = conn.execute(
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
count(*) FILTER (WHERE status = 'processing') AS processing,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
outbox = conn.execute(
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
coalesce(extract(epoch FROM now() - min(available_at)
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
FROM outbox WHERE published_at IS NULL""").fetchone()
STATS["db"] = {**db_stats, **outbox}
try:
STATS["broker"] = broker.check_topology(publisher.open())
except broker.BrokerError:
STATS["broker"] = None
publisher.close()
STATS["at"] = time.monotonic()
STOP = threading.Event()
ACTOR = "dispatcher@" + settings.INSTANCE_ID
def connect_db():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def publish_due(conn, publisher: broker.Publisher) -> int:
published = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, job_id, attempt, payload FROM outbox
WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
(settings.DISPATCHER_BATCH,),
).fetchall()
if not rows:
return published
for row in rows:
publish_one(conn, publisher, row)
published += 1
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
if len(rows) < settings.DISPATCHER_BATCH:
return published
return published
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
message = dict(row["payload"])
job_id, attempt = message.get("job_id"), message.get("attempt")
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
"messaging.destination.name": pipeline.EXCHANGE,
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
kind=SpanKind.PRODUCER, attributes=attributes,
record_exception=False, set_status_on_exception=False) as span:
carrier = telemetry.inject()
if carrier.get("traceparent"):
message["traceparent"] = carrier["traceparent"]
message.pop("tracestate", None)
if carrier.get("tracestate"):
message["tracestate"] = carrier["tracestate"]
faults.crash("dispatcher_crash_before_publish")
try:
publisher.publish(message)
except broker.BrokerError as exc:
telemetry.mark_error(span, type(exc).__name__)
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
"error.type": type(exc).__name__})
publisher.close()
raise
faults.crash("dispatcher_crash_after_confirm")
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
def note_failure(conn) -> None:
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
try:
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
WHERE published_at IS NULL AND available_at <= now()""")
except psycopg.Error:
pass
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def run() -> None:
publisher = broker.Publisher()
conn = None
last_leases = last_reconcile = 0.0
pause = 1.0
while not STOP.is_set():
try:
if conn is None or conn.closed:
conn = connect_db()
conn.execute("LISTEN outbox")
log.info("connected to PostgreSQL, listening for outbox notifications",
extra={"event": "db.connected"})
touch_alive()
publish_due(conn, publisher)
collect_stats(conn, publisher)
now = time.monotonic()
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
last_leases = now
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
log.warning("recovered %s job(s) with expired lease", recovered,
extra={"event": "lease.recovered", "count": recovered})
if now - last_reconcile >= 30:
last_reconcile = now
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
log.warning("re-published %s queued job(s) without a message in flight", lost,
extra={"event": "outbox.republished", "count": lost})
pause = 1.0
publisher.pump()
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
pass
except broker.BrokerError:
if conn is not None and not conn.closed:
note_failure(conn)
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
extra={"event": "broker.unavailable"})
STOP.wait(pause)
pause = min(pause * 2, 15)
except psycopg.Error as exc:
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
try:
if conn is not None:
conn.close()
except psycopg.Error:
pass
conn = None
STOP.wait(pause)
pause = min(pause * 2, 15)
publisher.close()
if conn is not None:
conn.close()
log.info("stopped", extra={"event": "service.stopped"})
telemetry.shutdown()
def main() -> None:
if not settings.DATABASE_URL or not settings.BROKER_URL:
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
run()
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
# Runbooks алертов (lab6)
Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`.
Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules).
Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability».
Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**.
Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу».
Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`).
---
## MessengerHighErrorRate
**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты,
при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят.
**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы.
**Найти причину.**
1. Какие маршруты и коды: dashboard «RPS по классу статуса» или
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`.
503 — недоступна зависимость, 500 — ошибка кода.
2. Медленная БД или медленный обработчик:
- `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и
`db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами;
- Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу:
если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой —
проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике;
`redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS.
3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` —
statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна);
`| event="redis.unavailable"`, `| event="s3.unavailable"`.
4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c
"SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity
WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы.
**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость
(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём.
**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты
(окно rate 1m). Пример разбора: REPORT.md, инцидент 2.
---
## MessengerQueueLagGrowing
**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`,
`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно.
**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается.
**Найти причину.**
1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён
(см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях.
2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт
`messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`).
3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`;
логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`).
4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до
брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`.
5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments):
есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял.
**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить
параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed.
**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`,
алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1.
---
## MessengerWorkerNoConsumers
**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`.
**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki
(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру;
отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен).
**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь).
---
## MessengerDbPoolSaturated
**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10).
**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки
соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`).
Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS.
---
## TelemetryExportFailing
**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь
exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются —
у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов
считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) —
это и есть буфер на время недоступности.
**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики
`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`).
**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`;
логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер).
**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна
повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3.
---
## TelemetryCollectorDown
**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`).
**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов.
Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}`
(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за
этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов.
**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят.
+40
View File
@@ -0,0 +1,40 @@
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
class ApiError(Exception):
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found(message: str) -> ApiError:
return ApiError(404, "not_found", message)
def conflict(message: str) -> ApiError:
return ApiError(409, "conflict", message)
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
return ApiError(503, "unavailable", message)
def payload_too_large(message: str) -> ApiError:
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(message: str) -> ApiError:
return ApiError(415, "unsupported_media_type", message)
+96
View File
@@ -0,0 +1,96 @@
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
# date: 2026-11-01 20:38:49, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_attachments.py
# exit code: 0, 10.3 s
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
exactly 10485760 bytes (10 MiB) -> 202, then ready
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
PASS: the 10 MiB limit is exact
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
PASS: oversized streamed body is cut off, the API keeps working
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 23200a7c
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
PASS: pixel limit is checked before decompression
PNG cut in half (55908 bytes, image/png) -> 415
PNG without IEND (111805 bytes, image/png) -> 415
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
PNG with garbage after IEND (111824 bytes, image/png) -> 415
PNG signature only (8 bytes, image/png) -> 415
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
JPEG cut in half (746 bytes, image/jpeg) -> 415
JPEG header only (20 bytes, image/jpeg) -> 415
random bytes (1600 bytes, image/png) -> 415
plain text (28 bytes, image/jpeg) -> 415
empty body -> 400
PASS: damaged and foreign files are rejected with 415, never 500
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
empty PLTE (79 bytes, valid CRCs) -> 415
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
palette image without PLTE (67 bytes, valid CRCs) -> 415
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
PLTE after IDAT (85 bytes, valid CRCs) -> 415
two PLTE chunks (103 bytes, valid CRCs) -> 415
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
tRNS before PLTE (98 bytes, valid CRCs) -> 415
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
IEND with data (70 bytes, valid CRCs) -> 415
unknown critical chunk (81 bytes, valid CRCs) -> 415
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
gAMA after IDAT (85 bytes, valid CRCs) -> 415
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
private chunk after IDAT (181 bytes, valid CRCs) -> 415
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
palette depth 2 with 4 entries -> 201, downloaded byte for byte
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
PNG bytes sent as image/jpeg -> 415
JPEG bytes sent as image/png -> 415
GIF bytes sent as image/png -> 415
PNG sent as image/gif -> 415
PNG sent as text/html -> 415
PNG sent as application/octet-stream -> 415
PNG sent as multipart/form-data -> 415
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
PASS: the type is taken from the content; a declared type that does not match is refused
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
no filename -> 400
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
PASS: filename never becomes a path; dangerous names are refused
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
upload into someone else's / unknown chat -> 404, without token -> 401
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
PASS: only chat members can see an attachment
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
attachment of another chat (author is a member of both) -> 404
own + foreign attachment -> 404
unknown attachment_id -> 404
outsider referencing an attachment of the chat -> 404
duplicate ids -> 400
11 ids -> 400
bad UUID -> 400
empty text without attachments -> 400
whitespace text without attachments -> 400
PASS: only attachments of the same chat can be linked; rejected requests change nothing
same key + same attachments -> same 201; other attachments -> 409; missing field == []
PASS: Idempotency-Key covers attachment_ids
sent 1.1 MB of a declared 5 MB body and closed the connection
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
PASS: an interrupted upload is dropped
12 passed, 0 failed
+18
View File
@@ -0,0 +1,18 @@
# balancing: balancing, shared sessions/data/files, X-Instance-Id
# date: 2026-11-01 20:42:35, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_balancing.py
# exit code: 0, 2.1 s
20 x GET /users/me with one token -> replicas: {'1db2fedff78f': 10, '171a473b0d28': 10}
PASS: one session works on every replica, no second login (sessions are shared in Redis)
message written by 171a473b0d28; read back by ['171a473b0d28', '1db2fedff78f']
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
logout on 1db2fedff78f; old token then rejected by ['171a473b0d28', '1db2fedff78f']
PASS: logout on one replica revokes the session on all of them
first POST on 1db2fedff78f -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae; same key on 171a473b0d28 -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
file uploaded via 1db2fedff78f; downloaded byte for byte via ['171a473b0d28', '1db2fedff78f']
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['171a473b0d28', '1db2fedff78f']; api containers: ['171a473b0d28', '1db2fedff78f']
PASS: X-Instance-Id is in every response and points to a real container
6 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# check: offline contract check (make check)
# date: 2026-11-01 20:07:43, commit: 470d9d6 + uncommitted changes
# command: python.exe scripts/check_contract.py
# exit code: 0, 0.1 s
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+32
View File
@@ -0,0 +1,32 @@
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
# date: 2026-11-01 20:07:53, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/check_configs.py
# exit code: 0, 5.5 s
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
Checking /etc/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /etc/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool check rules (recording + alerts): exit 0
Checking /etc/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /etc/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
SUCCESS
--- loki -verify-config (infra/loki/loki.yaml): exit 0
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
--- Grafana 12.4.14: database ok
datasource prometheus (prometheus): OK Successfully queried the Prometheus API.
datasource loki (loki): OK Data source successfully connected.
datasource tempo (tempo): OK Data source is working
dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo']
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps
+47
View File
@@ -0,0 +1,47 @@
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
# date: 2026-11-01 20:41:39, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_consistency.py
# exit code: 0, 159.9 s
declared 9 000 000 bytes, sent 211817, closed the connection
declared 9 000 000 bytes, sent 3111817, closed the connection
rows in attachments: 63 -> 63; objects in bucket: 240 -> 240
temporary upload files in api containers: {'171a473b0d28': 0, '1db2fedff78f': 0} (removed when the request ends)
api containers have no volumes: the original is never stored locally
PASS: an interrupted upload leaves no row, no object and no local file
crash after S3 PUT: row b1065ae1 state=uploading, object staging/b5763f64729a4b8a8c899bcd7009e599 exists in S3
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running)
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.132} -> nothing deleted
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.101} -> row and object deleted
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.134} -> idempotent
the ready attachment of the same chat is untouched
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
object staging/23223961df3644dbbaa0fa4f40d57b82 put into the bucket without a row in PostgreSQL
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.103} -> fresh object kept
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.123} -> reported, kept
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 0, "seconds": 0.134} -> orphan deleted
ready attachments older than the grace period are untouched (they have rows)
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=1 and keeps the row; download -> 503 (not 500)
PASS: orphans are found and deleted only after the grace period; missing objects are reported
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
upload -> 503 unavailable in 20234 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose start s3
after S3 recovery the old file downloads byte for byte
new upload -> 201
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 259, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.142}
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
--- S3 frozen (no answer, timeouts): docker compose pause s3
upload -> 503 unavailable in 14703 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose unpause s3
after S3 recovery the old file downloads byte for byte
new upload -> 201
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 267, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192}
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
5 passed, 0 failed
+36
View File
@@ -0,0 +1,36 @@
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
# date: 2026-11-01 20:09:12, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/check_dashboard.py
# exit code: 0, 1.0 s
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
empty now: CPU контейнеров / лимит
empty now: Collector: потери и отказы
empty now: SDK приложений: экспорт и потери
empty now: alerts.yml: MessengerDbPoolSaturated
empty now: alerts.yml: MessengerHighErrorRate
empty now: alerts.yml: MessengerQueueLagGrowing
empty now: alerts.yml: MessengerWorkerNoConsumers
empty now: alerts.yml: TelemetryCollectorDown
empty now: alerts.yml: TelemetryExportFailing
empty now: Алерты Prometheus (pending/firing)
empty now: Память: RSS процесса и контейнер
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 4 trace(s)
data: http_server_request_duration_seconds_count{job="messenger-api"} 48 series <- HTTP requests/duration (3)
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
data: messenger_queue_depth 2 series <- queue depth (4)
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
data: db_client_connection_count 4 series <- DB pool used/idle (4)
data: db_client_connection_pending_requests 2 series <- DB pool waiting (4)
data: messenger_worker_busy 1 series <- worker busy (4)
data: messenger_worker_capacity 1 series <- worker capacity (4)
data: process_cpu_time_seconds_total 12 series <- process CPU (4)
data: process_memory_usage_bytes 6 series <- process RAM (4)
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
+21
View File
@@ -0,0 +1,21 @@
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
# date: 2026-11-01 20:42:53, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_db.py
# exit code: 0, 11.6 s
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 71 -> 72
PASS: chat row was inserted and then rolled back together with members
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
PASS: the database itself rejects a message from a non-member
stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97)
PASS: password is stored as Argon2id hash
unique constraint on users: UNIQUE (username)
PASS: username is unique in the database
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date)
PASS: migrations are applied once, a repeated run does nothing
api replicas that checked the schema at start: 10; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago
PASS: only the migrate service applies migrations, replicas wait for the schema
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
7 passed, 0 failed
+109
View File
@@ -0,0 +1,109 @@
=== Data volume
messages_total | messages_in_big_chat
----------------+----------------------
68448 | 15000
=== History index
indexdef
----------------------------------------------------------------------------------------
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
QUERY PLAN
------------------------------------------------------------------------------------------------
Limit (actual time=0.007..0.019 rows=51 loops=1)
Buffers: shared hit=54
-> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1)
Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
Buffers: shared hit=54
Planning:
Buffers: shared hit=8
Planning Time: 0.067 ms
Execution Time: 0.026 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.007..0.012 rows=51 loops=1)
Buffers: shared hit=5
-> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1)
Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)))
Buffers: shared hit=5
Planning Time: 0.083 ms
Execution Time: 0.047 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.024..0.176 rows=51 loops=1)
Buffers: shared hit=59
-> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1)
Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)))
Buffers: shared hit=59
SubPlan 1
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
Sort Key: a."position"
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Recheck Cond: (message_id = m.id)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Index Cond: (message_id = m.id)
Buffers: shared hit=51
Planning:
Buffers: shared hit=63
Planning Time: 0.210 ms
Execution Time: 0.193 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
QUERY PLAN
--------------------------------------------------------------------------------
Limit (actual time=4.087..4.090 rows=51 loops=1)
Buffers: shared hit=1255
-> Sort (actual time=3.670..3.904 rows=10051 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 2377kB
Buffers: shared hit=1255
-> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1)
Filter: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
Rows Removed by Filter: 53448
Buffers: shared hit=1255
Planning Time: 0.041 ms
Execution Time: 4.136 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=1.375..1.378 rows=51 loops=1)
Buffers: shared hit=1006
-> Sort (actual time=1.374..1.376 rows=51 loops=1)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 31kB
Buffers: shared hit=1006
-> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1)
Recheck Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))
Rows Removed by Filter: 10000
Heap Blocks: exact=991
Buffers: shared hit=1006
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1)
Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
Buffers: shared hit=15
Planning:
Buffers: shared hit=4
Planning Time: 0.057 ms
Execution Time: 1.395 ms
=== ROLLBACK: test data removed, index is back
explain_users_left
--------------------
0
+46
View File
@@ -0,0 +1,46 @@
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
# date: 2026-11-01 20:42:57, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/explain.py
# exit code: 0, 1.2 s
=== Data volume
=== History index
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
Buffers: shared hit=54
-> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1)
Buffers: shared hit=54
Buffers: shared hit=8
Execution Time: 0.026 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
Buffers: shared hit=5
-> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1)
Buffers: shared hit=5
Execution Time: 0.047 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
Buffers: shared hit=59
-> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1)
Buffers: shared hit=59
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
Buffers: shared hit=63
Execution Time: 0.193 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
Buffers: shared hit=1255
Buffers: shared hit=1255
-> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1)
Buffers: shared hit=1255
Execution Time: 4.136 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
Buffers: shared hit=1006
Buffers: shared hit=1006
-> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1)
Buffers: shared hit=1006
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1)
Buffers: shared hit=15
Buffers: shared hit=4
Execution Time: 1.395 ms
=== ROLLBACK: test data removed, index is back
PASS: full plans saved to evidence\explain-history.txt
+33
View File
@@ -0,0 +1,33 @@
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
# date: 2026-11-01 20:43:39, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/run_failover.py
# exit code: 0, 42.0 s
hook: scripts\stop-one.cmd; api replicas: ['171a473b0d28', '1db2fedff78f']
=== run 1: api containers before
SERVICE CONTAINER ID STATUS
api 171a473b0d28 Up 41 seconds (healthy)
api 1db2fedff78f Up 41 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.75}␍
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api 171a473b0d28 Exited (0) 11 seconds ago
api 1db2fedff78f Up 58 seconds (healthy)
the hook stopped 171a473b0d28 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
restored: docker start 171a473b0d28 -> healthy and back in HAProxy
=== run 2: api containers before
SERVICE CONTAINER ID STATUS
api 171a473b0d28 Up 2 seconds (healthy)
api 1db2fedff78f Up About a minute (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.578}␍
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api 171a473b0d28 Up 19 seconds (healthy)
api 1db2fedff78f Exited (0) 11 seconds ago
the hook stopped 1db2fedff78f (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
restored: docker start 1db2fedff78f -> healthy and back in HAProxy
PASS: public failover test passed 2 times; every api replica was stopped once and restored
+29
View File
@@ -0,0 +1,29 @@
# failures: Redis and PostgreSQL outages via the balancer
# date: 2026-11-01 20:45:06, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_failures.py
# exit code: 0, 55.6 s
stopping redis
/health/live -> 200, /health/ready -> 503 (8016 ms)
valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked
unknown token -> 503, not 200: nobody is trusted without a check
no token -> 401
login -> 503: there is nowhere to store the session
registration -> 201: it needs only PostgreSQL
starting redis
after start: the old token works, confirmed messages are in place
PASS: Redis outage gives 503 and does not let anyone in without a session check
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
right after the crash: GET /users/me -> 503
starting redis
after start: the session issued before the crash still works (AOF, appendfsync always)
PASS: sessions survive a Redis crash, not only a clean restart
stopping postgres
/health/live -> 200, /health/ready -> 503 (1016 ms)
valid token: users/me, chats, history, new message -> 503 unavailable, no details
no token -> 401, unknown token -> 401: anonymous users are not let in
login -> 503, registration -> 503
starting postgres
after start: the old token works, confirmed messages are in place
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
3 passed, 0 failed
+22
View File
@@ -0,0 +1,22 @@
# idempotency: Idempotency-Key, parallel same key via replicas
# date: 2026-11-01 20:42:37, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_idempotency.py
# exit code: 0, 2.2 s
first -> 201; same JSON written differently -> 201; plain retry -> 201
PASS: a retry with the same key returns the original 201 and message, no duplicate
20 parallel POSTs with one key: statuses {201: 20}, replicas {'171a473b0d28': 10, '1db2fedff78f': 10}, different bodies in answers: 1, messages in history: 1
PASS: parallel requests with one key through different replicas create exactly one message
20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B']
PASS: the first body wins, the same body gets 201, the other body gets 409
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
key used by a 400 and by a 404 first, then valid request -> 201
PASS: rejected requests (400, 404) do not store a result for the key
same key and body by an outsider -> 404; by alice after logout -> 401
PASS: a retry checks the session and chat access again, the saved answer is not leaked
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
PASS: an unconfirmed POST can be safely retried with the same key
8 passed, 0 failed
+39
View File
@@ -0,0 +1,39 @@
{
"scenario": "collector-down",
"date": "2026-11-01 20:25:18",
"outage_s": 90,
"baseline": {
"requests": 870,
"codes": {
"201": 870
},
"success_ratio": 1.0,
"success_rps": 58.0,
"p95_ms_successful": 32.0
},
"during": {
"requests": 5852,
"codes": {
"201": 5852
},
"success_ratio": 1.0,
"success_rps": 57.9,
"p95_ms_successful": 31.0
},
"api_mib_before": 295.2,
"api_mib_max": 313.5,
"sdk_lost_spans_api": {
"processor_queue_full": 8252,
"exporter_failed": 10752
},
"sdk_exported_ok_after_restart": 10287,
"after": {
"requests": 1403,
"codes": {
"201": 1403
},
"success_ratio": 1.0,
"success_rps": 56.1,
"p95_ms_successful": 32.0
}
}
+20
View File
@@ -0,0 +1,20 @@
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
# date: 2026-11-01 20:27:53, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/incidents.py collector-down
# exit code: 0, 155.8 s
=== collector-down
[ 24.6s] baseline: {'requests': 870, 'codes': {'201': 870}, 'success_ratio': 1.0, 'success_rps': 58.0, 'p95_ms_successful': 32.0}, api containers 295.2 MiB
[ 25.3s] docker compose stop otel-collector
[ 29.3s] {'t': 0.7, 'api_mib': 292.8}
[ 43.6s] {'t': 14.7, 'api_mib': 298.9}
[ 58.0s] {'t': 29.0, 'api_mib': 306.5}
[ 72.6s] {'t': 43.4, 'api_mib': 307.8}
[ 87.0s] {'t': 58.0, 'api_mib': 308.2}
[ 101.0s] {'t': 72.4, 'api_mib': 313.5}
[ 115.0s] {'t': 86.4, 'api_mib': 313.3}
[ 125.6s] docker compose start otel-collector
[ 155.6s] API during collector outage: {'requests': 5852, 'codes': {'201': 5852}, 'success_ratio': 1.0, 'success_rps': 57.9, 'p95_ms_successful': 31.0}; api memory 295.2 -> max 313.5 MiB; SDK lost spans (api): {'processor_queue_full': 8252, 'exporter_failed': 10752}; exported OK in the last minute after restart: 10287
[ 155.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-collector-down.json
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
1 passed, 0 failed
+82
View File
@@ -0,0 +1,82 @@
{
"scenario": "db-slow",
"date": "2026-11-01 20:13:54",
"lock_seconds": 210,
"baseline": {
"requests": 2310,
"codes": {
"201": 2310
},
"success_ratio": 1.0,
"success_rps": 115.5,
"p95_ms_successful": 32.0,
"server_p95_s": 0.019
},
"firing_after_lock_s": 140.8,
"diagnosis": {
"http_5xx_ratio_1m": 1.0,
"http_p95_messages_s": 7.375,
"pool_used": 8.0,
"pool_wait_p95_s": 0.001,
"status_codes_1m": {
"201": 0.0,
"200": 0.0,
"404": 0.0,
"400": 0.0,
"409": 0.0,
"503": 1.6
},
"reads_during_lock_ms": {
"GET /users/me + GET /chats, avg per request": 17.2,
"max": 23.5
},
"api_log_db_unavailable_by_error_type": {
"QueryCanceled": 216
},
"error_trace": {
"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2",
"server_ms": 5013.3,
"slowest_db_span": "INSERT",
"slowest_db_ms": 5001.3,
"db_span_status": "ERROR",
"db_share": 0.998,
"tree": [
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503",
" - [api] redis GET (CLIENT, 1.1 ms)",
" - [api] SELECT (CLIENT, 1.6 ms)",
" - [api] INSERT (CLIENT, 5001.3 ms) ERROR"
]
},
"handler_cpu_contrast": {
"trace_id": "4a222e478c2d51006538a7bfeb485eb1",
"server_ms": 21.2,
"argon2id_ms": 16.4,
"db_ms_total": 0.7,
"tree": [
"- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201",
" - [api] argon2id hash (INTERNAL, 16.4 ms)",
" - [api] INSERT (CLIENT, 0.7 ms)"
]
}
},
"during_lock": {
"requests": 328,
"codes": {
"503": 320,
"201": 8
},
"success_ratio": 0.0244,
"success_rps": 0.0,
"p95_ms_successful": 2156.0
},
"resolved_after_unlock_s": 50.1,
"after": {
"requests": 5269,
"codes": {
"201": 5269
},
"success_ratio": 1.0,
"success_rps": 116.8,
"p95_ms_successful": 31.0
}
}
+64
View File
@@ -0,0 +1,64 @@
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
# date: 2026-11-01 20:18:39, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/incidents.py db-slow
# exit code: 0, 285.7 s
=== db-slow
[ 0.0s] load: 8 threads POST /messages
[ 25.0s] baseline: {'requests': 2310, 'codes': {'201': 2310}, 'success_ratio': 1.0, 'success_rps': 115.5, 'p95_ms_successful': 32.0, 'server_p95_s': 0.019}
[ 25.0s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
[ 25.1s] 5xx ratio=None p95=0.019 s pool used=0.0 waiting=0.0
[ 30.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0
[ 35.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0
[ 40.1s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0
[ 45.2s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0
[ 50.2s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0
[ 55.3s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0
[ 60.3s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
[ 65.3s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
[ 70.4s] 5xx ratio=0.0 p95=0.045 s pool used=8.0 waiting=0.0
[ 75.4s] 5xx ratio=0.015 p95=7.18 s pool used=8.0 waiting=0.0
[ 80.4s] 5xx ratio=0.015 p95=7.234 s pool used=8.0 waiting=0.0
[ 85.4s] 5xx ratio=0.015 p95=7.375 s pool used=8.0 waiting=0.0
[ 90.4s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
[ 95.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
[ 100.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
[ 105.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 110.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 115.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 120.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 125.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 130.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 135.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 140.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 145.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 150.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 155.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 160.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 165.8s] ALERT MessengerHighErrorRate firing (140.8 s after lock)
[ 172.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 17.2, "max": 23.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 216}}
[ 172.2s] error trace (DB): {"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", "server_ms": 5013.3, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.3, "db_span_status": "ERROR", "db_share": 0.998}
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503
- [api] redis GET (CLIENT, 1.1 ms)
- [api] SELECT (CLIENT, 1.6 ms)
- [api] INSERT (CLIENT, 5001.3 ms) ERROR
[ 172.2s] contrast, slow handler (CPU): {"trace_id": "4a222e478c2d51006538a7bfeb485eb1", "server_ms": 21.2, "argon2id_ms": 16.4, "db_ms_total": 0.7}
- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201
- [api] argon2id hash (INTERNAL, 16.4 ms)
- [api] INSERT (CLIENT, 0.7 ms)
[ 235.5s] lock released (psql exit 0), 210.5 s after LOCK
[ 235.5s] ... MessengerHighErrorRate: firing (0 s)
[ 240.5s] ... MessengerHighErrorRate: firing (5 s)
[ 245.5s] ... MessengerHighErrorRate: firing (10 s)
[ 250.5s] ... MessengerHighErrorRate: firing (15 s)
[ 255.5s] ... MessengerHighErrorRate: firing (20 s)
[ 260.6s] ... MessengerHighErrorRate: firing (25 s)
[ 265.6s] ... MessengerHighErrorRate: firing (30 s)
[ 270.6s] ... MessengerHighErrorRate: firing (35 s)
[ 275.6s] ... MessengerHighErrorRate: firing (40 s)
[ 280.6s] ... MessengerHighErrorRate: firing (45 s)
[ 285.6s] ALERT MessengerHighErrorRate resolved 50.1 s after unlock
[ 285.6s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2156.0}; after {'requests': 5269, 'codes': {'201': 5269}, 'success_ratio': 1.0, 'success_rps': 116.8, 'p95_ms_successful': 31.0}
[ 285.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-db-slow.json
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
1 passed, 0 failed
+273
View File
@@ -0,0 +1,273 @@
{
"scenario": "telemetry-down",
"date": "2026-11-01 20:18:47",
"stopped": [
"tempo",
"loki"
],
"outage_s": 150,
"baseline": {
"requests": 899,
"codes": {
"201": 899
},
"success_ratio": 1.0,
"success_rps": 59.9,
"p95_ms_successful": 31.0
},
"collector_heap_before_mib": 35.5,
"collector_container_before_mib": 75.9,
"during": {
"requests": 9555,
"codes": {
"201": 9555
},
"success_ratio": 1.0,
"success_rps": 59.6,
"p95_ms_successful": 31.0
},
"samples": [
{
"t": 0.0,
"collector_heap_mib": 56.4,
"collector_rss_mib": 237.6,
"collector_container_mib": 76.0,
"queue": {
"otlp_http/loki": 12.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 5.0
},
"alert": "inactive"
},
{
"t": 12.1,
"collector_heap_mib": 45.4,
"collector_rss_mib": 237.8,
"collector_container_mib": 76.0,
"queue": {
"otlp_http/loki": 20.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 9.0
},
"alert": "pending"
},
{
"t": 24.2,
"collector_heap_mib": 54.2,
"collector_rss_mib": 237.8,
"collector_container_mib": 76.0,
"queue": {
"otlp_http/loki": 27.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 12.0
},
"alert": "pending"
},
{
"t": 36.2,
"collector_heap_mib": 54.2,
"collector_rss_mib": 237.8,
"collector_container_mib": 78.1,
"queue": {
"otlp_http/loki": 35.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 15.0
},
"alert": "pending"
},
{
"t": 48.7,
"collector_heap_mib": 62.9,
"collector_rss_mib": 240.4,
"collector_container_mib": 82.2,
"queue": {
"otlp_http/loki": 35.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 15.0
},
"alert": "pending"
},
{
"t": 60.8,
"collector_heap_mib": 51.2,
"collector_rss_mib": 244.4,
"collector_container_mib": 82.2,
"queue": {
"otlp_http/loki": 42.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 19.0
},
"alert": "pending"
},
{
"t": 73.1,
"collector_heap_mib": 59.9,
"collector_rss_mib": 244.4,
"collector_container_mib": 82.3,
"queue": {
"otlp_http/loki": 50.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 22.0
},
"alert": "firing"
},
{
"t": 85.2,
"collector_heap_mib": 70.5,
"collector_rss_mib": 248.4,
"collector_container_mib": 88.7,
"queue": {
"otlp_http/loki": 57.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 26.0
},
"alert": "firing"
},
{
"t": 97.7,
"collector_heap_mib": 70.5,
"collector_rss_mib": 248.4,
"collector_container_mib": 94.5,
"queue": {
"otlp_http/loki": 63.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 27.0
},
"alert": "firing"
},
{
"t": 109.8,
"collector_heap_mib": 56.3,
"collector_rss_mib": 256.7,
"collector_container_mib": 94.5,
"queue": {
"otlp_http/loki": 63.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 27.0
},
"alert": "firing"
},
{
"t": 122.1,
"collector_heap_mib": 67.5,
"collector_rss_mib": 256.7,
"collector_container_mib": 94.5,
"queue": {
"otlp_http/loki": 70.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 31.0
},
"alert": "firing"
},
{
"t": 134.2,
"collector_heap_mib": 76.1,
"collector_rss_mib": 256.9,
"collector_container_mib": 98.8,
"queue": {
"otlp_http/loki": 78.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 34.0
},
"alert": "firing"
},
{
"t": 146.2,
"collector_heap_mib": 86.9,
"collector_rss_mib": 267.1,
"collector_container_mib": 107.0,
"queue": {
"otlp_http/loki": 85.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 38.0
},
"alert": "firing"
}
],
"collector_heap_max_mib": 86.9,
"collector_rss_max_mib": 267.1,
"collector_container_max_mib": 107.0,
"TelemetryExportFailing_firing_at_s": 73.1,
"queue_drained_after_start_s": 60.1,
"probe_traces": [
{
"sent_at_s": 0.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 12.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 24.2,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 36.2,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 48.7,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 60.8,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 73.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 85.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 97.7,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 109.8,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 122.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 134.2,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 146.2,
"http": 201,
"trace_in_tempo": true
}
],
"lost_in_collector": {
"spans": 4096,
"log_records": 235
},
"resolved_after_start_s": 165.6,
"after": {
"requests": 9370,
"codes": {
"201": 9370
},
"success_ratio": 1.0,
"success_rps": 58.3,
"p95_ms_successful": 31.0
}
}
+47
View File
@@ -0,0 +1,47 @@
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
# date: 2026-11-01 20:25:06, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/incidents.py telemetry-down
# exit code: 0, 379.8 s
=== telemetry-down
[ 22.5s] baseline: {'requests': 899, 'codes': {'201': 899}, 'success_ratio': 1.0, 'success_rps': 59.9, 'p95_ms_successful': 31.0}, collector heap 35.5 MiB, container 75.9 MiB
[ 53.8s] docker compose stop tempo loki: stopped
[ 55.9s] {'t': 0.0, 'collector_heap_mib': 56.4, 'collector_rss_mib': 237.6, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 5.0}, 'alert': 'inactive'}
[ 68.0s] {'t': 12.1, 'collector_heap_mib': 45.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 9.0}, 'alert': 'pending'}
[ 80.0s] {'t': 24.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 12.0}, 'alert': 'pending'}
[ 92.5s] {'t': 36.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 78.1, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'}
[ 104.5s] {'t': 48.7, 'collector_heap_mib': 62.9, 'collector_rss_mib': 240.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'}
[ 116.9s] {'t': 60.8, 'collector_heap_mib': 51.2, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
[ 128.9s] {'t': 73.1, 'collector_heap_mib': 59.9, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.3, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'}
[ 141.5s] {'t': 85.2, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 88.7, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'firing'}
[ 153.5s] {'t': 97.7, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
[ 165.9s] {'t': 109.8, 'collector_heap_mib': 56.3, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
[ 178.0s] {'t': 122.1, 'collector_heap_mib': 67.5, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 70.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 31.0}, 'alert': 'firing'}
[ 190.0s] {'t': 134.2, 'collector_heap_mib': 76.1, 'collector_rss_mib': 256.9, 'collector_container_mib': 98.8, 'queue': {'otlp_http/loki': 78.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 34.0}, 'alert': 'firing'}
[ 202.6s] {'t': 146.2, 'collector_heap_mib': 86.9, 'collector_rss_mib': 267.1, 'collector_container_mib': 107.0, 'queue': {'otlp_http/loki': 85.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 38.0}, 'alert': 'firing'}
[ 214.0s] docker compose start tempo loki
[ 274.1s] exporter queues drained 60.1 s after start
[ 289.4s] ... TelemetryExportFailing: firing (0 s)
[ 294.4s] ... TelemetryExportFailing: firing (5 s)
[ 299.5s] ... TelemetryExportFailing: firing (10 s)
[ 304.5s] ... TelemetryExportFailing: firing (15 s)
[ 309.5s] ... TelemetryExportFailing: firing (20 s)
[ 314.5s] ... TelemetryExportFailing: firing (25 s)
[ 319.5s] ... TelemetryExportFailing: firing (30 s)
[ 324.5s] ... TelemetryExportFailing: firing (35 s)
[ 329.5s] ... TelemetryExportFailing: firing (40 s)
[ 334.5s] ... TelemetryExportFailing: firing (45 s)
[ 339.6s] ... TelemetryExportFailing: firing (50 s)
[ 344.6s] ... TelemetryExportFailing: firing (55 s)
[ 349.6s] ... TelemetryExportFailing: firing (60 s)
[ 354.6s] ... TelemetryExportFailing: firing (65 s)
[ 359.6s] ... TelemetryExportFailing: firing (70 s)
[ 364.6s] ... TelemetryExportFailing: firing (75 s)
[ 369.6s] ... TelemetryExportFailing: firing (80 s)
[ 374.7s] ... TelemetryExportFailing: firing (85 s)
[ 379.7s] API during outage: {'requests': 9555, 'codes': {'201': 9555}, 'success_ratio': 1.0, 'success_rps': 59.6, 'p95_ms_successful': 31.0}
[ 379.7s] probe traces: 13/13 reached Tempo (lost: sent at [] s)
[ 379.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 235}; collector heap 35.5 -> max 86.9 MiB (memory_limiter 200 MiB), container max 107.0 MiB
[ 379.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-telemetry-down.json
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
1 passed, 0 failed
+50
View File
@@ -0,0 +1,50 @@
{
"scenario": "worker-stop",
"date": "2026-11-01 20:09:59",
"firing_after_stop_s": {
"MessengerWorkerNoConsumers": 101.7,
"MessengerQueueLagGrowing": 131.8
},
"diagnosis": {
"queue_consumers": 0.0,
"broker_queue_depth": 6.0,
"jobs_backlog_queued": 3.0,
"oldest_job_age_s": 127.3,
"outbox_oldest_age_s": 0.0,
"published_ok_last_5m": 16.41239671427261,
"worker_metrics_last_seen_s": 131.0,
"last_worker_log": {
"event": "service.stopped",
"message": "stopped"
},
"upload_trace_span_names": [
"INSERT",
"NOTIFY",
"POST /api/v1/chats/{chat_id}/attachments",
"S3 PutObject",
"SELECT",
"UPDATE",
"create attachments",
"inspect image",
"publish attachments",
"redis GET",
"republish lost message"
],
"upload_trace_has_consumer_span": false
},
"ready_after_start_s": [
4.2,
4.2,
4.7
],
"resolved_after_start_s": {
"MessengerWorkerNoConsumers": 29.7,
"MessengerQueueLagGrowing": 29.7
},
"after_recovery": {
"jobs_backlog_queued": 0.0,
"oldest_job_age_s": 0.0,
"queue_consumers": 2.0
},
"time_to_ready_p95_5m_s": 282.0
}
+49
View File
@@ -0,0 +1,49 @@
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
# date: 2026-11-01 20:12:40, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/incidents.py worker-stop
# exit code: 0, 161.8 s
=== worker-stop
[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s
[ 1.2s] docker compose stop worker
[ 1.3s] uploaded 3 attachments: 202 queued (API accepts without worker)
[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
[ 6.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {}
[ 16.5s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {}
[ 21.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {}
[ 26.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {}
[ 31.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {}
[ 36.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {}
[ 41.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 46.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 51.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 56.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 61.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 66.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 71.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 76.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 81.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 86.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 91.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 96.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 101.8s] ALERT MessengerWorkerNoConsumers firing (101.7 s after stop)
[ 101.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 106.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 111.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 116.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 121.8s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 126.9s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 131.9s] ALERT MessengerQueueLagGrowing firing (131.8 s after stop)
[ 132.0s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 6.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 127.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 16.41239671427261, "worker_metrics_last_seen_s": 131.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET", "republish lost message"], "upload_trace_has_consumer_span": false}
[ 135.3s] docker compose start worker
[ 136.6s] all attachments ready 4.7 s after worker start; backlog now 3.0
[ 136.6s] ... MessengerWorkerNoConsumers: firing (0 s)
[ 141.6s] ... MessengerWorkerNoConsumers: firing (5 s)
[ 146.6s] ... MessengerWorkerNoConsumers: firing (10 s)
[ 151.7s] ... MessengerWorkerNoConsumers: firing (15 s)
[ 156.7s] ... MessengerWorkerNoConsumers: firing (20 s)
[ 161.7s] alerts resolved: {'MessengerWorkerNoConsumers': 29.7, 'MessengerQueueLagGrowing': 29.7}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
[ 161.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-worker-stop.json
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
1 passed, 0 failed
+8
View File
@@ -0,0 +1,8 @@
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
# date: 2026-11-01 20:38:38, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py concurrency
# exit code: 0, 15.8 s
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples)
1 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
# date: 2026-11-01 20:33:58, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
# exit code: 0, 157.0 s
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
staging source still present after the crash (worker died before cleanup); waiting 60 s window
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
4 passed, 0 failed
+13
View File
@@ -0,0 +1,13 @@
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
# date: 2026-11-01 20:38:23, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_janitor_replay.py
# exit code: 0, 187.0 s
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
waiting 86 s until the leftovers are older than grace (120 s)
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
ops replay -> job 09f5cf6c-bca3-4759-9930-8d18968ed1be ready
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 0, deleted 0
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
1 passed, 0 failed
+11
View File
@@ -0,0 +1,11 @@
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
# date: 2026-11-01 20:30:08, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py limits
# exit code: 0, 72.7 s
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
10 MiB + 8 bytes -> 413 before any job is created
imaging.py on 8000x5000 PNG: exit 0, peak RSS 226.9 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s
1 passed, 0 failed
+9
View File
@@ -0,0 +1,9 @@
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
# date: 2026-11-01 20:31:21, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py outbox-crash
# exit code: 0, 17.2 s
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
1 passed, 0 failed
+16
View File
@@ -0,0 +1,16 @@
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
# date: 2026-11-01 20:31:04, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py worker-down broker-down
# exit code: 0, 55.2 s
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
worker stopped
upload while worker is down -> 202 queued, attachment 0d7622f1-7c85-4c52-9dc0-2dad8f729c8d
message with attachment 201, content 409, other messages 201; job 6a45774d-0d88-46ec-87b9-70fb4c122bbd published, queue messages/unacked = (1, 0)
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
rabbitmq stopped
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 34eb3c30-1942-4777-bce2-1ea2a4eca4c9
rabbitmq started -> dispatcher published the pending row -> ready
2 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
# date: 2026-11-01 20:35:16, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq
# exit code: 0, 77.6 s
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
4 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
# date: 2026-11-01 20:28:56, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_jobs.py variants
# exit code: 0, 20.1 s
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
1 passed, 0 failed
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.031,
"concurrency": 4,
"successful_requests": 7178,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 239.02,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 47.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
# load: public load generator, 30 s, concurrency 4
# date: 2026-11-01 20:44:10, commit: 470d9d6 + uncommitted changes
# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
# exit code: 0, 30.3 s
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.031,
"concurrency": 4,
"successful_requests": 7178,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 239.02,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 47.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+96
View File
@@ -0,0 +1,96 @@
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
# date: 2026-11-01 00:52:47, commit: unknown
# command: python student-tests/test_attachments.py
# exit code: 0, 13.5 s
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
exactly 10485760 bytes (10 MiB) -> 202, then ready
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
PASS: the 10 MiB limit is exact
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
PASS: oversized streamed body is cut off, the API keeps working
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f30c028f
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
PASS: pixel limit is checked before decompression
PNG cut in half (55908 bytes, image/png) -> 415
PNG without IEND (111805 bytes, image/png) -> 415
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
PNG with garbage after IEND (111824 bytes, image/png) -> 415
PNG signature only (8 bytes, image/png) -> 415
PNG with less pixel data than the header says (70 bytes, image/png) -> 415
JPEG cut in half (746 bytes, image/jpeg) -> 415
JPEG header only (20 bytes, image/jpeg) -> 415
random bytes (1600 bytes, image/png) -> 415
plain text (28 bytes, image/jpeg) -> 415
empty body -> 400
PASS: damaged and foreign files are rejected with 415, never 500
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
empty PLTE (79 bytes, valid CRCs) -> 415
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
palette image without PLTE (67 bytes, valid CRCs) -> 415
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
PLTE after IDAT (85 bytes, valid CRCs) -> 415
two PLTE chunks (103 bytes, valid CRCs) -> 415
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
tRNS before PLTE (98 bytes, valid CRCs) -> 415
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
tRNS in an image with alpha channel (84 bytes, valid CRCs) -> 415
IEND with data (70 bytes, valid CRCs) -> 415
unknown critical chunk (81 bytes, valid CRCs) -> 415
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
gAMA after IDAT (85 bytes, valid CRCs) -> 415
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
private chunk after IDAT (181 bytes, valid CRCs) -> 415
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
palette depth 2 with 4 entries -> 201, downloaded byte for byte
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
PNG bytes sent as image/jpeg -> 415
JPEG bytes sent as image/png -> 415
GIF bytes sent as image/png -> 415
PNG sent as image/gif -> 415
PNG sent as text/html -> 415
PNG sent as application/octet-stream -> 415
PNG sent as multipart/form-data -> 415
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
PASS: the type is taken from the content; a declared type that does not match is refused
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
no filename -> 400
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
PASS: filename never becomes a path; dangerous names are refused
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
upload into someone else's / unknown chat -> 404, without token -> 401
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
PASS: only chat members can see an attachment
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
attachment of another chat (author is a member of both) -> 404
own + foreign attachment -> 404
unknown attachment_id -> 404
outsider referencing an attachment of the chat -> 404
duplicate ids -> 400
11 ids -> 400
bad UUID -> 400
empty text without attachments -> 400
whitespace text without attachments -> 400
PASS: only attachments of the same chat can be linked; rejected requests change nothing
same key + same attachments -> same 201; other attachments -> 409; missing field == []
PASS: Idempotency-Key covers attachment_ids
sent 1.1 MB of a declared 5 MB body and closed the connection
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
PASS: an interrupted upload is dropped
12 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# check: offline contract check (make check)
# date: 2026-11-01 00:22:29, commit: unknown
# command: python scripts/check_contract.py
# exit code: 0, 0.1 s
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+28
View File
@@ -0,0 +1,28 @@
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
# date: 2026-11-01 00:22:31, commit: unknown
# command: python student-tests/check_configs.py
# exit code: 0, 0.7 s
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool check rules (recording + alerts): exit 0
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml
SUCCESS: 6 rules found
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml
SUCCESS: 10 rules found
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
SUCCESS
--- loki -verify-config (infra/loki/loki.yaml): exit 0
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
--- Grafana: not reachable at http://127.0.0.1:3000, skipped (local binaries mode)
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid
+36
View File
@@ -0,0 +1,36 @@
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
# date: 2026-11-01 00:23:49, commit: unknown
# command: python student-tests/check_dashboard.py
# exit code: 0, 0.5 s
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
empty now: CPU контейнеров / лимит
empty now: Collector: потери и отказы
empty now: SDK приложений: экспорт и потери
empty now: alerts.yml: MessengerDbPoolSaturated
empty now: alerts.yml: MessengerHighErrorRate
empty now: alerts.yml: MessengerQueueLagGrowing
empty now: alerts.yml: MessengerWorkerNoConsumers
empty now: alerts.yml: TelemetryCollectorDown
empty now: alerts.yml: TelemetryExportFailing
empty now: Алерты Prometheus (pending/firing)
empty now: Память: RSS процесса и контейнер
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s)
data: http_server_request_duration_seconds_count{job="messenger-api"} 39 series <- HTTP requests/duration (3)
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
data: messenger_queue_depth 2 series <- queue depth (4)
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
data: db_client_connection_count 4 series <- DB pool used/idle (4)
data: db_client_connection_pending_requests 2 series <- DB pool waiting (4)
data: messenger_worker_busy 1 series <- worker busy (4)
data: messenger_worker_capacity 1 series <- worker capacity (4)
data: process_cpu_time_seconds_total 12 series <- process CPU (4)
data: process_memory_usage_bytes 6 series <- process RAM (4)
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
+22
View File
@@ -0,0 +1,22 @@
# idempotency: Idempotency-Key, parallel same key via replicas
# date: 2026-11-01 00:52:50, commit: unknown
# command: python student-tests/test_idempotency.py
# exit code: 0, 2.2 s
first -> 201; same JSON written differently -> 201; plain retry -> 201
PASS: a retry with the same key returns the original 201 and message, no duplicate
20 parallel POSTs with one key: statuses {201: 20}, replicas {'a1a1a1a1a1a1': 10, 'a2a2a2a2a2a2': 10}, different bodies in answers: 1, messages in history: 1
PASS: parallel requests with one key through different replicas create exactly one message
20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B']
PASS: the first body wins, the same body gets 201, the other body gets 409
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
key used by a 400 and by a 404 first, then valid request -> 201
PASS: rejected requests (400, 404) do not store a result for the key
same key and body by an outsider -> 404; by alice after logout -> 401
PASS: a retry checks the session and chat access again, the saved answer is not leaked
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
PASS: an unconfirmed POST can be safely retried with the same key
8 passed, 0 failed
@@ -0,0 +1,39 @@
{
"scenario": "collector-down",
"date": "2026-11-01 00:39:14",
"outage_s": 90,
"baseline": {
"requests": 1002,
"codes": {
"201": 1002
},
"success_ratio": 1.0,
"success_rps": 66.8,
"p95_ms_successful": 15.2
},
"during": {
"requests": 6263,
"codes": {
"201": 6263
},
"success_ratio": 1.0,
"success_rps": 67.4,
"p95_ms_successful": 14.3
},
"api_mib_before": 280.5,
"api_mib_max": 292.5,
"sdk_lost_spans_api": {
"processor_queue_full": 0,
"exporter_failed": 24576
},
"sdk_exported_ok_after_restart": 10524,
"after": {
"requests": 1703,
"codes": {
"201": 1703
},
"success_ratio": 1.0,
"success_rps": 68.1,
"p95_ms_successful": 13.8
}
}
@@ -0,0 +1,22 @@
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
# date: 2026-11-01 00:41:37, commit: unknown
# command: python student-tests/incidents.py collector-down
# exit code: 0, 143.3 s
=== collector-down
[ 20.3s] baseline: {'requests': 1002, 'codes': {'201': 1002}, 'success_ratio': 1.0, 'success_rps': 66.8, 'p95_ms_successful': 15.2}, api containers 280.5 MiB
[ 20.6s] docker compose stop otel-collector
[ 20.8s] {'t': 0.3, 'api_mib': 283.0}
[ 31.0s] {'t': 10.5, 'api_mib': 283.2}
[ 41.3s] {'t': 20.8, 'api_mib': 288.6}
[ 51.5s] {'t': 31.0, 'api_mib': 289.0}
[ 61.7s] {'t': 41.2, 'api_mib': 289.7}
[ 71.9s] {'t': 51.4, 'api_mib': 292.5}
[ 82.1s] {'t': 61.6, 'api_mib': 292.4}
[ 92.4s] {'t': 71.9, 'api_mib': 292.3}
[ 102.6s] {'t': 82.1, 'api_mib': 291.2}
[ 113.2s] docker compose start otel-collector
[ 143.2s] API during collector outage: {'requests': 6263, 'codes': {'201': 6263}, 'success_ratio': 1.0, 'success_rps': 67.4, 'p95_ms_successful': 14.3}; api memory 280.5 -> max 292.5 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 24576}; exported OK in the last minute after restart: 10524
[ 143.2s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-collector-down.json
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
1 passed, 0 failed
@@ -0,0 +1,82 @@
{
"scenario": "db-slow",
"date": "2026-11-01 00:42:49",
"lock_seconds": 210,
"baseline": {
"requests": 2648,
"codes": {
"201": 2648
},
"success_ratio": 1.0,
"success_rps": 132.4,
"p95_ms_successful": 18.5,
"server_p95_s": 0.0214
},
"firing_after_lock_s": 120.1,
"diagnosis": {
"http_5xx_ratio_1m": 1.0,
"http_p95_messages_s": 7.375,
"pool_used": 8.0,
"pool_wait_p95_s": 0.001,
"status_codes_1m": {
"201": 0.0,
"200": 0.0,
"404": 0.0,
"400": 0.0,
"409": 0.0,
"503": 1.6
},
"reads_during_lock_ms": {
"GET /users/me + GET /chats, avg per request": 4.7,
"max": 5.4
},
"api_log_db_unavailable_by_error_type": {
"QueryCanceled": 184
},
"error_trace": {
"trace_id": "1c7f8dd06427574a5f155558aa4cd975",
"server_ms": 5006.7,
"slowest_db_span": "INSERT",
"slowest_db_ms": 5000.6,
"db_span_status": "ERROR",
"db_share": 0.999,
"tree": [
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503",
" - [api] redis GET (CLIENT, 0.2 ms)",
" - [api] SELECT (CLIENT, 0.4 ms)",
" - [api] INSERT (CLIENT, 5000.6 ms) ERROR"
]
},
"handler_cpu_contrast": {
"trace_id": "a6da2e3fe7b0c229773f36fa507293ca",
"server_ms": 34.0,
"argon2id_ms": 29.0,
"db_ms_total": 0.6,
"tree": [
"- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201",
" - [api] argon2id hash (INTERNAL, 29.0 ms)",
" - [api] INSERT (CLIENT, 0.6 ms)"
]
}
},
"during_lock": {
"requests": 326,
"codes": {
"503": 320,
"201": 6
},
"success_ratio": 0.0184,
"success_rps": 0.0,
"p95_ms_successful": 2718.4
},
"resolved_after_unlock_s": 30.0,
"after": {
"requests": 3298,
"codes": {
"201": 3298
},
"success_ratio": 1.0,
"success_rps": 131.9,
"p95_ms_successful": 21.0
}
}
+56
View File
@@ -0,0 +1,56 @@
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
# date: 2026-11-01 00:47:15, commit: unknown
# command: python student-tests/incidents.py db-slow
# exit code: 0, 265.4 s
=== db-slow
[ 0.1s] load: 8 threads POST /messages
[ 25.1s] baseline: {'requests': 2648, 'codes': {'201': 2648}, 'success_ratio': 1.0, 'success_rps': 132.4, 'p95_ms_successful': 18.5, 'server_p95_s': 0.0214}
[ 25.1s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
[ 25.1s] 5xx ratio=0.0 p95=0.021 s pool used=0.0 waiting=0.0
[ 30.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
[ 35.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
[ 40.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
[ 45.1s] 5xx ratio=0.005 p95=0.022 s pool used=8.0 waiting=0.0
[ 50.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0
[ 55.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0
[ 60.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0
[ 65.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0
[ 70.1s] 5xx ratio=0.01 p95=5.688 s pool used=8.0 waiting=0.0
[ 75.2s] 5xx ratio=0.069 p95=5.688 s pool used=8.0 waiting=0.0
[ 80.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0
[ 85.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0
[ 90.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 95.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 100.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 105.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 110.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 115.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 120.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 125.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 130.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 135.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 140.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
[ 145.2s] ALERT MessengerHighErrorRate firing (120.1 s after lock)
[ 151.5s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 4.7, "max": 5.4}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}}
[ 151.5s] error trace (DB): {"trace_id": "1c7f8dd06427574a5f155558aa4cd975", "server_ms": 5006.7, "slowest_db_span": "INSERT", "slowest_db_ms": 5000.6, "db_span_status": "ERROR", "db_share": 0.999}
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503
- [api] redis GET (CLIENT, 0.2 ms)
- [api] SELECT (CLIENT, 0.4 ms)
- [api] INSERT (CLIENT, 5000.6 ms) ERROR
[ 151.5s] contrast, slow handler (CPU): {"trace_id": "a6da2e3fe7b0c229773f36fa507293ca", "server_ms": 34.0, "argon2id_ms": 29.0, "db_ms_total": 0.6}
- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201
- [api] argon2id hash (INTERNAL, 29.0 ms)
- [api] INSERT (CLIENT, 0.6 ms)
[ 235.4s] lock released (psql exit 0), 210.3 s after LOCK
[ 235.4s] ... MessengerHighErrorRate: firing (0 s)
[ 240.4s] ... MessengerHighErrorRate: firing (5 s)
[ 245.4s] ... MessengerHighErrorRate: firing (10 s)
[ 250.4s] ... MessengerHighErrorRate: firing (15 s)
[ 255.4s] ... MessengerHighErrorRate: firing (20 s)
[ 260.4s] ... MessengerHighErrorRate: firing (25 s)
[ 265.4s] ALERT MessengerHighErrorRate resolved 30.0 s after unlock
[ 265.4s] load: during lock {'requests': 326, 'codes': {'503': 320, '201': 6}, 'success_ratio': 0.0184, 'success_rps': 0.0, 'p95_ms_successful': 2718.4}; after {'requests': 3298, 'codes': {'201': 3298}, 'success_ratio': 1.0, 'success_rps': 131.9, 'p95_ms_successful': 21.0}
[ 265.4s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-db-slow.json
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
1 passed, 0 failed
@@ -0,0 +1,307 @@
{
"scenario": "telemetry-down",
"date": "2026-11-01 00:32:59",
"stopped": [
"tempo",
"loki"
],
"outage_s": 150,
"baseline": {
"requests": 955,
"codes": {
"201": 955
},
"success_ratio": 1.0,
"success_rps": 63.7,
"p95_ms_successful": 17.9
},
"collector_heap_before_mib": 44.1,
"collector_container_before_mib": 224.4,
"during": {
"requests": 11500,
"codes": {
"201": 11500
},
"success_ratio": 1.0,
"success_rps": 68.3,
"p95_ms_successful": 13.2
},
"samples": [
{
"t": 0.0,
"collector_heap_mib": 39.8,
"collector_rss_mib": 228.5,
"collector_container_mib": 228.6,
"queue": {
"otlp_http/loki": 0.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 6.0
},
"alert": "inactive"
},
{
"t": 10.1,
"collector_heap_mib": 52.2,
"collector_rss_mib": 228.6,
"collector_container_mib": 228.6,
"queue": {
"otlp_http/loki": 5.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 10.0
},
"alert": "inactive"
},
{
"t": 20.3,
"collector_heap_mib": 52.2,
"collector_rss_mib": 228.6,
"collector_container_mib": 228.7,
"queue": {
"otlp_http/loki": 5.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 10.0
},
"alert": "inactive"
},
{
"t": 30.4,
"collector_heap_mib": 63.3,
"collector_rss_mib": 230.2,
"collector_container_mib": 233.6,
"queue": {
"otlp_http/loki": 12.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 14.0
},
"alert": "inactive"
},
{
"t": 40.6,
"collector_heap_mib": 48.5,
"collector_rss_mib": 234.5,
"collector_container_mib": 234.5,
"queue": {
"otlp_http/loki": 20.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 18.0
},
"alert": "pending"
},
{
"t": 50.7,
"collector_heap_mib": 48.5,
"collector_rss_mib": 234.5,
"collector_container_mib": 234.6,
"queue": {
"otlp_http/loki": 20.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 18.0
},
"alert": "pending"
},
{
"t": 60.8,
"collector_heap_mib": 59.5,
"collector_rss_mib": 234.6,
"collector_container_mib": 234.9,
"queue": {
"otlp_http/loki": 27.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 22.0
},
"alert": "pending"
},
{
"t": 71.0,
"collector_heap_mib": 70.7,
"collector_rss_mib": 239.4,
"collector_container_mib": 242.2,
"queue": {
"otlp_http/loki": 35.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 26.0
},
"alert": "pending"
},
{
"t": 81.2,
"collector_heap_mib": 70.7,
"collector_rss_mib": 239.4,
"collector_container_mib": 247.4,
"queue": {
"otlp_http/loki": 35.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 26.0
},
"alert": "pending"
},
{
"t": 91.3,
"collector_heap_mib": 58.1,
"collector_rss_mib": 247.4,
"collector_container_mib": 247.4,
"queue": {
"otlp_http/loki": 42.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 30.0
},
"alert": "pending"
},
{
"t": 101.4,
"collector_heap_mib": 69.3,
"collector_rss_mib": 247.5,
"collector_container_mib": 247.5,
"queue": {
"otlp_http/loki": 50.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 32.0
},
"alert": "firing"
},
{
"t": 111.6,
"collector_heap_mib": 69.3,
"collector_rss_mib": 247.5,
"collector_container_mib": 249.7,
"queue": {
"otlp_http/loki": 50.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 32.0
},
"alert": "firing"
},
{
"t": 121.8,
"collector_heap_mib": 80.2,
"collector_rss_mib": 249.9,
"collector_container_mib": 258.8,
"queue": {
"otlp_http/loki": 57.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 36.0
},
"alert": "firing"
},
{
"t": 131.9,
"collector_heap_mib": 92.7,
"collector_rss_mib": 262.3,
"collector_container_mib": 265.4,
"queue": {
"otlp_http/loki": 63.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 40.0
},
"alert": "firing"
},
{
"t": 142.1,
"collector_heap_mib": 92.7,
"collector_rss_mib": 262.3,
"collector_container_mib": 273.3,
"queue": {
"otlp_http/loki": 63.0,
"otlp_http/prometheus": 0.0,
"otlp_grpc/tempo": 40.0
},
"alert": "firing"
}
],
"collector_heap_max_mib": 92.7,
"collector_rss_max_mib": 262.3,
"collector_container_max_mib": 273.3,
"TelemetryExportFailing_firing_at_s": 101.4,
"queue_drained_after_start_s": 80.0,
"probe_traces": [
{
"sent_at_s": 0.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 10.1,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 20.3,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 30.4,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 40.6,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 50.7,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 60.8,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 71.0,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 81.2,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 91.3,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 101.4,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 111.6,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 121.7,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 131.9,
"http": 201,
"trace_in_tempo": true
},
{
"sent_at_s": 142.1,
"http": 201,
"trace_in_tempo": false
}
],
"lost_in_collector": {
"spans": 4096,
"log_records": 269
},
"resolved_after_start_s": 155.4,
"after": {
"requests": 10186,
"codes": {
"201": 10186
},
"success_ratio": 1.0,
"success_rps": 67.7,
"p95_ms_successful": 14.2
}
}
@@ -0,0 +1,41 @@
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
# date: 2026-11-01 00:39:14, commit: unknown
# command: python student-tests/incidents.py telemetry-down
# exit code: 0, 374.8 s
=== telemetry-down
[ 20.2s] baseline: {'requests': 955, 'codes': {'201': 955}, 'success_ratio': 1.0, 'success_rps': 63.7, 'p95_ms_successful': 17.9}, collector heap 44.1 MiB, container 224.4 MiB
[ 50.9s] docker compose stop tempo loki: stopped
[ 51.1s] {'t': 0.0, 'collector_heap_mib': 39.8, 'collector_rss_mib': 228.5, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 0.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 6.0}, 'alert': 'inactive'}
[ 61.2s] {'t': 10.1, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'}
[ 71.3s] {'t': 20.3, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.7, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'}
[ 81.5s] {'t': 30.4, 'collector_heap_mib': 63.3, 'collector_rss_mib': 230.2, 'collector_container_mib': 233.6, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 14.0}, 'alert': 'inactive'}
[ 91.6s] {'t': 40.6, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.5, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'}
[ 101.8s] {'t': 50.7, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.6, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'}
[ 112.0s] {'t': 60.8, 'collector_heap_mib': 59.5, 'collector_rss_mib': 234.6, 'collector_container_mib': 234.9, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'pending'}
[ 122.1s] {'t': 71.0, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 242.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'}
[ 132.2s] {'t': 81.2, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'}
[ 142.4s] {'t': 91.3, 'collector_heap_mib': 58.1, 'collector_rss_mib': 247.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'pending'}
[ 152.5s] {'t': 101.4, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 247.5, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'}
[ 162.7s] {'t': 111.6, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 249.7, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'}
[ 172.9s] {'t': 121.8, 'collector_heap_mib': 80.2, 'collector_rss_mib': 249.9, 'collector_container_mib': 258.8, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 36.0}, 'alert': 'firing'}
[ 183.0s] {'t': 131.9, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 265.4, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'}
[ 193.2s] {'t': 142.1, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 273.3, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'}
[ 219.3s] docker compose start tempo loki
[ 299.3s] exporter queues drained 80.0 s after start
[ 324.7s] ... TelemetryExportFailing: firing (0 s)
[ 329.7s] ... TelemetryExportFailing: firing (5 s)
[ 334.7s] ... TelemetryExportFailing: firing (10 s)
[ 339.7s] ... TelemetryExportFailing: firing (15 s)
[ 344.7s] ... TelemetryExportFailing: firing (20 s)
[ 349.7s] ... TelemetryExportFailing: firing (25 s)
[ 354.7s] ... TelemetryExportFailing: firing (30 s)
[ 359.7s] ... TelemetryExportFailing: firing (35 s)
[ 364.7s] ... TelemetryExportFailing: firing (40 s)
[ 369.7s] ... TelemetryExportFailing: firing (45 s)
[ 374.7s] API during outage: {'requests': 11500, 'codes': {'201': 11500}, 'success_ratio': 1.0, 'success_rps': 68.3, 'p95_ms_successful': 13.2}
[ 374.7s] probe traces: 14/15 reached Tempo (lost: sent at [142.1] s)
[ 374.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 269}; collector heap 44.1 -> max 92.7 MiB (memory_limiter 200 MiB), container max 273.3 MiB
[ 374.7s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-telemetry-down.json
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
1 passed, 0 failed
@@ -0,0 +1,49 @@
{
"scenario": "worker-stop",
"date": "2026-11-01 00:26:35",
"firing_after_stop_s": {
"MessengerWorkerNoConsumers": 66.4,
"MessengerQueueLagGrowing": 111.4
},
"diagnosis": {
"queue_consumers": 0.0,
"broker_queue_depth": 3.0,
"jobs_backlog_queued": 3.0,
"oldest_job_age_s": 97.3,
"outbox_oldest_age_s": 0.0,
"published_ok_last_5m": 3.102218085931441,
"worker_metrics_last_seen_s": 110.0,
"last_worker_log": {
"event": "service.stopped",
"message": "stopped"
},
"upload_trace_span_names": [
"INSERT",
"NOTIFY",
"POST /api/v1/chats/{chat_id}/attachments",
"S3 PutObject",
"SELECT",
"UPDATE",
"create attachments",
"inspect image",
"publish attachments",
"redis GET"
],
"upload_trace_has_consumer_span": false
},
"ready_after_start_s": [
1.5,
1.6,
2.0
],
"resolved_after_start_s": {
"MessengerWorkerNoConsumers": 17.0,
"MessengerQueueLagGrowing": 17.0
},
"after_recovery": {
"jobs_backlog_queued": 0.0,
"oldest_job_age_s": 0.0,
"queue_consumers": 2.0
},
"time_to_ready_p95_5m_s": 117.0
}
@@ -0,0 +1,43 @@
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
# date: 2026-11-01 00:28:44, commit: unknown
# command: python student-tests/incidents.py worker-stop
# exit code: 0, 128.6 s
=== worker-stop
[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s
[ 1.3s] docker compose stop worker
[ 1.4s] uploaded 3 attachments: 202 queued (API accepts without worker)
[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
[ 6.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 16.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 21.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 26.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 31.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 36.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 41.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 46.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
[ 51.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 56.4s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 61.5s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
[ 66.5s] ALERT MessengerWorkerNoConsumers firing (66.4 s after stop)
[ 66.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 71.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 76.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 81.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 86.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 91.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 96.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 101.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 106.5s] consumers=0.0 backlog queued=3.0 oldest age=97.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
[ 111.5s] ALERT MessengerQueueLagGrowing firing (111.4 s after stop)
[ 111.5s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 97.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 3.102218085931441, "worker_metrics_last_seen_s": 110.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false}
[ 111.6s] docker compose start worker
[ 113.5s] all attachments ready 2.0 s after worker start; backlog now 3.0
[ 113.5s] ... MessengerWorkerNoConsumers: firing (0 s)
[ 118.5s] ... MessengerWorkerNoConsumers: firing (5 s)
[ 123.5s] ... MessengerWorkerNoConsumers: firing (10 s)
[ 128.5s] alerts resolved: {'MessengerWorkerNoConsumers': 17.0, 'MessengerQueueLagGrowing': 17.0}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
[ 128.5s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-worker-stop.json
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
1 passed, 0 failed
@@ -0,0 +1,8 @@
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
# date: 2026-11-01 00:52:34, commit: unknown
# command: python student-tests/test_jobs.py concurrency
# exit code: 0, 7.1 s
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 4.1 s after worker start, peak processing 2 of 8 (22 samples)
1 passed, 0 failed
+9
View File
@@ -0,0 +1,9 @@
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
# date: 2026-11-01 00:51:38, commit: unknown
# command: python student-tests/test_jobs.py outbox-crash
# exit code: 0, 7.4 s
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 18ab06f7-657b-4508-9ba5-844bd2c7ba2e ready, attempt 1, no manual action
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
1 passed, 0 failed
+16
View File
@@ -0,0 +1,16 @@
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
# date: 2026-11-01 00:48:13, commit: unknown
# command: python student-tests/test_jobs.py worker-down broker-down
# exit code: 0, 44.2 s
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
worker stopped
upload while worker is down -> 202 queued, attachment 748b0b56-e355-48a0-9ac6-4b731a3432a9
message with attachment 201, content 409, other messages 201; job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 published, queue messages/unacked = (1, 0)
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
worker started -> same job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 ready, thumbnail downloadable; staging object deleted
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
rabbitmq stopped
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 5784e1e7-7173-4a27-a3ad-72216f4a43a9
rabbitmq started -> dispatcher published the pending row -> ready
2 passed, 0 failed
+15
View File
@@ -0,0 +1,15 @@
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
# date: 2026-11-01 00:52:27, commit: unknown
# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq
# exit code: 0, 49.1 s
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
2 x storage_unavailable -> retries after 1.40 s and 3.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
ops replay -> same job_id fce6904a-fe4b-429c-821a-e3b55a43b0ef attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
4 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
# date: 2026-11-01 00:47:29, commit: unknown
# command: python student-tests/test_jobs.py variants
# exit code: 0, 14.3 s
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
1 passed, 0 failed
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.083,
"concurrency": 4,
"successful_requests": 8762,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 291.265,
"successful_latency_ms": {
"p50": 11.557,
"p95": 26.659,
"p99": 56.425
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
# load: public load generator, 30 s, concurrency 4
# date: 2026-11-01 00:42:08, commit: unknown
# command: python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
# exit code: 0, 30.4 s
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.083,
"concurrency": 4,
"successful_requests": 8762,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 291.265,
"successful_latency_ms": {
"p50": 11.557,
"p95": 26.659,
"p99": 56.425
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+10
View File
@@ -0,0 +1,10 @@
# race: lab2 races via the balancer
# date: 2026-11-01 00:52:53, commit: unknown
# command: python student-tests/test_race.py
# exit code: 0, 1.5 s
sent 100 messages from 10 clients in 0.34 s; read 100 in 4 pages, unique ids: 100
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
20 parallel registrations of one username: 201 x1, 409 x19
PASS: concurrent registration created exactly one user
2 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment
# date: 2026-11-01 00:22:30, commit: unknown
# command: python tests/smoke.py --base-url http://127.0.0.1:8080
# exit code: 0, 1.7 s
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+26
View File
@@ -0,0 +1,26 @@
# student-tests: lab1/lab2 API tests via the balancer
# date: 2026-11-01 00:52:52, commit: unknown
# command: python student-tests/test_api.py
# exit code: 0, 2.0 s
PASS: duplicate username returns 409
PASS: invalid JSON returns 400
PASS: wrong and extra fields return 400
PASS: foreign chat returns 404
PASS: empty and too long messages return 400, 4000 chars is ok
PASS: pagination works across multiple pages, foreign cursor returns 400
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
PASS: chat list pagination
PASS: unicode text is stored as is
PASS: strings with U+0000 are stored and returned as is
PASS: user response has no password or hash
PASS: password length 12..128, blank password is rejected
PASS: Basic login returns opaque Bearer token with expiry
PASS: wrong password and unknown user both return the same 401
PASS: malformed Basic header returns 401
PASS: password with colon and unicode works
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
PASS: X-User-Id does not change the current user
PASS: 401 comes before 400 and 404
PASS: logout revokes only the current session
20 passed, 0 failed
+48
View File
@@ -0,0 +1,48 @@
# Сводка проверок (нативная песочница, без Docker)
Стенд теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, dispatcher, worker, janitor (Python 3.11, requirements.txt), otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из infra/ (адреса -> 127.0.0.1). Команды `docker compose` тестов выполнялись через подмену на управление процессами. Машина: Linux x86_64, 2 vCPU, 8 GiB RAM.
| Шаг | Что проверяет | Результат | Время, с | Файл |
| --- | --- | --- | --- | --- |
| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt |
| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 2 | evidence/native-sandbox/smoke.txt |
| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 1 | evidence/native-sandbox/configs.txt |
| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 77 | evidence/native-sandbox/telemetry.txt |
| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 0 | evidence/native-sandbox/dashboard.txt |
| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 52 | evidence/native-sandbox/telemetry-history.txt |
| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 129 | evidence/native-sandbox/incident-worker.txt |
| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 265 | evidence/native-sandbox/incident-db.txt |
| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 375 | evidence/native-sandbox/incident-telemetry.txt |
| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 143 | evidence/native-sandbox/incident-collector.txt |
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 14 | evidence/native-sandbox/jobs-variants.txt |
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 44 | evidence/native-sandbox/jobs-recovery.txt |
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 7 | evidence/native-sandbox/jobs-outbox.txt |
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt |
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt |
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 14 | evidence/native-sandbox/attachments.txt |
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/native-sandbox/idempotency.txt |
| student-tests | lab1/lab2 API tests via the balancer | PASS | 2 | evidence/native-sandbox/student-tests.txt |
| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt |
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/native-sandbox/load.txt |
# Замеры
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
| --- | --- | --- | --- | --- |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) |
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) |
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) |
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0.
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок.
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
| --- | --- | --- | --- |
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с |
| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) |
Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной).
@@ -0,0 +1,9 @@
# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)
# date: 2026-11-01 00:26:35, commit: unknown
# command: python student-tests/test_telemetry.py history
# exit code: 0, 52.2 s
before restart: trace 52668075eebc7fd4938de9690ba2aad1 found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 719
after restart of prometheus, loki, tempo: trace 52668075eebc7fd4938de9690ba2aad1 (4 spans), 1 log record(s), metric at t-30s = 719 (same value)
PASS: persistent volumes keep traces, logs and metrics across restarts of the stores
1 passed, 0 failed
+145
View File
@@ -0,0 +1,145 @@
# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets
# date: 2026-11-01 00:23:48, commit: unknown
# command: python student-tests/test_telemetry.py request upload-chain retry-chain replay-link
# exit code: 0, 77.1 s
POST message: 201, X-Request-Id=4d281ed1-39de-484a-add2-8480ef33ab9c, trace_id=97ebaba763fec926fa1e1ed3fc0fae7a (client traceparent)
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5.8 ms) http.response.status_code=201
- [api] redis GET (CLIENT, 0.3 ms)
- [api] SELECT (CLIENT, 0.5 ms)
- [api] INSERT (CLIENT, 0.8 ms)
Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (5.9 ms)' event=http.request severity=INFO service=messenger-api trace_id=97ebaba763fec926fa1e1ed3fc0fae7a span_id=82ab66080cb50d2a request_id=4d281ed1-39de-484a-add2-8480ef33ab9c
PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 64.6 ms) http.response.status_code=202
- [api] redis GET (CLIENT, 0.4 ms)
- [api] SELECT (CLIENT, 0.5 ms)
- [api] inspect image (INTERNAL, 45.7 ms)
- [api] INSERT (CLIENT, 1.1 ms)
- [api] S3 PutObject (CLIENT, 5.1 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
- [api] UPDATE (CLIENT, 0.9 ms)
- [api] create attachments (PRODUCER, 3.5 ms) messenger.job.attempt=1
- [api] INSERT (CLIENT, 0.9 ms)
- [api] INSERT (CLIENT, 0.9 ms)
- [api] INSERT (CLIENT, 0.8 ms)
- [api] NOTIFY (CLIENT, 0.3 ms)
- [dispatcher] publish attachments (PRODUCER, 4.4 ms) messenger.job.attempt=1
- [dispatcher] UPDATE (CLIENT, 0.9 ms)
- [worker] process attachments.process.v1 (CONSUMER, 280.0 ms) messenger.job.attempt=1 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 1.4 ms)
- [worker] UPDATE (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 1.4 ms)
- [worker] INSERT (CLIENT, 0.7 ms)
- [worker] S3 GetObject (CLIENT, 2.9 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
- [worker] verify source sha256 (INTERNAL, 0.1 ms)
- [worker] imaging (INTERNAL, 238.2 ms)
- [worker] S3 PutObject (CLIENT, 4.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/original
- [worker] S3 PutObject (CLIENT, 3.4 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/crop.jpg
- [worker] commit result (INTERNAL, 7.5 ms)
- [worker] SELECT (CLIENT, 0.7 ms)
- [worker] INSERT (CLIENT, 1.0 ms)
- [worker] INSERT (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.7 ms)
- [worker] UPDATE (CLIENT, 1.0 ms)
- [worker] UPDATE (CLIENT, 0.7 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] cleanup staging (INTERNAL, 4.1 ms)
- [worker] S3 DeleteObject (CLIENT, 2.4 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
- [worker] UPDATE (CLIENT, 1.4 ms)
worker logs with trace_id 0eecc096d0191a19e41ad1547110183c: ['job.claimed', 'job.ready']
PASS: one trace 0eecc096d0191a19e41ad1547110183c: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker)
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 17.4 ms) http.response.status_code=202
- [api] redis GET (CLIENT, 0.3 ms)
- [api] SELECT (CLIENT, 0.5 ms)
- [api] inspect image (INTERNAL, 0.3 ms)
- [api] INSERT (CLIENT, 0.7 ms)
- [api] S3 PutObject (CLIENT, 5.0 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
- [api] UPDATE (CLIENT, 0.6 ms)
- [api] create attachments (PRODUCER, 2.9 ms) messenger.job.attempt=1
- [api] INSERT (CLIENT, 0.6 ms)
- [api] INSERT (CLIENT, 0.5 ms)
- [api] INSERT (CLIENT, 0.5 ms)
- [api] NOTIFY (CLIENT, 0.2 ms)
- [dispatcher] publish attachments (PRODUCER, 3.2 ms) messenger.job.attempt=1
- [dispatcher] UPDATE (CLIENT, 0.5 ms)
- [worker] process attachments.process.v1 (CONSUMER, 255.7 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable
- [worker] SELECT (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] S3 GetObject (CLIENT, 3.4 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 238.9 ms)
- [worker] SELECT (CLIENT, 0.6 ms)
- [worker] UPDATE (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] create attachments (PRODUCER, 1.5 ms) messenger.job.attempt=2
- [worker] INSERT (CLIENT, 0.8 ms)
- [worker] NOTIFY (CLIENT, 0.2 ms)
- [dispatcher] publish attachments (PRODUCER, 3.3 ms) messenger.job.attempt=2
- [dispatcher] UPDATE (CLIENT, 0.6 ms)
- [worker] process attachments.process.v1 (CONSUMER, 284.4 ms) messenger.job.attempt=2 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 0.7 ms)
- [worker] UPDATE (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 0.7 ms)
- [worker] INSERT (CLIENT, 0.6 ms)
- [worker] S3 GetObject (CLIENT, 3.5 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
- [worker] verify source sha256 (INTERNAL, 0.1 ms)
- [worker] imaging (INTERNAL, 250.7 ms)
- [worker] S3 PutObject (CLIENT, 4.7 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/original
- [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 3.0 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/crop.jpg
- [worker] commit result (INTERNAL, 6.6 ms)
- [worker] SELECT (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.7 ms)
- [worker] INSERT (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 0.8 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.5 ms)
- [worker] cleanup staging (INTERNAL, 4.6 ms)
- [worker] S3 DeleteObject (CLIENT, 2.7 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
- [worker] UPDATE (CLIENT, 1.3 ms)
- [worker] INSERT (CLIENT, 0.5 ms)
worker log: 'job 9a82f341-ec57-49e5-aa86-6263681529c1 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable
messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1
PASS: retry in one trace 1abe27ebd519ac0854a844d18d679bf6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged
original trace 5b3ea90278cb5e18b6ad08cd32349c0e: 5 attempts, last outcome failed
- [ops] ops replay (INTERNAL, 16.5 ms)
- [ops] SELECT (CLIENT, 2.3 ms)
- [ops] UPDATE (CLIENT, 0.9 ms)
- [ops] UPDATE (CLIENT, 0.8 ms)
- [ops] INSERT (CLIENT, 0.7 ms)
- [ops] create attachments (PRODUCER, 1.1 ms) links=1 messenger.job.attempt=6
- [ops] INSERT (CLIENT, 0.7 ms)
- [ops] NOTIFY (CLIENT, 0.2 ms)
- [dispatcher] publish attachments (PRODUCER, 5.3 ms) messenger.job.attempt=6
- [worker] process attachments.process.v1 (CONSUMER, 281.7 ms) messenger.job.attempt=6 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.2 ms)
- [worker] S3 GetObject (CLIENT, 2.7 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 251.3 ms)
- [worker] S3 PutObject (CLIENT, 3.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/original
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/crop.jpg
- [worker] commit result (INTERNAL, 6.0 ms)
- [worker] SELECT (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.7 ms)
- [worker] INSERT (CLIENT, 0.6 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] UPDATE (CLIENT, 0.7 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] cleanup staging (INTERNAL, 5.9 ms)
- [worker] S3 DeleteObject (CLIENT, 3.3 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00
- [worker] UPDATE (CLIENT, 2.1 ms)
- [dispatcher] UPDATE (CLIENT, 1.3 ms)
replay trace 6d0327f64ceb5758c4a19b30aceed617: create span links -> ['5b3ea90278cb5e18b6ad08cd32349c0e'] (original upload trace 5b3ea90278cb5e18b6ad08cd32349c0e)
PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it
4 passed, 0 failed
+40
View File
@@ -0,0 +1,40 @@
# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence)
# date: 2026-11-01 20:46:08, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/run_persistence.py
# exit code: 0, 62.0 s
before: container, ID, started at
messenger-lab6-api-1 171a473b0d28 2026-11-01T17:43:16.63584434Z
messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:43:36.466962173Z
messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:31:14.251298706Z
messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:06:55.450862118Z
messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:07:17.636225869Z
messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:07:20.27137747Z
messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:22:20.690589644Z
messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:27:23.730596273Z
messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:04.263026279Z
messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:09:18.447414724Z
messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:30:47.892983401Z
messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:44:43.823155102Z
messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:40:41.919464505Z
messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:22:20.681510324Z
messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:38:33.102546081Z
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.␍
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
after (hook + waiting for readiness took 60.9 s): container, ID, started at, restarted?
messenger-lab6-api-1 171a473b0d28 2026-11-01T17:46:02.488153329Z same container: True, restarted: True
messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:46:02.317979748Z same container: True, restarted: True
messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:46:00.495060123Z same container: True, restarted: True
messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:45:57.449417453Z same container: True, restarted: True
messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:45:57.621731679Z same container: True, restarted: True
messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:46:05.984667885Z same container: True, restarted: True
messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:45:21.206788727Z same container: True, restarted: True
messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:45:56.829306342Z same container: True, restarted: True
messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:19.869545292Z same container: True, restarted: True
messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:45:20.880337926Z same container: True, restarted: True
messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:45:21.007029551Z same container: True, restarted: True
messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:45:19.748047785Z same container: True, restarted: True
messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:45:29.752000764Z same container: True, restarted: True
messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:45:50.068926735Z same container: True, restarted: True
messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:46:00.664535867Z same container: True, restarted: True
PASS: the hook restarted postgres, redis, s3, rabbitmq, api, janitor, dispatcher, worker, lb, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived
+10
View File
@@ -0,0 +1,10 @@
# race: lab2 races via the balancer
# date: 2026-11-01 20:42:41, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_race.py
# exit code: 0, 1.2 s
sent 100 messages from 10 clients in 0.30 s; read 100 in 4 pages, unique ids: 100
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
20 parallel registrations of one username: 201 x1, 409 x19
PASS: concurrent registration created exactly one user
2 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# redis: Redis as the session store
# date: 2026-11-01 20:42:56, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/redis_info.py
# exit code: 0, 3.2 s
session key: session:0fe0e5550e07d95d...; value is the user id: True
TTL of the key, seconds: 3599
sessions in Redis: 147
config appendonly = yes
config appendfsync = always
config maxmemory-policy = noeviction
aof_enabled:1
aof_last_write_status:ok
without password: NOAUTH Authentication required.
+53
View File
@@ -0,0 +1,53 @@
# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap
# date: 2026-11-01 20:38:59, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_s3_security.py
# exit code: 0, 10.5 s
anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied
anonymous head_object directly to S3 (from the data network) -> 403
anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied
anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied
anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied
the same file through the API for a chat member -> 200
PASS: the bucket is private: no anonymous read, list, write or policy read
API key: list buckets -> ['messenger-attachments']
API key: get own object -> allowed
API key: put object -> allowed
API key: delete object -> allowed
API key: list bucket -> allowed
API key: create another bucket -> denied: AccessDenied
API key: make bucket public (put policy) -> denied: NotImplemented
API key: read bucket policy -> denied: NoSuchBucketPolicy
API key: delete bucket -> denied: AccessDenied
API key: change versioning -> denied: NotImplemented
PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied
compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand)
127.0.0.1:8333 from the host -> connection refused
127.0.0.1:9333 from the host -> connection refused
127.0.0.1:8888 from the host -> connection refused
127.0.0.1:19333 from the host -> connection refused
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
secrets of grafana: ['grafana_admin_password']
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of lb: -
secrets of loki: -
secrets of migrate: ['database_url']
secrets of otel-collector: -
secrets of postgres: ['postgres_password']
secrets of prometheus: -
secrets of rabbitmq: ['rabbitmq_definitions']
secrets of redis: ['redis_auth']
secrets of s3: ['s3_config']
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
secrets of telemetry-init: -
secrets of tempo: -
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init
s3-init: s3-init: bucket messenger-attachments exists
s3-init: s3-init: api key can put/get/list/delete objects of its bucket
s3-init: s3-init: api key cannot create buckets (AccessDenied)
s3-init: s3-init: bucket is private (anonymous list/get/put -> [403])
after the repeated bootstrap: old file downloads, new upload works
PASS: bucket/user/policy bootstrap can be repeated safely
4 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment
# date: 2026-11-01 20:07:47, commit: 470d9d6 + uncommitted changes
# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080
# exit code: 0, 1.0 s
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+66
View File
@@ -0,0 +1,66 @@
# stack: stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services
# date: 2026-11-01 20:07:46, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/check_stack.py
# exit code: 0, 3.1 s
host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1
Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop
SERVICE CONTAINER ID STATUS PORTS
api 171a473b0d28 Up 26 seconds (healthy) 8080/tcp
api 1db2fedff78f Up 26 seconds (healthy) 8080/tcp
dispatcher 2f899202c4d1 Up 47 seconds (healthy)
grafana 23e7b8bb1b3a Up 49 seconds (healthy) 127.0.0.1:3000->3000/tcp
janitor 6dad6b402ed9 Up 26 seconds (healthy)
lb 0bffe8d8d7d2 Up 24 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
loki c62735c8baca Up 54 seconds 127.0.0.1:3100->3100/tcp
migrate dd9753ec4186 Exited (0) 48 seconds ago
otel-collector 8c416bdc0221 Up 55 seconds
postgres ced386ea9bbd Up 55 seconds (healthy)
prometheus 451bf5466e96 Up 55 seconds (healthy) 127.0.0.1:9090->9090/tcp
rabbitmq ec502659b532 Up 55 seconds (healthy)
redis 3c003ba9b32a Up 55 seconds (healthy)
s3 1e70a509e403 Up 55 seconds (healthy)
s3-init 64471229b155 Exited (0) 27 seconds ago
telemetry-init 809e227f3528 Exited (0) 55 seconds ago
tempo 3daf3848457f Up 54 seconds 127.0.0.1:3200->3200/tcp
worker cbd8138493bf Up 26 seconds (healthy)
container image cpus memory host ports networks
messenger-lab6-api-1 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry
messenger-lab6-api-2 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry
messenger-lab6-dispatcher-1 messenger-lab6-dispatcher 0.25 128 MiB - data,telemetry
messenger-lab6-grafana-1 grafana/grafana:12.4.14 0.50 384 MiB 127.0.0.1:3000 ops,telemetry
messenger-lab6-janitor-1 messenger-lab6-janitor 0.25 192 MiB - data,telemetry
messenger-lab6-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge
messenger-lab6-loki-1 grafana/loki:3.7.9 1.00 512 MiB 127.0.0.1:3100 ops,telemetry
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.164.0 0.50 384 MiB - telemetry
messenger-lab6-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data
messenger-lab6-prometheus-1 prom/prometheus:v3.15.1 1.00 512 MiB 127.0.0.1:9090 ops,telemetry
messenger-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data
messenger-lab6-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data
messenger-lab6-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data
messenger-lab6-tempo-1 grafana/tempo:3.1.1 1.00 512 MiB 127.0.0.1:3200 ops,telemetry
messenger-lab6-worker-1 messenger-lab6-worker 2.00 1536 MiB - data,telemetry
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
secrets of grafana: ['grafana_admin_password']
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of lb: -
secrets of loki: -
secrets of migrate: ['database_url']
secrets of otel-collector: -
secrets of postgres: ['postgres_password']
secrets of prometheus: -
secrets of rabbitmq: ['rabbitmq_definitions']
secrets of redis: ['redis_auth']
secrets of s3: ['s3_config']
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
secrets of telemetry-init: -
secrets of tempo: -
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
through http://127.0.0.1:8080 one session is served by 2 replicas: ['171a473b0d28', '1db2fedff78f']
/health/ready -> 200 from 1db2fedff78f
PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on 127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker
+26
View File
@@ -0,0 +1,26 @@
# student-tests: lab1/lab2 API tests via the balancer
# date: 2026-11-01 20:42:40, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_api.py
# exit code: 0, 2.8 s
PASS: duplicate username returns 409
PASS: invalid JSON returns 400
PASS: wrong and extra fields return 400
PASS: foreign chat returns 404
PASS: empty and too long messages return 400, 4000 chars is ok
PASS: pagination works across multiple pages, foreign cursor returns 400
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
PASS: chat list pagination
PASS: unicode text is stored as is
PASS: strings with U+0000 are stored and returned as is
PASS: user response has no password or hash
PASS: password length 12..128, blank password is rejected
PASS: Basic login returns opaque Bearer token with expiry
PASS: wrong password and unknown user both return the same 401
PASS: malformed Basic header returns 401
PASS: password with colon and unicode works
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
PASS: X-User-Id does not change the current user
PASS: 401 comes before 400 and 404
PASS: logout revokes only the current session
20 passed, 0 failed
+108
View File
@@ -0,0 +1,108 @@
# Сводка проверок (Docker-стенд, 2026-11-01)
Шаги запускались группами (`collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history`, затем каждый инцидент и `load` отдельно, затем `--from jobs-variants`); `load` выполнен дважды — в группе lab6 и в регрессии, evidence/load.txt и load.json — от второго прогона.
| Шаг | Что проверяет | Результат | Время, с | Файл |
| --- | --- | --- | --- | --- |
| check | offline contract check (make check) | PASS | 0 | evidence/check.txt |
| stack | stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services | PASS | 3 | evidence/stack.txt |
| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 1 | evidence/smoke.txt |
| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 6 | evidence/configs.txt |
| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 79 | evidence/telemetry.txt |
| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 1 | evidence/dashboard.txt |
| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 39 | evidence/telemetry-history.txt |
| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 162 | evidence/incident-worker.txt |
| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 286 | evidence/incident-db.txt |
| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 380 | evidence/incident-telemetry.txt |
| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 156 | evidence/incident-collector.txt |
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt |
| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 73 | evidence/jobs-limits.txt |
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt |
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 17 | evidence/jobs-outbox.txt |
| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 157 | evidence/jobs-crash.txt |
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 78 | evidence/jobs-retry.txt |
| jobs-janitor-replay | lab5: janitor does not delete objects re-written by replay (review fix) | PASS | 187 | evidence/jobs-janitor-replay.txt |
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 16 | evidence/jobs-concurrency.txt |
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt |
| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt |
| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 160 | evidence/consistency.txt |
| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 53 | evidence/upload-memory.txt |
| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt |
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt |
| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt |
| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt |
| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 12 | evidence/db.txt |
| redis | Redis as the session store | PASS | 3 | evidence/redis.txt |
| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt |
| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 42 | evidence/failover.txt |
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt |
| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 62 | evidence/persistence.txt |
# Замеры
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
| --- | --- | --- | --- | --- |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) |
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) |
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) |
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) |
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0.
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок.
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
| --- | --- | --- | --- |
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) |
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
| --- | --- | --- | --- | --- | --- |
| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 |
| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 |
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 |
| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 |
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 |
| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 |
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 |
| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 |
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет |
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет |
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия |
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready |
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены |
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
+9
View File
@@ -0,0 +1,9 @@
# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)
# date: 2026-11-01 20:09:51, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_telemetry.py history
# exit code: 0, 38.6 s
before restart: trace 0b9106c1057e9753186aec51b529946b found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 434
after restart of prometheus, loki, tempo: trace 0b9106c1057e9753186aec51b529946b (4 spans), 1 log record(s), metric at t-30s = 434 (same value)
PASS: persistent volumes keep traces, logs and metrics across restarts of the stores
1 passed, 0 failed
+145
View File
@@ -0,0 +1,145 @@
# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets
# date: 2026-11-01 20:09:11, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/test_telemetry.py request upload-chain retry-chain replay-link
# exit code: 0, 78.6 s
POST message: 201, X-Request-Id=df174aef-1d0d-4340-b2d7-907241684f5e, trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce (client traceparent)
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 4.6 ms) http.response.status_code=201
- [api] redis GET (CLIENT, 0.2 ms)
- [api] SELECT (CLIENT, 0.3 ms)
- [api] INSERT (CLIENT, 0.4 ms)
Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (4.6 ms)' event=http.request severity=INFO service=messenger-api trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce span_id=78cd8621fce4f3b8 request_id=df174aef-1d0d-4340-b2d7-907241684f5e
PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 13.5 ms) http.response.status_code=202
- [api] redis GET (CLIENT, 0.2 ms)
- [api] SELECT (CLIENT, 0.4 ms)
- [api] inspect image (INTERNAL, 0.5 ms)
- [api] INSERT (CLIENT, 0.6 ms)
- [api] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
- [api] UPDATE (CLIENT, 0.7 ms)
- [api] create attachments (PRODUCER, 2.4 ms) messenger.job.attempt=1
- [api] INSERT (CLIENT, 0.7 ms)
- [api] INSERT (CLIENT, 0.6 ms)
- [api] INSERT (CLIENT, 0.6 ms)
- [api] NOTIFY (CLIENT, 0.1 ms)
- [dispatcher] publish attachments (PRODUCER, 4.0 ms) messenger.job.attempt=1
- [dispatcher] UPDATE (CLIENT, 0.4 ms)
- [worker] process attachments.process.v1 (CONSUMER, 161.4 ms) messenger.job.attempt=1 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.3 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] S3 GetObject (CLIENT, 2.0 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 139.8 ms)
- [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/original
- [worker] S3 PutObject (CLIENT, 1.9 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 2.3 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/crop.jpg
- [worker] commit result (INTERNAL, 4.9 ms)
- [worker] SELECT (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] cleanup staging (INTERNAL, 4.7 ms)
- [worker] S3 DeleteObject (CLIENT, 2.2 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
- [worker] UPDATE (CLIENT, 2.2 ms)
worker logs with trace_id f5d1897588244cad5fc6ded83b930725: ['job.claimed', 'job.ready']
PASS: one trace f5d1897588244cad5fc6ded83b930725: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker)
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 12.6 ms) http.response.status_code=202
- [api] redis GET (CLIENT, 0.2 ms)
- [api] SELECT (CLIENT, 0.5 ms)
- [api] inspect image (INTERNAL, 0.3 ms)
- [api] INSERT (CLIENT, 0.6 ms)
- [api] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
- [api] UPDATE (CLIENT, 0.6 ms)
- [api] create attachments (PRODUCER, 1.4 ms) messenger.job.attempt=1
- [api] INSERT (CLIENT, 0.4 ms)
- [api] INSERT (CLIENT, 0.3 ms)
- [api] INSERT (CLIENT, 0.3 ms)
- [api] NOTIFY (CLIENT, 0.1 ms)
- [dispatcher] publish attachments (PRODUCER, 4.2 ms) messenger.job.attempt=1
- [dispatcher] UPDATE (CLIENT, 0.4 ms)
- [worker] process attachments.process.v1 (CONSUMER, 135.6 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] S3 GetObject (CLIENT, 2.6 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 123.0 ms)
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] UPDATE (CLIENT, 0.3 ms)
- [worker] create attachments (PRODUCER, 1.0 ms) messenger.job.attempt=2
- [worker] INSERT (CLIENT, 0.6 ms)
- [worker] NOTIFY (CLIENT, 0.2 ms)
- [dispatcher] publish attachments (PRODUCER, 5.2 ms) messenger.job.attempt=2
- [dispatcher] UPDATE (CLIENT, 0.5 ms)
- [worker] process attachments.process.v1 (CONSUMER, 144.9 ms) messenger.job.attempt=2 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 125.7 ms)
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/original
- [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 1.7 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/crop.jpg
- [worker] commit result (INTERNAL, 4.4 ms)
- [worker] SELECT (CLIENT, 0.4 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.2 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
- [worker] cleanup staging (INTERNAL, 4.5 ms)
- [worker] S3 DeleteObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
- [worker] UPDATE (CLIENT, 2.5 ms)
- [worker] INSERT (CLIENT, 0.4 ms)
worker log: 'job 7bd96513-9f85-471e-b553-ec49f9fe4560 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable
messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1
PASS: retry in one trace 5d34bb1eecbc90702d2f5e7de5b9bca6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged
original trace 38319800cdce5da47cb2b821dc03bf6e: 5 attempts, last outcome failed
- [ops] ops replay (INTERNAL, 12.3 ms)
- [ops] SELECT (CLIENT, 1.9 ms)
- [ops] UPDATE (CLIENT, 0.6 ms)
- [ops] UPDATE (CLIENT, 0.4 ms)
- [ops] INSERT (CLIENT, 0.5 ms)
- [ops] create attachments (PRODUCER, 0.8 ms) links=1 messenger.job.attempt=6
- [ops] INSERT (CLIENT, 0.5 ms)
- [ops] NOTIFY (CLIENT, 0.2 ms)
- [dispatcher] publish attachments (PRODUCER, 70.0 ms) messenger.job.attempt=6
- [dispatcher] UPDATE (CLIENT, 66.1 ms)
- [worker] process attachments.process.v1 (CONSUMER, 145.2 ms) messenger.job.attempt=6 messenger.job.outcome=ready
- [worker] SELECT (CLIENT, 0.5 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] UPDATE (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.2 ms)
- [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
- [worker] imaging (INTERNAL, 125.7 ms)
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/original
- [worker] S3 PutObject (CLIENT, 2.0 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/optimized.webp
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/thumbnail.jpg
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/crop.jpg
- [worker] commit result (INTERNAL, 4.4 ms)
- [worker] SELECT (CLIENT, 0.5 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.2 ms)
- [worker] INSERT (CLIENT, 0.2 ms)
- [worker] UPDATE (CLIENT, 0.4 ms)
- [worker] UPDATE (CLIENT, 0.3 ms)
- [worker] INSERT (CLIENT, 0.3 ms)
- [worker] cleanup staging (INTERNAL, 4.1 ms)
- [worker] S3 DeleteObject (CLIENT, 2.1 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f
- [worker] UPDATE (CLIENT, 1.8 ms)
replay trace 9417040b8f913c26894caaaf0996e442: create span links -> ['38319800cdce5da47cb2b821dc03bf6e'] (original upload trace 38319800cdce5da47cb2b821dc03bf6e)
PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it
4 passed, 0 failed
+124
View File
@@ -0,0 +1,124 @@
{
"date": "2026-11-01 20:42",
"api_limit_mib": 512,
"api_limit_cpus": 1.0,
"runs": [
{
"scenario": "10 x ~9.5 MiB PNG (noise)",
"concurrency": 10,
"file_bytes": [
9953047
],
"total_mib": 94.9,
"all_202": true,
"wall_seconds": 1.95,
"latency_s": {
"p50": 1.77,
"max": 1.95
},
"sampler_rounds": 16,
"replicas": [
{
"replica": "171a473b0d28",
"uploads_served": 5,
"rss_before_mib": 109.6,
"rss_peak_mib": 124.5,
"rss_growth_mib": 14.9,
"rss_peak_sampled_mib": 124.5,
"cgroup_peak_mib": 162.8,
"cgroup_peak_sampled_mib": 160.2
},
{
"replica": "1db2fedff78f",
"uploads_served": 5,
"rss_before_mib": 110.5,
"rss_peak_mib": 125.6,
"rss_growth_mib": 15.1,
"rss_peak_sampled_mib": 125.6,
"cgroup_peak_mib": 163.3,
"cgroup_peak_sampled_mib": 160.8
}
]
},
{
"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)",
"concurrency": 10,
"file_bytes": [
116714
],
"total_mib": 1.1,
"all_202": true,
"wall_seconds": 1.3,
"latency_s": {
"p50": 0.98,
"max": 1.3
},
"sampler_rounds": 9,
"replicas": [
{
"replica": "171a473b0d28",
"uploads_served": 5,
"rss_before_mib": 107.6,
"rss_peak_mib": 112.4,
"rss_growth_mib": 4.8,
"rss_peak_sampled_mib": 112.4,
"cgroup_peak_mib": 101.5,
"cgroup_peak_sampled_mib": 101.4
},
{
"replica": "1db2fedff78f",
"uploads_served": 5,
"rss_before_mib": 107.1,
"rss_peak_mib": 109.8,
"rss_growth_mib": 2.7,
"rss_peak_sampled_mib": 109.8,
"cgroup_peak_mib": 100.0,
"cgroup_peak_sampled_mib": 100.0
}
]
},
{
"scenario": "20 mixed: large + 40 Mpx + JPEG",
"concurrency": 20,
"file_bytes": [
1492,
116714,
9953047
],
"total_mib": 133.2,
"all_202": true,
"wall_seconds": 1.95,
"latency_s": {
"p50": 1.45,
"max": 1.95
},
"sampler_rounds": 11,
"replicas": [
{
"replica": "171a473b0d28",
"uploads_served": 10,
"rss_before_mib": 109.0,
"rss_peak_mib": 136.3,
"rss_growth_mib": 27.3,
"rss_peak_sampled_mib": 136.3,
"cgroup_peak_mib": 191.4,
"cgroup_peak_sampled_mib": 191.4
},
{
"replica": "1db2fedff78f",
"uploads_served": 10,
"rss_before_mib": 110.6,
"rss_peak_mib": 135.0,
"rss_growth_mib": 24.3,
"rss_peak_sampled_mib": 135.0,
"cgroup_peak_mib": 172.8,
"cgroup_peak_sampled_mib": 171.4
}
]
}
],
"single_file_extra_rss_mib": {
"naive": 161,
"streaming": 0
}
}
+13
View File
@@ -0,0 +1,13 @@
# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas
# date: 2026-11-01 20:42:33, commit: 470d9d6 + uncommitted changes
# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json
# exit code: 0, 53.4 s
client: Python 3.11.1 on Windows 10
preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...
large: 9953047 bytes; 40 Mpx: 116714 bytes
{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.77, "max": 1.95}, "sampler_rounds": 16, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 109.6, "rss_peak_mib": 124.5, "rss_growth_mib": 14.9, "rss_peak_sampled_mib": 124.5, "cgroup_peak_mib": 162.8, "cgroup_peak_sampled_mib": 160.2}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 110.5, "rss_peak_mib": 125.6, "rss_growth_mib": 15.1, "rss_peak_sampled_mib": 125.6, "cgroup_peak_mib": 163.3, "cgroup_peak_sampled_mib": 160.8}]}
{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.3, "latency_s": {"p50": 0.98, "max": 1.3}, "sampler_rounds": 9, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 107.6, "rss_peak_mib": 112.4, "rss_growth_mib": 4.8, "rss_peak_sampled_mib": 112.4, "cgroup_peak_mib": 101.5, "cgroup_peak_sampled_mib": 101.4}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 107.1, "rss_peak_mib": 109.8, "rss_growth_mib": 2.7, "rss_peak_sampled_mib": 109.8, "cgroup_peak_mib": 100.0, "cgroup_peak_sampled_mib": 100.0}]}
{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.45, "max": 1.95}, "sampler_rounds": 11, "replicas": [{"replica": "171a473b0d28", "uploads_served": 10, "rss_before_mib": 109.0, "rss_peak_mib": 136.3, "rss_growth_mib": 27.3, "rss_peak_sampled_mib": 136.3, "cgroup_peak_mib": 191.4, "cgroup_peak_sampled_mib": 191.4}, {"replica": "1db2fedff78f", "uploads_served": 10, "rss_before_mib": 110.6, "rss_peak_mib": 135.0, "rss_growth_mib": 24.3, "rss_peak_sampled_mib": 135.0, "cgroup_peak_mib": 172.8, "cgroup_peak_sampled_mib": 171.4}]}
one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB
PASS: all uploads succeeded; largest RSS growth of one replica: 27.3 MiB (limit 512 MiB); results in evidence/upload-memory.json
+51
View File
@@ -0,0 +1,51 @@
"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1).
Тест создаёт в общем volume /faults файлы <точка>.<n>, по одному на срабатывание:
docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1'
Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса)
и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски.
Точки:
dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish)
dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана
worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет
worker_crash_after_commit — COMMIT результата был, ack ещё нет
worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease
worker_s3_error — временная ошибка S3 при записи результата
imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения
"""
import logging
import os
import sys
from pathlib import Path
import settings
log = logging.getLogger("messenger.faults")
def hit(point: str) -> bool:
if not settings.FAULT_INJECTION:
return False
directory = Path(settings.FAULTS_DIR)
try:
candidates = sorted(directory.glob(point + ".*"))
except OSError:
return False
for path in candidates:
try:
path.unlink()
except FileNotFoundError:
continue # этот файл забрал другой процесс
log.warning("FAULT INJECTION: %s", point)
return True
return False
def crash(point: str) -> None:
"""Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений."""
if hit(point):
sys.stderr.flush()
os._exit(137)
+209
View File
@@ -0,0 +1,209 @@
"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами.
python imaging.py <source> <out_dir> <content_type> <max_pixels>
Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя
ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память,
IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker
убивает его по таймауту JOB_TIMEOUT_SECONDS.
Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру.
Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout.
Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти;
прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной.
Правила вариантов (contracts/README.md):
- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз
(минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160.
- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются).
- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется.
Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное
отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job
пишет в те же ключи S3 те же байты.
"""
import json
import os
import resource
import sys
import time
def apply_limits() -> None:
"""Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов."""
memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024
cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60"))
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024))
if __name__ == "__main__":
apply_limits()
from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом
PIPELINE_VERSION = 1
THUMB_BOX = 256
CROP_SIZE = 128
OPTIMIZED_BOX = 1280
THUMB_QUALITY = 85
CROP_QUALITY = 85
WEBP_QUALITY = 80
WEBP_METHOD = 4
EXIT_INVALID = 3
EXIT_TOO_LARGE = 4
EXIT_MEMORY = 5
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose)
ORIENTATION = {
2: [Image.Transpose.FLIP_LEFT_RIGHT],
3: [Image.Transpose.ROTATE_180],
4: [Image.Transpose.FLIP_TOP_BOTTOM],
5: [Image.Transpose.TRANSPOSE],
6: [Image.Transpose.ROTATE_270],
7: [Image.Transpose.TRANSVERSE],
8: [Image.Transpose.ROTATE_90],
}
class TooLarge(Exception):
pass
def fit(width: int, height: int, box: int) -> tuple[int, int]:
"""Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float."""
if width <= box and height <= box:
return width, height
if width >= height:
return box, max(1, height * box // width)
return max(1, width * box // height), box
def orientation_of(image: Image.Image) -> int:
try:
value = image.getexif().get(0x0112, 1)
except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем
return 1
return value if value in ORIENTATION else 1
def normalize(image: Image.Image) -> Image.Image:
"""Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность)."""
mode = image.mode
transparent = "transparency" in image.info
if mode in ("I;16", "I;16B", "I;16L", "I"):
image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L")
mode = "L"
if mode == "1":
return image.convert("L")
if mode in ("P", "PA"):
return image.convert("RGBA" if transparent or mode == "PA" else "RGB")
if mode in ("LA", "La", "RGBa"):
return image.convert("RGBA")
if mode in ("L", "RGB") and transparent:
return image.convert("RGBA")
if mode in ("L", "RGB", "RGBA"):
return image
return image.convert("RGB") # CMYK, YCbCr, LAB, ...
def on_white(image: Image.Image) -> Image.Image:
"""Прозрачность -> белый фон (для JPEG)."""
if image.mode != "RGBA":
return image
background = Image.new("RGB", image.size, (255, 255, 255))
background.paste(image, mask=image.getchannel("A"))
return background
def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict:
Image.MAX_IMAGE_PIXELS = max_pixels
with Image.open(source) as image:
if image.format != FORMATS.get(content_type):
raise ValueError("format does not match stored content type")
width, height = image.size
if width * height > max_pixels:
raise TooLarge()
orientation = orientation_of(image)
icc = image.info.get("icc_profile")
swap = orientation in (5, 6, 7, 8)
oriented = (height, width) if swap else (width, height)
# Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128
# не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной.
target = fit(*oriented, OPTIMIZED_BOX)
stored_target = (target[1], target[0]) if swap else target
if image.format == "JPEG":
image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает
image.load()
work = normalize(image)
if work.size != stored_target:
work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0)
elif work is image:
work = work.copy()
for step in ORIENTATION.get(orientation, []):
work = work.transpose(step)
extra = {"icc_profile": icc} if icc else {}
results = []
optimized = work if work.mode != "L" else work.convert("RGB")
path = os.path.join(out_dir, "optimized.webp")
optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra)
results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width,
"height": optimized.height, "file": "optimized.webp"})
thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии
thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS)
thumb = on_white(thumb)
path = os.path.join(out_dir, "thumbnail.jpg")
thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra)
results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width,
"height": thumb.height, "file": "thumbnail.jpg"})
side = min(work.size)
left, top = (work.width - side) // 2, (work.height - side) // 2
crop = work.crop((left, top, left + side, top + side))
crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS))
path = os.path.join(out_dir, "crop.jpg")
crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra)
results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE,
"file": "crop.jpg"})
return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height,
"orientation": orientation}, "variants": results}
def injected_fault() -> None:
"""Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1)."""
fault = os.environ.get("FAULT_IMAGING", "")
if fault == "hang":
time.sleep(3600)
elif fault == "invalid":
raise ValueError("injected invalid image")
elif fault == "oom":
blocks = []
while True:
blocks.append(bytearray(64 * 1024 * 1024))
def main() -> int:
source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
try:
injected_fault()
result = process(source, out_dir, content_type, max_pixels)
except (TooLarge, Image.DecompressionBombError):
return EXIT_TOO_LARGE
except MemoryError:
return EXIT_MEMORY
except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError):
# Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу.
return EXIT_INVALID
print(json.dumps(result), flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,13 @@
# Grafana provisioning: dashboard из файлов (infra/grafana/dashboards/*.json монтируется только для чтения).
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 30
options:
path: /etc/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,67 @@
# Grafana provisioning: источники данных (импортируются при старте, ручной настройки нет).
# Grafana сама ничего не хранит из трёх сигналов — только интерфейс и запросы к хранилищам.
# "$$" — буквальный "$" (иначе Grafana подставит переменную окружения).
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
prometheusType: Prometheus
prometheusVersion: 3.15.1
timeInterval: 15s
manageAlerts: false # правила алертов в Git (infra/prometheus/rules), Grafana их только показывает
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
maxLines: 1000
# trace_id — structured metadata записи (OTLP): ссылка «лог -> трасса» в Tempo.
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
urlDisplayLabel: Открыть трассу
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
# «спан -> логи»: те же trace_id в Loki, окно ±5 минут вокруг спана.
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: false
filterBySpanID: false
customQuery: true
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__trace.traceId}"'
# «спан -> метрики»: задержка и ошибки сервиса за время спана.
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
queries:
- name: API p95
query: 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'
nodeGraph:
enabled: true
search:
hide: false
lokiSearch:
datasourceUid: loki
+77
View File
@@ -0,0 +1,77 @@
# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики).
# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить
# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky).
global
log stdout format raw local0 info
maxconn 4096
# При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с.
hard-stop-after 10s
# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик.
# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска.
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s # перечитывать список реплик раз в 2 с
hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с
hold nx 2s
hold other 2s
hold refused 2s
hold timeout 2s
accepted_payload_size 8192
defaults
mode http
log global
option httplog
option dontlog-normal # в лог только ошибки, а не каждый запрос
timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос
timeout client 30s
# Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка
# при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт,
# сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT).
timeout server 30s
timeout http-request 10s
timeout http-keep-alive 5s
timeout queue 5s
timeout check 1s
# Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода.
# Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key.
retries 2
option redispatch 1
frontend api_in
bind :8080
option forwardfor
default_backend api
backend api
balance roundrobin
# Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis.
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
http-check expect status 200
# До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT.
# Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP.
# observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки.
server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
# Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis
# или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате
# контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает.
# Пока готова хоть одна реплика, эти записи не используются.
# allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend).
server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
option allbackups
# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1).
# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда.
frontend stats
bind :8404
monitor-uri /lb-health
monitor fail if { nbsrv(api) lt 1 }
stats enable
stats uri /stats
stats refresh 2s
+52
View File
@@ -0,0 +1,52 @@
# Loki 3.7.9 — хранилище ЛОГОВ (single binary, файловая система в volume lokidata).
# Логи приходят от Collector по OTLP (/otlp/v1/logs). Ресурсные атрибуты service.name,
# service.instance.id, deployment.environment.name становятся индексными метками (service_name, ...);
# атрибуты записи (event, request_id, trace_id, span_id, severity) — structured metadata, по ним
# фильтруют без индекса: {service_name="messenger-api"} | trace_id="<id>".
# Проверка: loki -config.file=loki.yaml -verify-config
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9095
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true # нужно для OTLP: trace_id/span_id и атрибуты записи
retention_period: 168h # 7 дней учебной истории
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
max_query_lookback: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
pattern_ingester:
enabled: false
analytics:
reporting_enabled: false
+129
View File
@@ -0,0 +1,129 @@
# OpenTelemetry Collector (otel/opentelemetry-collector-contrib:0.164.0) — единственный приёмник
# телеметрии приложения. Не хранит данные: принимает OTLP, ограничивает память, пакетирует и
# отправляет в три хранилища. Проверка конфигурации: otelcol-contrib validate --config=collector.yaml
#
# api/worker/dispatcher/janitor --OTLP/HTTP :4318--> receiver otlp
# traces -> memory_limiter -> attributes/scrub -> batch -> otlp_grpc/tempo (Tempo :4317)
# logs -> memory_limiter -> attributes/scrub -> batch -> otlp_http/loki (Loki /otlp)
# metrics -> memory_limiter -> batch -> otlp_http/prometheus (Prometheus /api/v1/otlp)
#
# Порты 4317/4318/8888/13133 слушаются только во внутренней сети telemetry (в compose не публикуются).
#
# Поведение при недоступном хранилище (ограниченная буферизация, RAM не растёт бесконечно):
# - у каждого exporter своя очередь в памяти (sending_queue) фиксированного размера: заполнилась ->
# новые пакеты отбрасываются и считаются (otelcol_exporter_enqueue_failed_*);
# - повтор с экспоненциальной задержкой не дольше max_elapsed_time, потом пакет отбрасывается
# (otelcol_exporter_send_failed_*);
# - memory_limiter: при приближении к лимиту receiver отказывает клиентам (SDK получает ошибку и
# сам отбрасывает данные в своей ограниченной очереди), процесс не уходит в OOM;
# - остальные pipeline работают независимо: упал Tempo — логи и метрики продолжают доходить.
# Очередь только в памяти: при рестарте Collector неотправленное теряется (осознанный компромисс,
# file_storage не нужен для учебного стенда).
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
# Первым в каждом pipeline: проверка heap Go раз в секунду. Мягкий предел = limit - spike = 160 MiB
# (receiver начинает отказывать), жёсткий 200 MiB (принудительный GC). Лимит контейнера 384 MiB:
# RSS включает ещё runtime Go и отображённый в память бинарник contrib (~150 MiB, вытесняемые страницы).
memory_limiter:
check_interval: 1s
limit_mib: 200
spike_limit_mib: 40
# Защита в глубину: приложение не пишет секреты в атрибуты, но если бы инструментирование ошиблось,
# такие ключи не дойдут до хранилищ. Удаляются по шаблону имени ключа (спаны и логи).
attributes/scrub:
actions:
- pattern: (?i).*(password|passwd|authorization|cookie|secret|token|api[_.-]?key|signature).*
action: delete
- key: db.statement.parameters
action: delete
- key: db.query.parameter
action: delete
batch:
timeout: 2s
send_batch_size: 1024
send_batch_max_size: 2048
exporters:
otlp_grpc/tempo:
endpoint: tempo:4317
tls:
insecure: true # lab6: внутренняя сеть; TLS/mTLS по trust matrix — в lab7
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200 # пакетов (до 2048 спанов каждый)
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 120s
otlp_http/loki:
endpoint: http://loki:3100/otlp # -> /otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 120s
otlp_http/prometheus:
endpoint: http://prometheus:9090/api/v1/otlp # -> /api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 1 # один поток: точки одной серии приходят по порядку
queue_size: 100
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 120s
service:
extensions: [health_check]
telemetry:
logs:
level: info
# Собственные метрики Collector (принято/отправлено/отброшено, размер очередей, RSS) —
# Prometheus забирает их pull-ом с otel-collector:8888. Так потери телеметрии видны и считаются.
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp_grpc/tempo]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp_http/loki]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/prometheus]
+42
View File
@@ -0,0 +1,42 @@
# Prometheus 3.15.1 — хранилище МЕТРИК. Принимает метрики приложения push-ом по OTLP от Collector
# (флаг --web.enable-otlp-receiver, путь /api/v1/otlp/v1/metrics) и сам забирает (pull) служебные
# метрики Collector, Loki и Tempo. Вычисляет recording rules (SLI) и alert rules; состояние алертов
# (pending/firing/resolved) видно на /alerts и через /api/v1/alerts — внешняя доставка не нужна.
# Проверка: promtool check config prometheus.yml
global:
scrape_interval: 15s
scrape_timeout: 5s
evaluation_interval: 15s
# Как имена OTLP переводятся в имена Prometheus (фиксируем явно, чтобы dashboard не сломался при
# смене значения по умолчанию): точки -> подчёркивания, суффиксы единиц и _total.
# http.server.request.duration (s, histogram) -> http_server_request_duration_seconds_{bucket,sum,count}
# атрибут http.route -> метка http_route; service.name -> job; service.instance.id -> instance.
otlp:
translation_strategy: UnderscoreEscapingWithSuffixes
storage:
tsdb:
# Collector после недоступности Prometheus дошлёт пакеты из очереди с прошлыми временными метками.
out_of_order_time_window: 10m
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
# Собственные метрики Collector: принято/отправлено/отброшено по сигналам, очереди, RSS.
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: loki
static_configs:
- targets: ["loki:3100"]
- job_name: tempo
static_configs:
- targets: ["tempo:3200"]
+113
View File
@@ -0,0 +1,113 @@
# Alert rules (Prometheus). Внешней доставки нет: состояние inactive/pending/firing видно на
# http://127.0.0.1:9090/alerts, через GET /api/v1/alerts и в Grafana (Alerting -> data source rules);
# метрика ALERTS{alertstate="firing"} хранится в TSDB, поэтому видно и когда алерт стал resolved.
# Каждое правило: условие, окно rate, порог, for, описание и ссылка на локальный runbook.
# Проверка: promtool check rules alerts.yml; promtool test rules ../tests/alerts_test.yml
groups:
- name: messenger-alerts
interval: 15s
rules:
# (1) Высокая доля ошибок API. Окно 1m, порог 5% (в 5 раз выше бюджета SLO-1 = 1%), for 1m,
# и не меньше 0,2 запроса/с, чтобы единичный 503 при пустом трафике не будил дежурного.
- alert: MessengerHighErrorRate
expr: |
(
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or on() vector(0))
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
) > 0.05
and on()
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) > 0.2
for: 1m
labels:
severity: page
service: messenger-api
slo: availability
annotations:
summary: "API: {{ $value | humanizePercentage }} бизнес-запросов завершаются 5xx (порог 5%)"
description: >-
Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше
минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка
кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу.
runbook_url: docs/runbooks.md#messengerhigherrorrate
# (2) Растущий lag очереди: возраст старейшего ГОТОВОГО к обработке задания (available_at прошёл,
# статус queued) больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно,
# так что правило ловит именно рост, а не единичный всплеск глубины очереди.
- alert: MessengerQueueLagGrowing
expr: max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"}) > 30
for: 1m
labels:
severity: page
service: messenger-worker
slo: time_to_ready
annotations:
summary: "Очередь attachments.process.v1: старейшее задание ждёт {{ $value | humanizeDuration }}"
description: >-
Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с
дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди,
занятость worker (busy/capacity) и ошибки worker в логах.
runbook_url: docs/runbooks.md#messengerqueuelaggrowing
# (3) Ни одного consumer у очереди: worker остановлен/упал. Дополняет (2): срабатывает даже без нагрузки.
- alert: MessengerWorkerNoConsumers
expr: max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}) == 0
for: 1m
labels:
severity: warning
service: messenger-worker
annotations:
summary: "У очереди attachments.process.v1 нет consumers"
description: "RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру."
runbook_url: docs/runbooks.md#messengerworkernoconsumers
# (4) Насыщение пула соединений API: запросы ждут соединение дольше минуты.
- alert: MessengerDbPoolSaturated
expr: max by (instance) (db_client_connection_pending_requests{job="messenger-api"}) > 0
for: 1m
labels:
severity: warning
service: messenger-api
annotations:
summary: "Пул PostgreSQL реплики {{ $labels.instance }}: {{ $value }} запросов ждут соединение"
description: "Все DB_POOL_MAX соединений заняты. Причина обычно — медленные запросы или блокировки в PostgreSQL, а не нехватка соединений."
runbook_url: docs/runbooks.md#messengerdbpoolsaturated
# (5) Collector не может доставить телеметрию в Tempo/Loki/Prometheus: либо пакеты уже теряются
# (send_failed — истёк повтор 120 с у пакета, enqueue_failed — очередь полна), либо очередь exporter
# дольше минуты заполнена больше чем на 5% (хранилище не принимает, пакеты копятся). Во время пакет
# ждёт в очереди его окно повторов не идёт, поэтому без второго условия алерт сработал бы только
# после первых потерь, то есть через 2+ минуты недоступности.
- alert: TelemetryExportFailing
expr: |
sum by (exporter) (
rate(otelcol_exporter_send_failed_spans[2m])
or rate(otelcol_exporter_send_failed_log_records[2m])
or rate(otelcol_exporter_send_failed_metric_points[2m])
or rate(otelcol_exporter_enqueue_failed_spans[2m])
or rate(otelcol_exporter_enqueue_failed_log_records[2m])
or rate(otelcol_exporter_enqueue_failed_metric_points[2m])
) > 0
or
max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.05
for: 1m
labels:
severity: warning
service: otel-collector
annotations:
summary: "Collector не доставляет телеметрию: exporter {{ $labels.exporter }}"
description: "Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*."
runbook_url: docs/runbooks.md#telemetryexportfailing
# (6) Collector недоступен: Prometheus не может забрать его служебные метрики. Метрики приложения
# в этот период не доходят (их везёт Collector), поэтому алерт смотрит на up самого Collector.
- alert: TelemetryCollectorDown
expr: up{job="otel-collector"} == 0
for: 1m
labels:
severity: warning
service: otel-collector
annotations:
summary: "OTel Collector недоступен"
description: "Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском."
runbook_url: docs/runbooks.md#telemetrycollectordown
+50
View File
@@ -0,0 +1,50 @@
# Recording rules: SLI и агрегаты для dashboard. Проверка: promtool check rules recording.yml
#
# Числитель/знаменатель ошибок API (contracts/observability.md):
# знаменатель — все бизнес-запросы API (job="messenger-api", маршрут НЕ /health/*), любой код;
# числитель — из них ответы 5xx (отказ сервера: БД/Redis/S3 недоступны, внутренняя ошибка).
# 4xx (неверный пароль, 404 чужого чата, 409) — поведение клиента, в ошибки SLO не входят.
# Percentile — только из buckets гистограммы, просуммированных по всем репликам ДО histogram_quantile
# (sum by (le)); p95 отдельных реплик не усредняются.
groups:
- name: messenger-api-sli
interval: 15s
rules:
- record: messenger:http_requests:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
- record: messenger:http_errors_5xx:rate1m
expr: |
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m]))
or on() vector(0)
- record: messenger:http_error_ratio:rate1m
expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m
- record: messenger:http_request_duration_seconds:p50_5m
expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
- record: messenger:http_request_duration_seconds:p95_5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
- record: messenger:http_request_duration_seconds:p99_5m
expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
# SLO (учебное скользящее окно 1 ч; в эксплуатации то же выражение с [28d]):
# SLO-1 доступность: >= 99% бизнес-запросов без 5xx;
# SLO-2 задержка: >= 95% POST /api/v1/chats/{chat_id}/messages быстрее 250 мс;
# SLO-3 фоновая обработка: >= 95% вложений ready не позже 30 с после приёма (202).
- name: messenger-slo
interval: 30s
rules:
- record: messenger:slo_availability:ratio_1h
expr: |
1 - (
(sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1h])) or on() vector(0))
/ sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h]))
)
- record: messenger:slo_availability:error_budget_remaining_1h
expr: 1 - (1 - messenger:slo_availability:ratio_1h) / (1 - 0.99)
- record: messenger:slo_latency_messages:ratio_1h
expr: |
sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx", le="0.25"}[1h]))
/ sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}[1h]))
- record: messenger:slo_time_to_ready:ratio_1h
expr: |
sum(increase(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h]))
/ sum(increase(messenger_attachment_time_to_ready_seconds_count[1h]))
+121
View File
@@ -0,0 +1,121 @@
# Unit-тесты alert rules на синтетических рядах (без стенда):
# promtool test rules infra/prometheus/tests/alerts_test.yml
# docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml
# Проверяют порог, окно и for: правило не срабатывает раньше времени и при пустом трафике.
rule_files:
- ../rules/alerts.yml
evaluation_interval: 15s
tests:
# MessengerHighErrorRate: 2 rps успешных, с 3-й минуты +1 rps 5xx (доля 33%).
- interval: 15s
input_series:
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}'
values: '0+30x60'
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="5xx"}'
values: '0x12 0+15x48'
# health-проверки с ошибками не считаются (знаменатель и числитель без /health)
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/health/ready", http_response_status_class="5xx"}'
values: '0+100x60'
alert_rule_test:
- eval_time: 3m
alertname: MessengerHighErrorRate
exp_alerts: []
- eval_time: 3m45s # условие выполняется, но for: 1m ещё не прошёл
alertname: MessengerHighErrorRate
exp_alerts: []
- eval_time: 5m
alertname: MessengerHighErrorRate
exp_alerts:
- exp_labels: {severity: page, service: messenger-api, slo: availability}
exp_annotations:
summary: 'API: 33.33% бизнес-запросов завершаются 5xx (порог 5%)'
description: Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу.
runbook_url: docs/runbooks.md#messengerhigherrorrate
# Пустой трафик: единичные 5xx при 0,05 rps не будят дежурного (порог 0,2 запроса/с).
- interval: 15s
input_series:
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/users/me", http_response_status_class="5xx"}'
values: '0+1x40'
alert_rule_test:
- eval_time: 9m
alertname: MessengerHighErrorRate
exp_alerts: []
# MessengerQueueLagGrowing: возраст старейшего задания растёт со 2-й минуты, затем worker вернулся.
- interval: 15s
input_series:
- series: 'messenger_queue_oldest_age_seconds{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}'
values: '0x8 0+15x20 0x20'
- series: 'messenger_queue_consumers{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}'
values: '2x8 0x20 2x20'
alert_rule_test:
- eval_time: 2m30s # возраст 30 с — ещё не больше порога
alertname: MessengerQueueLagGrowing
exp_alerts: []
- eval_time: 4m
alertname: MessengerQueueLagGrowing
exp_alerts:
- exp_labels: {severity: page, service: messenger-worker, slo: time_to_ready}
exp_annotations:
summary: 'Очередь attachments.process.v1: старейшее задание ждёт 1m 45s'
description: 'Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, занятость worker (busy/capacity) и ошибки worker в логах.'
runbook_url: docs/runbooks.md#messengerqueuelaggrowing
- eval_time: 4m
alertname: MessengerWorkerNoConsumers
exp_alerts:
- exp_labels: {severity: warning, service: messenger-worker}
exp_annotations:
summary: У очереди attachments.process.v1 нет consumers
description: 'RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру.'
runbook_url: docs/runbooks.md#messengerworkernoconsumers
- eval_time: 9m # worker вернулся, очередь разобрана -> resolved
alertname: MessengerQueueLagGrowing
exp_alerts: []
# TelemetryExportFailing и TelemetryCollectorDown
- interval: 15s
input_series:
- series: 'otelcol_exporter_send_failed_spans{exporter="otlp_grpc/tempo"}'
values: '0x8 0+50x20'
- series: 'up{job="otel-collector"}'
values: '1x8 0x20'
alert_rule_test:
- eval_time: 5m
alertname: TelemetryExportFailing
exp_alerts:
- exp_labels: {severity: warning, service: otel-collector, exporter: otlp_grpc/tempo}
exp_annotations:
summary: 'Collector не доставляет телеметрию: exporter otlp_grpc/tempo'
description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.'
runbook_url: docs/runbooks.md#telemetryexportfailing
- eval_time: 5m
alertname: TelemetryCollectorDown
exp_alerts:
- exp_labels: {severity: warning, service: otel-collector, job: otel-collector}
exp_annotations:
summary: OTel Collector недоступен
description: Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском.
runbook_url: docs/runbooks.md#telemetrycollectordown
# TelemetryExportFailing по заполнению очереди: Loki недоступен, потерь ещё нет, но очередь растёт.
- interval: 15s
input_series:
- series: 'otelcol_exporter_queue_size{exporter="otlp_http/loki"}'
values: '0x8 0+2x30'
- series: 'otelcol_exporter_queue_capacity{exporter="otlp_http/loki"}'
values: '200x38'
alert_rule_test:
- eval_time: 3m # 6 пакетов из 200 (3%) — ниже порога 5%
alertname: TelemetryExportFailing
exp_alerts: []
- eval_time: 5m
alertname: TelemetryExportFailing
exp_alerts:
- exp_labels: {severity: warning, service: otel-collector, exporter: otlp_http/loki}
exp_annotations:
summary: 'Collector не доставляет телеметрию: exporter otlp_http/loki'
description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.'
runbook_url: docs/runbooks.md#telemetryexportfailing
+22
View File
@@ -0,0 +1,22 @@
# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json
# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет:
# при импорте определений на старте пользователь по умолчанию не создаётся.
definitions.import_backend = local_filesystem
definitions.local.path = /run/secrets/rabbitmq_definitions
definitions.skip_if_unchanged = true
listeners.tcp.default = 5672
loopback_users = none
# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет),
# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста).
vm_memory_high_watermark.absolute = 320MB
disk_free_limit.absolute = 256MB
# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение
# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3.
consumer_timeout = 300000
log.console = true
log.console.level = info
log.file = false
+13
View File
@@ -0,0 +1,13 @@
bind 0.0.0.0
port 6379
protected-mode yes
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 64mb
maxmemory-policy noeviction
include /run/secrets/redis_auth
+39
View File
@@ -0,0 +1,39 @@
# Tempo 3.1.1 — хранилище ТРАСС (single binary: distributor -> live-store -> блоки в volume tempodata,
# без Kafka). Трассы приходят от Collector по OTLP gRPC :4317 (только внутренняя сеть telemetry),
# запросы Grafana и тестов — HTTP :3200 (/api/traces/<trace_id>, /api/search, TraceQL).
# Проверка: tempo -config.file=tempo.yaml -config.verify=true
stream_over_http_enabled: true
server:
http_listen_port: 3200
grpc_listen_port: 9095
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/traces
# Хранение 7 дней (как у Loki); удаление старых блоков делают backend-scheduler/worker.
backend_scheduler:
local_work_path: /var/tempo/scheduler
provider:
compaction:
compaction:
block_retention: 168h
backend_worker:
compaction:
block_retention: 168h
usage_report:
reporting_enabled: false
+322
View File
@@ -0,0 +1,322 @@
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
python janitor.py --once [--dry-run] [--grace-seconds N]
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
в очередь. Удалить объект staging (идемпотентно), затем строку.
2. Staging после безопасной фиксации:
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
(обычно это делает worker сразу после ack; здесь — если он не успел);
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
queued/processing не трогаются никогда: это текущая работа.
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
4. Обход bucket по всем prefix (объекты старше grace):
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
- attachments/<id>/...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен:
вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention);
оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing
и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их
теми же ключами, и удаление устаревшим проходом сломало бы новый результат.
Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения:
replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним.
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
janitor не удаляет то, что ещё пишется.
"""
import argparse
import json
import logging
import re
import signal
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
import settings
import telemetry
telemetry.setup("messenger-janitor")
import pipeline # noqa: E402
import storage # noqa: E402
log = logging.getLogger("messenger.janitor")
JANITOR_LOCK = 20260404
BATCH = 100
STOP = threading.Event()
ACTOR = "janitor"
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def idle(seconds: float) -> bool:
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
deadline = time.monotonic() + seconds
while (left := deadline - time.monotonic()) > 0:
touch_alive()
if STOP.wait(min(left, 10)):
return True
return False
def connect():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
"""Шаг 1. Возвращает (найдено, удалено)."""
found = deleted = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, source_key FROM attachments
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
ORDER BY created_at LIMIT %s
FOR UPDATE SKIP LOCKED""",
(grace, BATCH),
).fetchall()
found += len(rows)
if dry_run or not rows:
return found, deleted
for row in rows:
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
deleted += 1
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
if len(rows) < BATCH:
return found, deleted
return found, deleted
def cleanup_staging(conn, dry_run: bool) -> dict:
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
WHERE a.source_key IS NOT NULL
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
(settings.STAGING_RETENTION_SECONDS, BATCH),
).fetchall()
if dry_run or not rows:
if dry_run:
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
break
for row in rows:
storage.delete(row["source_key"])
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
if row["state"] == "ready":
report["staging_ready_deleted"] += 1
else:
report["staging_failed_expired"] += 1
if row["job_id"] is not None:
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
"replay no longer possible")
if len(rows) < BATCH:
break
report["staging_kept_for_replay"] = conn.execute(
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
return report
def cleanup_outbox(conn, dry_run: bool) -> int:
if dry_run:
return conn.execute(
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
return conn.execute(
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
def classify_final(conn, keys: list[str]) -> set[str]:
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
ids = {}
for key in keys:
match = FINAL_KEY_RE.match(key)
if match:
try:
ids[key] = UUID(match.group(1))
except ValueError:
pass
unique = list(set(ids.values()))
rows = {r["id"]: r for r in conn.execute(
"""SELECT a.id, a.state, a.object_key, a.source_key,
ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
orphans = set()
for key in keys:
attachment_id = ids.get(key)
row = rows.get(attachment_id) if attachment_id else None
if row is None:
orphans.add(key)
elif row["state"] == "failed" and row["source_key"] is None:
# failed без исходника — конечное состояние: replay отказывает («source file is no longer in
# staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи.
orphans.add(key)
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
orphans.add(key)
return orphans
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
started = datetime.now(timezone.utc)
cutoff = started - timedelta(seconds=grace)
scanned = orphans = orphans_deleted = 0
listed = set()
def orphan_keys(prefix, items):
keys = [key for key, _ in items]
if prefix == storage.FINAL_PREFIX:
return classify_final(conn, keys)
known = {r["source_key"] for r in conn.execute(
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
return {key for key in keys if key not in known}
def delete_if_orphan(prefix, key) -> bool:
"""Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker."""
if prefix != storage.FINAL_PREFIX:
# staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется,
# поэтому «нет строки сейчас» означает «не будет никогда».
if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None:
return False
storage.delete(key)
return True
match = FINAL_KEY_RE.match(key)
try:
attachment_id = UUID(match.group(1)) if match else None
except ValueError:
attachment_id = None
with conn.transaction():
if attachment_id is not None:
# FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но
# конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT.
conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,))
if key not in classify_final(conn, [key]):
return False
storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе
return True
def check(prefix, items):
nonlocal orphans, orphans_deleted
candidates = orphan_keys(prefix, items)
for key, modified in items:
if key not in candidates or modified >= cutoff:
continue
orphans += 1
if not dry_run and delete_if_orphan(prefix, key):
orphans_deleted += 1
log.info("deleted orphan object %s", key)
for prefix in storage.ALL_PREFIXES:
page = []
for key, modified, _size in storage.list_objects(prefix):
scanned += 1
listed.add(key)
page.append((key, modified))
if len(page) >= 1000:
check(prefix, page)
page = []
if page:
check(prefix, page)
missing = []
for r in conn.execute(
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
missing.append(str(r["id"]))
if missing:
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
", ".join(missing[:20]))
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
"missing_objects": len(missing)}
def run_once(grace: int, dry_run: bool) -> dict | None:
# Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети.
with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span:
report = _run_once(grace, dry_run)
if report:
span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items()
if isinstance(v, (int, float, bool))})
return report
def _run_once(grace: int, dry_run: bool) -> dict | None:
started = time.monotonic()
with connect() as conn:
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
log.info("another janitor is running, skip")
return None
try:
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
report["seconds"] = round(time.monotonic() - started, 3)
return report
def main():
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
parser.add_argument("--once", action="store_true", help="one pass and exit")
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
args = parser.parse_args()
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
while True:
touch_alive()
try:
report = run_once(args.grace_seconds, args.dry_run)
if report is not None:
print(json.dumps(report), flush=True)
log.info("janitor pass done", extra={"event": "janitor.pass", **{
k: v for k, v in report.items() if isinstance(v, (int, float, bool))}})
failed = False
except (psycopg.Error, storage.StorageUnavailable) as exc:
log.warning("pass failed, will retry: %s", type(exc).__name__,
extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__})
failed = True
if args.once:
telemetry.shutdown()
raise SystemExit(1 if failed else 0)
if idle(settings.JANITOR_INTERVAL_SECONDS):
telemetry.shutdown()
return
if __name__ == "__main__":
main()
+666
View File
@@ -0,0 +1,666 @@
import base64
import binascii
import contextlib
import hashlib
import json
import logging
import re
import threading
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime
from pathlib import Path
from typing import Literal
from urllib.parse import quote
from uuid import UUID
import uvicorn
from fastapi import FastAPI, Header, Query, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, StreamingResponse
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.routing import Match
import settings
import telemetry
# lab6: телеметрия настраивается до первого соединения с БД (pool.open в lifespan).
telemetry.setup("messenger-api")
import attachments # noqa: E402
import auth # noqa: E402
import db # noqa: E402
import storage # noqa: E402
import uploads # noqa: E402
from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable # noqa: E402
log = logging.getLogger("messenger")
# HTTP-метрики (contracts/observability.md). Метки: метод, ШАБЛОН маршрута, код и класс статуса —
# никаких реальных UUID, raw URL, request_id. Длительность — до отправки заголовков ответа
# (для скачивания файла тело потом стримится из S3 и в длительность не входит).
HTTP_DURATION = telemetry.meter.create_histogram(
"http.server.request.duration", unit="s", description="Duration of HTTP server requests")
HTTP_ACTIVE = telemetry.meter.create_up_down_counter(
"http.server.active_requests", unit="{request}", description="HTTP requests being processed now")
KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}
DRAINING = threading.Event()
class Server(uvicorn.Server):
"""Остановка по SIGTERM в два шага.
1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS
HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы.
2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы
(не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM,
получал бы 502 от балансировщика.
Повторный сигнал останавливает сразу.
"""
def handle_exit(self, sig, frame):
if DRAINING.is_set():
return super().handle_exit(sig, frame)
DRAINING.set()
log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s",
sig, settings.SHUTDOWN_DRAIN_SECONDS)
threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start()
@asynccontextmanager
async def lifespan(app: FastAPI):
settings.check()
# Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему.
await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS)
# Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3.
Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True)
db.pool.open()
log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, "
"processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV,
settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION,
extra={"event": "service.started"})
yield
db.pool.close()
auth.close()
log.info("Stopped: connections closed, data stays in PostgreSQL and Redis")
telemetry.shutdown()
app = FastAPI(lifespan=lifespan)
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH)
@field_validator("display_name", "password")
@classmethod
def validate_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("value cannot be blank")
return value
class User(BaseModel):
id: UUID
username: str
display_name: str
created_at: datetime
class Session(BaseModel):
token: str
token_type: str
expires_at: datetime
user: User
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def validate_title(cls, value: str) -> str:
if not value.strip():
raise ValueError("title cannot be empty")
return value
class Chat(BaseModel):
id: UUID
title: str
member_ids: list[UUID]
created_at: datetime
class ChatPage(BaseModel):
items: list[Chat]
next_cursor: str | None
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: str = Field(max_length=4000)
# С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением.
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def validate_content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("attachment_ids must be unique")
if not self.attachment_ids and not self.text.strip():
raise ValueError("text cannot be empty without attachments")
return self
class Message(BaseModel):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: list[UUID]
class Variant(BaseModel):
name: Literal["thumbnail", "crop", "optimized"]
content_type: Literal["image/jpeg", "image/png", "image/webp"]
size_bytes: int
width: int
height: int
sha256: str
class Attachment(BaseModel):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: Literal["queued", "processing", "ready", "failed"]
variants: list[Variant]
error_code: str | None
created_at: datetime
updated_at: datetime
class MessagePage(BaseModel):
items: list[Message]
next_cursor: str | None
def error_response(
request: Request,
code: str,
message: str,
status_code: int,
headers: dict | None = None,
):
request_id = request.state.request_id
return JSONResponse(
status_code=status_code,
content={
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
},
headers={"X-Request-Id": request_id, **(headers or {})},
)
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(
request: Request,
exc: RequestValidationError,
):
return error_response(
request,
"invalid_request",
"Invalid request data",
400,
)
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return error_response(request, "not_found", "Not found", 404)
return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers)
@app.exception_handler(Exception)
async def general_exception_handler(
request: Request,
exc: Exception,
):
return error_response(
request,
"internal_error",
"Internal server error",
500,
)
PUBLIC_ENDPOINTS = {
("POST", "/api/v1/users"),
("POST", "/api/v1/auth/sessions"),
}
def route_template(request: Request) -> str:
"""Шаблон маршрута (/api/v1/chats/{chat_id}/messages) — метка метрик и имя спана вместо raw URL."""
for route in app.router.routes:
match, _ = route.matches(request.scope)
if match != Match.NONE:
return getattr(route, "path", "unmatched")
return "unmatched"
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid.uuid4())
request.state.request_id = request_id
started = time.perf_counter()
path = request.url.path
method = request.method if request.method in KNOWN_METHODS else "_OTHER"
route = route_template(request)
# Проверки здоровья (HAProxy раз в секунду, Docker) — только в метриках, без трасс: иначе это
# большая часть всех трасс. Входящий traceparent клиента продолжается (W3C Trace Context).
traced = not path.startswith("/health/")
span_scope = (telemetry.tracer.start_as_current_span(
f"{method} {route}", context=telemetry.extract(request.headers), kind=SpanKind.SERVER,
attributes={"http.request.method": method, "http.route": route, "url.scheme": "http",
"request.id": request_id},
record_exception=False, set_status_on_exception=False)
if traced else contextlib.nullcontext(trace.INVALID_SPAN))
HTTP_ACTIVE.add(1, {"http.request.method": method})
status = 500
try:
with span_scope as span:
try:
if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS:
request.state.session = await run_in_threadpool(
auth.check_bearer, request.headers.get("Authorization")
)
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
except Exception as exc:
log.exception("Unhandled error", extra={"event": "http.unhandled_error", "request_id": request_id})
telemetry.mark_error(span, type(exc).__name__)
response = error_response(request, "internal_error", "Internal server error", 500)
status = response.status_code
span.set_attribute("http.response.status_code", status)
if status >= 500:
telemetry.mark_error(span, str(status))
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила
trace_id = telemetry.trace_id_hex() if traced else None
if trace_id:
response.headers["X-Trace-Id"] = trace_id # расширение: найти трассу по ответу
elapsed = time.perf_counter() - started
# Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки.
level = logging.DEBUG if not traced and status == 200 else logging.INFO
log.log(level, "%s %s -> %s (%.1f ms)", method, route, status, elapsed * 1000,
extra={"event": "http.request", "http.request.method": method, "http.route": route,
"http.response.status_code": status, "duration_ms": round(elapsed * 1000, 1),
"request_id": request_id})
finally:
HTTP_ACTIVE.add(-1, {"http.request.method": method})
elapsed = time.perf_counter() - started
HTTP_DURATION.record(elapsed, {"http.request.method": method, "http.route": route,
"http.response.status_code": status,
"http.response.status_class": f"{status // 100}xx"})
return response
def current_user_id(request: Request) -> UUID:
session = getattr(request.state, "session", None)
if session is None:
raise unauthorized("Bearer token is required")
return session.user_id
def encode_cursor(
created_at: datetime,
object_id: UUID,
*,
chat_id: UUID | None = None,
user_id: UUID | None = None,
) -> str:
data = {
"created_at": created_at.isoformat(),
"id": str(object_id),
}
if chat_id is not None:
data["chat_id"] = str(chat_id)
if user_id is not None:
data["user_id"] = str(user_id)
raw = json.dumps(data).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii")
def decode_cursor(cursor: str):
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
data = json.loads(raw.decode("utf-8"))
created_at = datetime.fromisoformat(data["created_at"])
if created_at.tzinfo is None:
raise ValueError("Invalid cursor")
if not isinstance(data["id"], str):
raise ValueError("Invalid cursor")
object_id = UUID(data["id"])
chat_id = UUID(data["chat_id"]) if "chat_id" in data else None
user_id = UUID(data["user_id"]) if "user_id" in data else None
return created_at, object_id, chat_id, user_id
except (
ValueError,
KeyError,
TypeError,
AttributeError,
json.JSONDecodeError,
binascii.Error,
):
raise ValueError("Invalid cursor")
def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None):
if cursor is None:
return None
try:
created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor)
except ValueError:
raise bad_request("Invalid cursor") from None
if cursor_chat_id != chat_id or cursor_user_id != user_id:
raise bad_request("Invalid cursor")
return created_at, object_id
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
if DRAINING.is_set():
raise unavailable("Instance is shutting down")
db.ping()
auth.ping_redis()
return {"status": "ok"}
@app.post("/api/v1/users", response_model=User, status_code=201)
def create_user(user_data: CreateUser):
password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется
return db.create_user(user_data.username, user_data.display_name, password_hash)
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
def create_session(
response: Response,
authorization: str | None = Header(default=None),
):
credentials = auth.parse_basic(authorization)
if credentials is None:
raise auth.login_failed()
username, password = credentials
found = None
if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH:
found = db.get_user_with_password(username)
if not auth.password_matches(found["password_hash"] if found else None, password):
raise auth.login_failed()
token, expires_at = auth.create_session(found["id"])
response.headers["Cache-Control"] = "no-store"
user = {key: found[key] for key in ("id", "username", "display_name", "created_at")}
return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def delete_session(request: Request):
current_user_id(request)
auth.delete_session(request.state.session)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=User)
def get_current_user(request: Request):
user = db.get_user(current_user_id(request))
if user is None:
raise unauthorized("Session is invalid or expired")
return user
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
def create_chat(request: Request, chat_data: CreateChat):
user_id = current_user_id(request)
if user_id not in chat_data.member_ids:
raise bad_request("Creator must be a chat member")
if len(chat_data.member_ids) != len(set(chat_data.member_ids)):
raise bad_request("Duplicate member IDs")
return db.create_chat(chat_data.title, chat_data.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def list_chats(
request: Request,
limit: int = Query(default=50, ge=1, le=100),
cursor: str | None = Query(default=None),
):
user_id = current_user_id(request)
after = page_after(cursor, user_id=user_id)
rows = db.list_chats(user_id, after, limit)
items = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = items[-1]
next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id)
return ChatPage(items=items, next_cursor=next_cursor)
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
def get_chat(request: Request, chat_id: UUID):
return db.get_chat(chat_id, current_user_id(request))
def request_fingerprint(data: BaseModel) -> bytes:
"""SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.
Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []}
равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними.
"""
body = data.model_dump(mode="json")
if not body.get("attachment_ids"):
body.pop("attachment_ids", None)
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
@app.post(
"/api/v1/chats/{chat_id}/messages",
response_model=Message,
status_code=201,
)
def create_message(
request: Request,
message_data: CreateMessage,
chat_id: UUID,
idempotency_key: str | None = Header(default=None),
):
# С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику)
# вернёт то же сообщение, а не создаст второе. См. db.add_message.
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -")
return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids,
idempotency_key, request_fingerprint(message_data))
@app.get(
"/api/v1/chats/{chat_id}/messages",
response_model=MessagePage,
)
def list_messages(
request: Request,
chat_id: UUID,
limit: int = Query(default=50, ge=1, le=100),
cursor: str | None = Query(default=None),
):
user_id = current_user_id(request)
after = page_after(cursor, chat_id=chat_id)
rows = db.list_messages(chat_id, user_id, after, limit)
items = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = items[-1]
next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id)
return MessagePage(items=items, next_cursor=next_cursor)
@app.post(
"/api/v1/chats/{chat_id}/attachments",
response_model=Attachment,
status_code=202,
)
async def upload_attachment(
request: Request,
chat_id: UUID,
filename: str | None = Query(default=None),
):
"""Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение
для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker.
Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415)
-> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413),
проверка содержимого (415/413) и запись.
"""
user_id = current_user_id(request)
try:
await run_in_threadpool(db.check_member, chat_id, user_id)
name = uploads.validate_filename(filename)
declared = uploads.declared_type(request.headers.get("content-type"))
uploads.declared_length(request.headers.get("content-length"))
except ApiError:
await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке
raise
upload = await uploads.receive(request)
try:
image = await run_in_threadpool(inspect_traced, upload, declared)
row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image)
finally:
upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3)
return attachments.public(row)
def inspect_traced(upload, declared):
"""Проверка содержимого — CPU в обработчике (не БД): в трассе видно отдельным спаном."""
with telemetry.internal_span("inspect image", {"http.request.body.size": upload.size}):
return uploads.inspect(upload, declared)
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
def get_attachment(request: Request, attachment_id: UUID):
return attachments.public(db.get_attachment(attachment_id, current_user_id(request)))
VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"}
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(
request: Request,
attachment_id: UUID,
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"),
):
"""Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3.
queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404.
"""
row = db.get_attachment(attachment_id, current_user_id(request))
if row["state"] != "ready":
raise conflict("Attachment is not ready")
if variant == "original":
key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"]
name = row["filename"]
else:
found = [v for v in row["variants"] if v["name"] == variant]
if not found:
raise not_found("Variant not found")
v = found[0]
key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"]
stem = row["filename"].rsplit(".", 1)[0] or "image"
name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}"
try:
stream = storage.open_read(key)
except storage.ObjectMissing:
log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"],
extra={"event": "attachment.object_missing", "attachment.id": str(row["id"])})
raise unavailable("File is temporarily unavailable") from None
except storage.StorageUnavailable:
raise unavailable("File storage is temporarily unavailable") from None
headers = {
"Content-Length": str(size),
"Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
"ETag": '"' + sha256 + '"',
}
return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers)
if __name__ == "__main__":
Server(uvicorn.Config(
app,
host=settings.APP_HOST,
port=settings.APP_PORT,
timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS,
access_log=False,
log_config=None, # логи uvicorn идут в общий JSON/OTLP-обработчик (telemetry.py)
)).run()
+8
View File
@@ -0,0 +1,8 @@
import logging
import db
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
if __name__ == "__main__":
db.wait_and_migrate()
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT now(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
);
CREATE INDEX messages_history ON messages (chat_id, created_at, id);

Some files were not shown because too many files have changed in this diff Show More