56 changed files with 4419 additions and 53 deletions
+18 -11
View File
@@ -1,30 +1,37 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Local synthetic credentials for compose. Copy to .env; never commit .env.
# The application reads DATABASE_URL, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY and BROKER_URL.
# *_FILE variables from the course worksheet are accepted too when the direct variable is empty.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace-with-local-secret
POSTGRES_DB=messenger
S3_ACCESS_KEY=replace-with-local-secret
S3_SECRET_KEY=replace-with-local-secret
RABBITMQ_USER=messenger
RABBITMQ_PASSWORD=replace-with-local-secret
GRAFANA_ADMIN_PASSWORD=replace-with-local-secret
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
DATABASE_URL=postgresql://messenger:replace-with-local-secret@postgres:5432/messenger
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_ACCESS_KEY_ID=replace-with-local-secret
S3_SECRET_ACCESS_KEY=replace-with-local-secret
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
BROKER_URL=amqp://messenger:replace-with-local-secret@rabbitmq:5672/%2F
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 6
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Казанцева Виктория / БПИ2301
Целевая оценка (3/4/5): 5
Трек 5, если заявлен: 2, SLO и обнаружение инцидентов
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: мессенджер лабораторий 1–5 (сессии, keyset, идемпотентность, вложения, две API-реплики, PostgreSQL, SeaweedFS, RabbitMQ, worker) и наблюдаемость: OTLP в Collector, Prometheus, Loki, Tempo, Grafana provisioning, burn-rate алерты.
Команды запуска из чистого clone: см. REPORT.md. Кратко: `python scripts/render-s3-config.py`, затем `docker compose --env-file .env up -d --build`.
Результаты проверок и ссылка на REPORT.md: smoke и check_contract PASS; persistence и failover PASS; доказательства в REPORT.md и `evidence/`.
Ограничения / известные ошибки: на Windows `.sh` хуки не исполняются напрямую; прогон шёл через Git Bash. Короткий и длинный эксперименты SLO шли подряд, поэтому детекция длинной деградации быстрее холодного старта.
Источники: документация OpenTelemetry, Prometheus, Grafana.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] Проверки `scripts/check_contract.py` и `tests/smoke.py` (то же, что make check и make test) прошли; на Windows они запускались через `PYTHONUTF8=1 python`.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+3
View File
@@ -28,5 +28,8 @@ coverage/
.DS_Store
.idea/
.vscode/
faults/*
!faults/README.md
infra/seaweed/s3.json
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
+21
View File
@@ -0,0 +1,21 @@
FROM python:3.12.8-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_NO_CACHE_DIR=1 \
AWS_EC2_METADATA_DISABLED=true
WORKDIR /app
RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY app /app/app
USER app
EXPOSE 8080 8081
CMD ["python", "-m", "app.main"]
+104 -20
View File
@@ -1,37 +1,121 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Казанцева Виктория / БПИ2301
- Целевая оценка: 5, трек 2 (SLO и обнаружение инцидентов)
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/15
- Источники: OpenTelemetry Collector, Prometheus histograms, W3C Trace Context, Grafana provisioning.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Compose v2 и Python 3.9+ на хосте (проверки). Образы закреплены в `compose.yaml`, приложение собирается из этого репозитория.
## Архитектура и решения
```
python scripts/render-s3-config.py
docker compose --env-file .env config --quiet
docker compose --env-file .env build api-1
docker compose --env-file .env up -d
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
`scripts/render-s3-config.py` читает `S3_ACCESS_KEY` и `S3_SECRET_KEY` из локального `.env` и пишет gitignored `infra/seaweed/s3.json`. Шаблон переменных — `.env.example`. Файл `.env` не коммитится.
Схема БД применяется дважды и одинаково: `app/schema.sql` монтируется в `docker-entrypoint-initdb.d` и повторно накатывается приложением под `pg_advisory_lock`. Бакет S3 создаёт API/worker при старте, если его ещё нет. Очередь `attachment.process.v1` объявляет приложение.
Готовность: `GET http://127.0.0.1:8080/health/ready` возвращает 200. На Windows публичные скрипты запускать с `PYTHONUTF8=1`, потому что `Path.read_text()` в проверках курса читает JSON в кодировке локали.
```
python scripts/check_contract.py
python tests/smoke.py --base-url http://127.0.0.1:8080
```
Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных томов только явно: `docker compose --env-file .env down -v`.
`scripts/restart.sh` перезапускает postgres, SeaweedFS, RabbitMQ, обе API-реплики и worker, ждёт их healthcheck и затем перечитывает upstream в nginx. Тома не удаляются. `scripts/stop-one.sh` останавливает только `api-1` или `api-2` из `TARGET_INSTANCE_ID`. На Windows эти `.sh` напрямую не исполняются (`WinError 193`); проверка ниже запускала их через Git Bash, не меняя `tests/resilience.py`.
## Архитектура
Бизнес-API лабораторий 1–5 не менялся. Две реплики `api-1` и `api-2` за nginx (`127.0.0.1:8080`, round-robin). PostgreSQL — источник истины для пользователей, сессий, чатов, сообщений, идемпотентности и outbox. SeaweedFS хранит байты вложений (`staging/`, `originals/`, `variants/`). RabbitMQ — durable очередь `attachment.process.v1`, persistent messages, confirm и ручной ack. Worker отдельно от API.
Запись вложения: проверка доступа, байты в staging, строка attachment и outbox в одной транзакции, затем publish. Сбой брокера после commit не отменяет `202 queued`: poller допубликует outbox. Worker идемпотентен, lease 60 с, повтор того же `traceparent`.
Наблюдаемость: API и worker шлют OTLP/HTTP protobuf на Collector (`:4318`). Collector: memory limiter 192 MiB, удаление URL/SQL/Authorization, batch, очередь exporter 200 и retry не дольше 30 с. Дальше Prometheus (scrape `:8889`), Loki (OTLP), Tempo (gRPC). Grafana только запрашивает эти три хранилища и не хранит сигналы сама. Дашборд `messenger-overview` и datasources поднимаются provisioning-ом из Git. Правила — `infra/prometheus/rules.yml`.
| Роль | Сервис | Образ |
| --- | --- | --- |
| API, worker | api-1, api-2, worker | локальная сборка `mtusi-lab6-messenger:1.6.0` на `python:3.12.8-slim-bookworm` |
| Балансировщик | nginx | `nginx:1.27.4-alpine` |
| БД | postgres | `postgres:16.6-alpine` |
| S3 | s3 | `chrislusf/seaweedfs:4.51` |
| Брокер | rabbitmq | `rabbitmq:3.13.7-management-alpine` |
| Collector | otel-collector | `otel/opentelemetry-collector-contrib:0.120.0` |
| Метрики | prometheus | `prom/prometheus:v2.55.1` |
| Логи | loki | `grafana/loki:3.3.2` |
| Трассы | tempo | `grafana/tempo:2.7.2` |
| UI | grafana | `grafana/grafana:11.5.2` |
Prometheus хранит ряды и состояние алертов. Loki хранит структурные JSON-логи. Tempo хранит спаны. Grafana их показывает.
Метрики не используют `user_id`, `chat_id`, `message_id`, `attachment_id` как labels. Маршрут в метрике — шаблон. В логах есть `trace_id` и `span_id`. Пароли, токены, текст сообщений и байты картинок в telemetry не пишутся.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда | Наблюдение | Файл |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт skeleton | `PYTHONUTF8=1 python scripts/check_contract.py` | PASS | вывод в этой сессии |
| Публичный smoke | `PYTHONUTF8=1 python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS после включённой telemetry | — |
| Готовность не зависит от Grafana | `GET /health/ready` при остановленном Tempo | 200, бизнес-GET тоже 200 | `evidence/incidents.json` |
| Метрики API и worker | PromQL после smoke | есть `http_server_requests_total`, `http_server_duration_seconds_bucket`, `messenger_jobs_total`, `messenger_job_duration_seconds_bucket`, глубина/возраст очереди, пул БД, busy/capacity/consumers | `scripts/probe_signals.py` |
| Лог и трасса одного upload | Loki `job.ready` и Tempo по тому же `trace_id` | лог `attempt: 2` и трасса ниже | `evidence/retry-trace.json` |
| Дашборд из Git | `GET /api/dashboards/uid/messenger-overview` | uid `messenger-overview`, Grafana 11.5.2 | provisioning в `infra/grafana` |
| p50/p95/p99 из histogram | панели дашборда, `histogram_quantile` по сумме bucket | не среднее и не среднее p99 реплик | `infra/grafana/dashboards/messenger.json` |
| W3C API → broker → worker, retry, S3/DB | один upload с `process_fail_remaining=1` | см. цепочку | `evidence/retry-trace.json` |
| SLI/SLO и burn rate | правила Prometheus | семь alert rules загружены | `infra/prometheus/rules.yml` |
| Короткий всплеск не открывает fast burn | `scripts/slo_experiment.py` | pending, не firing | `evidence/slo-track2.json` |
| Длинная деградация | тот же прогон | fast burn firing через 51.6 с, resolved через 70.6 с | `evidence/slo-track2.json` |
| Остановка worker | `scripts/incident_followup.py` | consumers firing 75.5 с, queue lag 105.7 с, оба resolved за 20.1 с | `evidence/incidents.json` |
| Остановка Tempo | Tempo stop 40 с | 76× HTTP 200, failed spans 9→24, очередь 22, RAM collector 68.8 MiB / 320 MiB | `evidence/incidents.json` |
| Сохранность | публичный persistence через `scripts/restart.sh` | PASS, сессия и вложение на месте | — |
| Отказ одной реплики | публичный failover через `scripts/stop-one.sh` | 2 instance id до отказа, 77 успешных записей, 0 ошибок, восстановление 2.094 с | — |
| Нагрузка | `python tests/load.py --duration 15 --concurrency 2` | 1247 успехов, 0 ошибок, 82.7 rps, p50 31 мс, p95/p99 47 мс | `evidence/load.json` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Цепочка одного upload, trace `cf5025ad9d9a2a248da8335a0c6caee2`:
## Эксперименты и отказы
1. `POST /api/v1/chats/{chat_id}/attachments` — корень.
2. `attachment.process publish`, parent = POST.
3. Два `attachment.process consume` с одним и тем же parent = publish. Первый — внедрённый transient (`job.retry`, attempt 1). Второй — sibling, attempt 2, `job.ready`.
4. У второго consume дети `s3.get`, два `s3.put` и `db.transaction`. У POST свои `db.transaction` и `s3.put` staging.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Перцентили на дашборде считаются как `histogram_quantile` от `sum by (le)` rate bucket. p99 реплик не усредняется: сначала складываются bucket, потом берётся квантиль.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## SLI / SLO
## Ограничения и следующие шаги
SLI API: доля ответов бизнес-маршрутов, которые не 5xx. Знаменатель — все бизнес-запросы. `/health/*` и несовпавший маршрут исключены. 4xx не ошибка SLI. SLO: 99% за скользящие окна burn rate (бюджет 1%).
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Быстрое сжигание: и 1 мин, и 5 мин строго выше 14.4% дольше 30 с, при трафике больше 0.2 rps.
- Медленное: и 5 мин, и 15 мин выше 6% дольше 1 мин.
- Дополнительно: error rate 5 мин выше 5% дольше 1 мин; возраст старейшей job больше 30 с дольше 1 мин; число consumers по данным API равно 0 дольше 1 мин; p95 времени до ready выше 10 с.
Доставка наружу не настраивалась. Состояние — локальные ряды `ALERTS`.
## Трек 2
Гипотеза: при уже идущем успешном трафике всплеск около 10 с поднимает минутное окно и не поднимает пятиминутное выше 14.4%, поэтому fast burn не становится firing. Непрерывные 5xx дольше примерно 45 с поднимают оба коротких окна, и fast burn зажигается. Остановленный worker виден по возрасту очереди и нулю consumers, а не по error rate API. Остановленный Tempo не мешает принять сообщения, exporter упирается в очередь и лимит памяти.
Методика на этом хосте (16 ГБ RAM, лимиты контейнеров в `compose.yaml`): закрытый цикл `GET /api/v1/chats` около 2 rps. Прогрев 90 с дал 180 ответов 200 и ноль alerts. Затем 10 с с `http_error_ratio=1`: 20 ответов 500. Сразу после этого `ApiHighErrorRate` и `ApiErrorBudgetSlowBurn` были только pending, `ApiErrorBudgetFastBurn` не появился. Дальше те же 100% ошибок: fast burn, high error rate и slow burn стали firing через 51.6 с (104 ответа 500). После снятия fault fast burn ушёл через 70.6 с; high и slow оставались firing и к началу следующего прогона (около двух минут спустя) уже были inactive.
Ограничения измерения. Короткий всплеск и длинный прогон шли подряд, поэтому ошибки всплеска ещё лежали в пятиминутном окне и ускорили детект длинной деградации относительно «холодного» старта. p99 нагрузки 15 с на 2 потоках совпал с p95 (47 мс): выборка короткая, это закрытый цикл, не открытый поток. Время resolved для slow burn снято по факту «уже нет» к 21:49:48 UTC, без секундомера. `AttachmentReadyLatencyHigh` стал firing из-за job, которая ждала остановленный worker: это следствие того же инцидента, гистограмма 15 мин ещё держит медленное завершение.
## Два инцидента
Остановка worker. Сразу после stop API показывал `messenger_worker_consumers = 0` и растущий возраст очереди. `WorkerConsumersZero` firing на 75.5 с, `QueueLagHigh` на 105.7 с при возрасте около 98 с. После start оба alert исчезли за 20.1 с. Runbook: `docs/runbooks/queue-lag.md`.
Остановка Tempo на 40 с. Бизнес-GET: 76 ответов 200, ошибок нет. `otelcol_exporter_send_failed_spans` вырос с 9 до 24. Размер очереди exporter дошёл до 22 при лимите 200. Память collector 68.8 MiB при лимите контейнера 320 MiB и memory limiter 192 MiB. Сообщения не терялись. Runbook: `docs/runbooks/telemetry-backend.md`.
Как отличить медленную БД от медленного handler: `db_slow_ms` спит внутри спана `db.transaction`, `handler_slow_ms` спит в серверном спане и вне `db.transaction`.
## Ограничения
Публичные `tests/resilience.py` на этом Windows запускают `.sh` как Win32 и получают `WinError 193`. Сами хуки прогнаны тем же тестом через Git Bash. На Linux `make test-persistence` и `make test-failover` вызывают скрипты напрямую.
Чтобы вообще скачать образы, с диска C: были удалены неиспользуемые образы других локальных проектов. Их тома не удалялись. Образы этих проектов при следующем запуске нужно скачать снова.
SeaweedFS выбран вместо MinIO: образы `minio/minio` с Docker Hub в этой среде не скачиваются. Контракт разрешает SeaweedFS.
+1
View File
@@ -0,0 +1 @@
"""Messenger application for lab 6."""
+124
View File
@@ -0,0 +1,124 @@
import asyncio
import random
import time
import uuid
from opentelemetry.trace import SpanKind
from app import faults, http, telemetry
from app.errors import AppError
from app.settings import settings
class _TooLarge(Exception):
pass
async def _read_body(receive, limit):
chunks = []
size = 0
while True:
message = await receive()
if message["type"] == "http.disconnect":
return None
if message["type"] != "http.request":
continue
data = message.get("body", b"") or b""
size += len(data)
if size > limit:
raise _TooLarge()
chunks.append(data)
if not message.get("more_body"):
return b"".join(chunks)
def _too_large(scope, limit):
for name, value in scope.get("headers", []):
if name == b"content-length":
try:
return int(value) > limit
except ValueError:
return True
return False
async def app(scope, receive, send):
if scope["type"] == "lifespan":
while True:
message = await receive()
if message["type"] == "lifespan.startup":
telemetry.log_event("service.listening", port=8080)
await send({"type": "lifespan.startup.complete"})
elif message["type"] == "lifespan.shutdown":
await send({"type": "lifespan.shutdown.complete"})
return
return
if scope["type"] != "http":
return
method = scope.get("method", "GET")
path = scope.get("path", "/")
raw_headers = {}
for name, value in scope.get("headers", []):
raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1")
request_id = http.request_id_from(raw_headers)
template, _name, _groups = http.match(method, path)
limit = http.body_limit(path)
started = time.perf_counter()
status = 500
response_headers = {}
body = b""
with telemetry.tracer().start_as_current_span(
f"{method} {template}",
kind=SpanKind.SERVER,
attributes={"http.request.method": method, "http.route": template},
) as span:
try:
if _too_large(scope, limit):
raise AppError(413, "payload_too_large", "Слишком большой объект")
try:
payload = await _read_body(receive, limit)
except _TooLarge as exc:
raise AppError(413, "payload_too_large", "Слишком большой объект") from exc
if payload is None:
return
if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload:
raise AppError(413, "payload_too_large", "Слишком большой объект")
query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace"))
if path.startswith("/api/") and random.random() < faults.http_error_ratio():
raise AppError(500, "internal_error", "Внутренняя ошибка")
def run():
return http.handle(method, path, query, raw_headers, payload)
template, status, response_headers, body = await asyncio.to_thread(run)
span.set_attribute("http.route", template)
except Exception as exc:
status, response_headers, body = http.error_response(exc, request_id)
if not isinstance(exc, AppError):
telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id)
if status >= 500:
telemetry.mark_error(span)
span.set_attribute("error.type", type(exc).__name__)
duration = time.perf_counter() - started
span.set_attribute("http.response.status_code", status)
telemetry.record_http(method, template, status, duration)
telemetry.log_event(
"http.request.completed",
request_id=request_id,
http_route=template,
http_method=method,
status=status,
duration_ms=int(duration * 1000),
)
header_list = [
(b"x-request-id", request_id.encode("ascii")),
(b"x-instance-id", settings.instance_id.encode("ascii")),
]
for key, value in response_headers.items():
header_list.append((key.lower().encode("latin-1"), value.encode("latin-1")))
await send({"type": "http.response.start", "status": status, "headers": header_list})
await send({"type": "http.response.body", "body": body})
def new_request_id():
return uuid.uuid4().hex
+153
View File
@@ -0,0 +1,153 @@
import base64
import json
import uuid
from urllib.parse import quote, urlparse, unquote
from urllib.request import Request, urlopen
import pika
from app import telemetry
from app.db import jsonb
from app.settings import settings
QUEUE = "attachment.process.v1"
def _params():
parsed = urlparse(settings.broker_url)
return pika.ConnectionParameters(
host=parsed.hostname or "rabbitmq",
port=parsed.port or 5672,
virtual_host=unquote(parsed.path[1:] or "/"),
credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")),
heartbeat=60,
blocked_connection_timeout=2,
socket_timeout=2,
connection_attempts=1,
retry_delay=0,
)
def _credentials():
parsed = urlparse(settings.broker_url)
return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/")
def init():
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
finally:
connection.close()
def ready():
try:
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True, passive=True)
finally:
connection.close()
return True
except Exception:
return False
def insert_outbox(conn, payload, created_at):
conn.execute(
"""
INSERT INTO outbox (job_id, attachment_id, payload, created_at)
VALUES (%s, %s, %s, %s)
""",
(uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at),
)
def publish_pending(conn, job_id=None):
query = """
SELECT job_id, payload
FROM outbox
WHERE published_at IS NULL
"""
args = []
if job_id is not None:
query += " AND job_id = %s"
args.append(job_id)
query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20"
rows = conn.execute(query, args).fetchall()
for row in rows:
payload = row["payload"]
if isinstance(payload, str):
payload = json.loads(payload)
_publish(payload)
conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],))
return len(rows)
def _publish(payload):
carrier = {}
if payload.get("traceparent"):
carrier["traceparent"] = payload["traceparent"]
if payload.get("tracestate"):
carrier["tracestate"] = payload["tracestate"]
parent = telemetry.extract_context(carrier)
with telemetry.tracer().start_as_current_span(
"attachment.process publish",
context=parent,
kind=telemetry.trace.SpanKind.PRODUCER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "publish",
},
):
outgoing = telemetry.inject_carrier()
body = dict(payload)
if outgoing.get("traceparent"):
body["traceparent"] = outgoing["traceparent"]
if outgoing.get("tracestate"):
body["tracestate"] = outgoing["tracestate"]
_send(body, outgoing)
telemetry.log_event(
"job.published",
job_id=str(payload.get("job_id")),
attempt=int(payload.get("attempt") or 1),
)
def _send(payload, headers):
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.confirm_delivery()
channel.basic_publish(
exchange="",
routing_key=QUEUE,
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
headers=headers,
),
mandatory=True,
)
finally:
connection.close()
def consumer_count():
user, password, vhost = _credentials()
url = (
settings.rabbit_management_url.rstrip("/")
+ "/api/queues/"
+ quote(vhost, safe="")
+ "/"
+ quote(QUEUE, safe="")
)
token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii")
request = Request(url, headers={"Authorization": "Basic " + token})
with urlopen(request, timeout=2) as response:
data = json.loads(response.read().decode("utf-8"))
return int(data.get("consumers") or 0)
+139
View File
@@ -0,0 +1,139 @@
import base64
import datetime as dt
import hashlib
import json
import re
import uuid
from app.errors import AppError, bad_request
USERNAME = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
def utcnow():
return dt.datetime.now(dt.timezone.utc)
def iso(value):
if value.tzinfo is None:
value = value.replace(tzinfo=dt.timezone.utc)
value = value.astimezone(dt.timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z"
def parse_iso(value):
if not isinstance(value, str) or not value.endswith("Z"):
raise bad_request()
try:
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
except ValueError as exc:
raise bad_request() from exc
def parse_uuid(value):
if not isinstance(value, str):
raise bad_request()
try:
return str(uuid.UUID(value))
except ValueError as exc:
raise bad_request() from exc
def require_text(value, minimum, maximum, allow_blank=False):
if not isinstance(value, str):
raise bad_request()
if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value):
raise bad_request()
if not allow_blank and not value.strip():
raise bad_request()
if not (minimum <= len(value) <= maximum):
raise bad_request()
return value
def canonical_message(text, attachment_ids):
body = {"attachment_ids": list(attachment_ids), "text": text}
raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
return hashlib.sha256(raw.encode("utf-8")).digest()
def encode_cursor(kind, resource_id, created_at, row_id):
raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}"
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=")
def decode_cursor(token, kind, resource_id):
if not isinstance(token, str) or not token:
raise bad_request()
padded = token + ("=" * (-len(token) % 4))
try:
raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")
cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3)
created_at = parse_iso(stamp)
row_uuid = str(uuid.UUID(row_id))
except (ValueError, UnicodeError, AppError) as exc:
raise bad_request() from exc
if cursor_kind != kind or cursor_resource != resource_id:
raise bad_request()
return created_at, row_uuid
def fitted_size(width, height, box=256):
if width < 1 or height < 1:
raise ValueError("dimensions")
if width <= box and height <= box:
return width, height
if width >= height:
return box, max(1, (height * box) // width)
return max(1, (width * box) // height), box
def detect_image(data, declared):
if data.startswith(b"\x89PNG\r\n\x1a\n"):
detected = "image/png"
elif data.startswith(b"\xff\xd8\xff"):
detected = "image/jpeg"
else:
return None
if declared != detected:
return None
return detected
def media_type(header):
if not header or not isinstance(header, str):
return ""
return header.split(";", 1)[0].strip().lower()
def loads_object(raw):
try:
text = raw.decode("utf-8")
except UnicodeError as exc:
raise bad_request() from exc
def pairs(items):
obj = {}
for key, value in items:
if key in obj:
raise ValueError("duplicate")
obj[key] = value
return obj
try:
value = json.loads(text, object_pairs_hook=pairs)
except (ValueError, UnicodeError) as exc:
raise bad_request() from exc
if not isinstance(value, dict):
raise bad_request()
return value
def expect_fields(data, allowed, required):
if set(data) - set(allowed):
raise bad_request()
for key in required:
if key not in data:
raise bad_request()
+98
View File
@@ -0,0 +1,98 @@
import time
from contextlib import contextmanager
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool, PoolTimeout
from app import telemetry
from app.errors import AppError, unavailable
from app.faults import db_fail, db_slow_seconds
from app.settings import settings
pool = None
def init():
global pool
pool = ConnectionPool(
settings.database_url,
min_size=1,
max_size=settings.pool_max,
timeout=2,
max_waiting=16,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": False,
"connect_timeout": 3,
"options": "-c timezone=UTC -c statement_timeout=4000",
},
open=True,
reconnect_timeout=2,
)
def migrate():
script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8")
statements = []
buffer = []
for line in script.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("--"):
continue
buffer.append(line)
if stripped.endswith(";"):
statements.append("\n".join(buffer))
buffer = []
with pool.connection() as conn:
conn.execute("SELECT pg_advisory_lock(861006)")
for statement in statements:
conn.execute(statement)
conn.execute("SELECT pg_advisory_unlock(861006)")
conn.commit()
def jsonb(value):
return Jsonb(value)
@contextmanager
def connection():
if db_fail():
raise unavailable()
delay = db_slow_seconds()
started = time.perf_counter()
span = telemetry.tracer().start_as_current_span(
"db.transaction",
attributes={"db.system": "postgresql"},
)
span.__enter__()
try:
if delay:
time.sleep(delay)
with pool.connection() as conn:
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
except AppError:
raise
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc:
raise unavailable() from exc
finally:
span.__exit__(None, None, None)
telemetry.record_db(time.perf_counter() - started)
def stats():
if pool is None:
return 0, 0
data = pool.get_stats()
used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0))
waiting = int(data.get("requests_waiting", 0))
return max(used, 0), waiting
+584
View File
@@ -0,0 +1,584 @@
import datetime as dt
import hashlib
import uuid
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from argon2.low_level import Type
from psycopg.errors import UniqueViolation
from app import broker, storage, telemetry
from app.codec import (
USERNAME,
IDEMPOTENCY,
canonical_message,
decode_cursor,
detect_image,
encode_cursor,
expect_fields,
iso,
parse_uuid,
require_text,
utcnow,
)
from app.db import connection, jsonb
from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unsupported
from app.images import validate_image
from app.settings import settings
BASIC = 'Basic realm="messenger", charset="UTF-8"'
BEARER = 'Bearer realm="messenger"'
_hasher = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1, hash_len=32, salt_len=16, type=Type.ID)
_dummy = None
def init_hasher():
global _dummy
_dummy = _hasher.hash("not-a-real-password")
def _user_json(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": iso(row["created_at"]),
}
def _token_hash(token):
return hashlib.sha256(token.encode("utf-8")).digest()
def _password(value):
if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128):
raise bad_request()
return value
def as_uuid(value):
if isinstance(value, uuid.UUID):
return value
return uuid.UUID(str(value))
def _username(value):
if not isinstance(value, str) or USERNAME.fullmatch(value) is None:
raise bad_request()
return value
def register(data):
expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"])
username = _username(data["username"])
display_name = require_text(data["display_name"], 1, 100)
password = _password(data["password"])
password_hash = _hasher.hash(password)
user_id = uuid.uuid4()
created_at = utcnow()
try:
with connection() as conn:
conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(user_id, username, display_name, password_hash, created_at),
)
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
except UniqueViolation as exc:
raise conflict("Имя пользователя занято") from exc
return 201, _user_json(row)
def login(username, password):
if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password:
raise unauthorized(BASIC)
with connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
(username,),
).fetchone()
stored = row["password_hash"] if row else _dummy
try:
_hasher.verify(stored, password)
valid = row is not None
except (VerifyMismatchError, VerificationError, InvalidHashError):
valid = False
if not valid:
raise unauthorized(BASIC)
token = secrets_token()
expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl)
with connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(_token_hash(token), row["id"], expires_at),
)
return 201, {
"token": token,
"token_type": "Bearer",
"expires_at": iso(expires_at),
"user": _user_json(row),
}
def secrets_token():
import secrets
return secrets.token_urlsafe(32)
def current_user(token):
if not token or len(token) < 32:
raise unauthorized(BEARER)
with connection() as conn:
row = conn.execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions s
JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW()
""",
(_token_hash(token),),
).fetchone()
if not row:
raise unauthorized(BEARER)
return row
def ready_check():
with connection() as conn:
conn.execute("SELECT 1")
if not storage.ready() or not broker.ready():
raise unavailable()
def revoke(token):
user = current_user(token)
with connection() as conn:
conn.execute(
"UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL",
(_token_hash(token),),
)
return user
def create_chat(user, data):
expect_fields(data, ["title", "member_ids"], ["title", "member_ids"])
title = require_text(data["title"], 1, 100)
raw_members = data["member_ids"]
if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100):
raise bad_request()
members = [parse_uuid(item) for item in raw_members]
if len(set(members)) != len(members):
raise bad_request()
if str(user["id"]) not in members:
raise bad_request()
chat_id = uuid.uuid4()
created_at = utcnow()
with connection() as conn:
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall()
if len(found) != len(members):
raise bad_request()
conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat_id, title, created_at),
)
for position, member_id in enumerate(members):
conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
(chat_id, as_uuid(member_id), position),
)
return 201, {
"id": str(chat_id),
"title": title,
"member_ids": members,
"created_at": iso(created_at),
}
def _chat_json(conn, row):
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position",
(row["id"],),
).fetchall()
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(item["user_id"]) for item in members],
"created_at": iso(row["created_at"]),
}
def list_chats(user, limit, cursor):
created_at = None
row_id = None
if cursor is not None:
created_at, row_id = decode_cursor(cursor, "chats", str(user["id"]))
with connection() as conn:
if created_at is None:
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = %s
ORDER BY c.created_at, c.id
LIMIT %s
""",
(user["id"], limit + 1),
).fetchall()
else:
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s)
ORDER BY c.created_at, c.id
LIMIT %s
""",
(user["id"], created_at, as_uuid(row_id), limit + 1),
).fetchall()
extra = len(rows) > limit
rows = rows[:limit]
items = [_chat_json(conn, row) for row in rows]
next_cursor = None
if extra and rows:
last = rows[-1]
next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"]))
return {"items": items, "next_cursor": next_cursor}
def get_chat(user, chat_id):
with connection() as conn:
row = _visible_chat(conn, user["id"], chat_id)
return _chat_json(conn, row)
def _visible_chat(conn, user_id, chat_id):
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = %s AND m.user_id = %s
""",
(as_uuid(chat_id), as_uuid(user_id)),
).fetchone()
if not row:
raise not_found()
return row
def _message_json(row, attachment_ids):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": iso(row["created_at"]),
"attachment_ids": attachment_ids,
}
def _attachment_ids(conn, message_id):
rows = conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
).fetchall()
return [str(row["attachment_id"]) for row in rows]
def create_message(user, chat_id, key, data, path):
chat_uuid = as_uuid(chat_id)
if IDEMPOTENCY.fullmatch(key or "") is None:
raise bad_request()
expect_fields(data, ["text", "attachment_ids"], ["text"])
if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000:
raise bad_request()
text = data["text"]
if "attachment_ids" in data:
raw_ids = data["attachment_ids"]
if not isinstance(raw_ids, list) or len(raw_ids) > 10:
raise bad_request()
attachment_ids = [parse_uuid(item) for item in raw_ids]
if len(set(attachment_ids)) != len(attachment_ids):
raise bad_request()
else:
attachment_ids = []
if not text.strip() and not attachment_ids:
raise bad_request()
body_hash = canonical_message(text, attachment_ids)
lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True)
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
existing = conn.execute(
"""
SELECT body_hash, response_json, created_at
FROM idempotency
WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s
""",
(user["id"], path, key),
).fetchone()
if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl):
if bytes(existing["body_hash"]) != body_hash:
raise conflict("Ключ идемпотентности уже использован с другим телом")
stored = existing["response_json"]
if isinstance(stored, str):
import json
stored = json.loads(stored)
return 201, stored
if existing:
conn.execute(
"DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s",
(user["id"], path, key),
)
if attachment_ids:
rows = conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
([as_uuid(item) for item in attachment_ids],),
).fetchall()
found = {str(row["id"]): row for row in rows}
if len(found) != len(attachment_ids):
raise not_found()
for attachment_id in attachment_ids:
row = found[attachment_id]
if str(row["chat_id"]) != chat_id:
raise not_found()
if row["status"] == "failed":
raise conflict("Нельзя прикрепить неуспешное вложение")
message_id = uuid.uuid4()
created_at = utcnow()
conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
(message_id, chat_uuid, user["id"], text, created_at),
)
for position, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, as_uuid(attachment_id), position),
)
stored = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s",
(message_id,),
).fetchone()
payload = _message_json(stored, attachment_ids)
conn.execute(
"""
INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at)
VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s)
""",
(user["id"], path, key, body_hash, jsonb(payload), created_at),
)
return 201, payload
def list_messages(user, chat_id, limit, cursor):
chat_uuid = as_uuid(chat_id)
created_at = None
row_id = None
if cursor is not None:
created_at, row_id = decode_cursor(cursor, "messages", chat_id)
with connection() as conn:
_visible_chat(conn, user["id"], chat_uuid)
if created_at is None:
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s
ORDER BY created_at, id
LIMIT %s
""",
(chat_uuid, limit + 1),
).fetchall()
else:
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s AND (created_at, id) > (%s, %s)
ORDER BY created_at, id
LIMIT %s
""",
(chat_uuid, created_at, as_uuid(row_id), limit + 1),
).fetchall()
extra = len(rows) > limit
rows = rows[:limit]
items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows]
next_cursor = None
if extra and rows:
last = rows[-1]
next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"]))
return {"items": items, "next_cursor": next_cursor}
def _variants(conn, attachment_id):
rows = conn.execute(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = 1
ORDER BY name
""",
(attachment_id,),
).fetchall()
return [
{
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
for row in rows
]
def _attachment_json(row, variants):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": variants,
"error_code": row["error_code"],
"created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def upload(user, chat_id, filename, content_type, data):
import hashlib as hl
if not data:
raise unsupported()
if len(data) > settings.max_upload:
raise too_large()
detected = detect_image(data, content_type)
if not detected:
raise unsupported()
validate_image(data)
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
attachment_id = uuid.uuid4()
created_at = utcnow()
digest = hl.sha256(data).hexdigest()
staging_key = f"staging/{attachment_id}"
storage.put(staging_key, data, detected, "staging")
carrier = telemetry.inject_carrier()
job_id = uuid.uuid4()
payload = {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": iso(created_at),
}
if carrier.get("traceparent"):
payload["traceparent"] = carrier["traceparent"]
if carrier.get("tracestate"):
payload["tracestate"] = carrier["tracestate"]
row = {
"id": attachment_id,
"chat_id": chat_id,
"owner_id": user["id"],
"filename": filename,
"content_type": detected,
"size_bytes": len(data),
"sha256": digest,
"status": "queued",
"error_code": None,
"created_at": created_at,
"updated_at": created_at,
}
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status,
error_code, staging_key, pipeline_version, attempt, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s)
""",
(
attachment_id,
as_uuid(chat_id),
user["id"],
filename,
detected,
len(data),
digest,
staging_key,
created_at,
created_at,
),
)
broker.insert_outbox(conn, payload, created_at)
try:
with connection() as conn:
broker.publish_pending(conn, job_id)
except Exception:
telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id))
telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected)
return 202, _attachment_json(row, [])
def get_attachment(user, attachment_id):
attachment_uuid = as_uuid(attachment_id)
with connection() as conn:
row = _visible_attachment(conn, user["id"], attachment_uuid)
return _attachment_json(row, _variants(conn, attachment_uuid))
def _visible_attachment(conn, user_id, attachment_id):
row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone()
if not row:
raise not_found()
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(row["chat_id"], user_id),
).fetchone()
if not member:
raise not_found()
return row
def download(user, attachment_id, variant):
attachment_uuid = as_uuid(attachment_id)
if variant not in {"original", "thumbnail", "crop", "optimized"}:
raise bad_request()
with connection() as conn:
row = _visible_attachment(conn, user["id"], attachment_uuid)
if row["status"] != "ready":
raise conflict("Вложение ещё не готово")
if variant == "original":
key = row["original_key"]
content_type = row["content_type"]
else:
found = conn.execute(
"""
SELECT object_key, content_type
FROM attachment_variants
WHERE attachment_id = %s AND name = %s AND pipeline_version = 1
""",
(attachment_uuid, variant),
).fetchone()
if not found:
raise not_found()
key = found["object_key"]
content_type = found["content_type"]
return storage.get(key, variant), content_type
+40
View File
@@ -0,0 +1,40 @@
class AppError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message="Неверный запрос"):
return AppError(400, "invalid_request", message)
def unauthorized(challenge):
return AppError(
401,
"unauthorized",
"Требуется аутентификация",
{"WWW-Authenticate": challenge},
)
def not_found():
return AppError(404, "not_found", "Объект не найден")
def conflict(message="Конфликт"):
return AppError(409, "conflict", message)
def too_large(message="Слишком большой объект"):
return AppError(413, "payload_too_large", message)
def unsupported(message="Неподдерживаемый тип изображения"):
return AppError(415, "unsupported_media_type", message)
def unavailable(message="Сервис временно недоступен"):
return AppError(503, "unavailable", message)
+73
View File
@@ -0,0 +1,73 @@
from pathlib import Path
from app.settings import settings
def _text(name):
path = Path(settings.fault_dir) / name
try:
return path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return ""
def db_slow_seconds():
raw = _text("db_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def db_fail():
return _text("db_fail") == "1"
def http_error_ratio():
raw = _text("http_error_ratio")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 1.0)
def handler_slow_seconds():
raw = _text("handler_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def consume_process_failure():
path = Path(settings.fault_dir) / "process_fail_remaining"
try:
raw = path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return False
try:
remaining = int(raw or "0")
except ValueError:
return False
if remaining <= 0:
return False
try:
path.write_text(str(remaining - 1), encoding="utf-8")
except OSError:
return False
return True
+221
View File
@@ -0,0 +1,221 @@
import json
import re
from urllib.parse import parse_qs
from app import domain, faults
from app.codec import REQUEST_ID, loads_object, media_type, require_text
from app.errors import AppError, bad_request, not_found, unauthorized
from app.settings import settings
_ROUTES = [
("GET", re.compile(r"^/health/live$"), "/health/live", "live"),
("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"),
("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"),
("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"),
("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"),
("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"),
("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"),
("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"),
("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"),
("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"),
("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"),
("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"),
("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"),
("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"),
]
def match(method, path):
for route_method, pattern, template, name in _ROUTES:
found = pattern.fullmatch(path)
if found and route_method == method:
return template, name, found.groups()
return "unmatched", "unmatched", ()
def _single(query, name):
values = query.get(name)
if not values:
return None
if len(values) != 1 or values[0] == "":
raise bad_request()
return values[0]
def _limit(query):
raw = _single(query, "limit")
if raw is None:
return 50
if re.fullmatch(r"[1-9][0-9]*", raw) is None:
raise bad_request()
value = int(raw)
if value > 100:
raise bad_request()
return value
def _json_body(headers, body):
if media_type(headers.get("content-type", "")) != "application/json":
raise bad_request()
if not body:
raise bad_request()
return loads_object(body)
def _bearer(headers):
value = headers.get("authorization", "")
if not value.lower().startswith("bearer "):
raise unauthorized(domain.BEARER)
token = value.split(" ", 1)[1].strip()
return token
def _basic(headers):
import base64
value = headers.get("authorization", "")
if not value.lower().startswith("basic "):
raise unauthorized(domain.BASIC)
try:
decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8")
except (ValueError, UnicodeError) as exc:
raise unauthorized(domain.BASIC) from exc
if ":" not in decoded:
raise unauthorized(domain.BASIC)
return decoded.split(":", 1)
def _filename(query):
raw = _single(query, "filename")
if raw is None:
raise bad_request()
if len(raw) < 1 or len(raw) > 255 or not raw.strip():
raise bad_request()
if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw):
raise bad_request()
return raw
def handle(method, path, query, headers, body):
template, name, groups = match(method, path)
if name == "unmatched":
raise not_found()
if name not in {"live", "ready"}:
delay = faults.handler_slow_seconds()
if delay:
import time
time.sleep(delay)
if name == "live":
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
if name == "ready":
domain.ready_check()
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
if name == "register":
status, payload = domain.register(_json_body(headers, body))
return template, status, _json_headers(), _dump(payload)
if name == "login":
if body:
raise bad_request()
username, password = _basic(headers)
status, payload = domain.login(username, password)
return template, status, _json_headers(), _dump(payload)
if name == "me":
user = domain.current_user(_bearer(headers))
return template, 200, _json_headers(), _dump(domain._user_json(user))
if name == "logout":
domain.revoke(_bearer(headers))
return template, 204, {}, b""
if name == "create_chat":
user = domain.current_user(_bearer(headers))
status, payload = domain.create_chat(user, _json_body(headers, body))
return template, status, _json_headers(), _dump(payload)
if name == "list_chats":
user = domain.current_user(_bearer(headers))
payload = domain.list_chats(user, _limit(query), _single(query, "cursor"))
return template, 200, _json_headers(), _dump(payload)
if name == "get_chat":
user = domain.current_user(_bearer(headers))
payload = domain.get_chat(user, _uuid_segment(groups[0]))
return template, 200, _json_headers(), _dump(payload)
if name == "create_message":
user = domain.current_user(_bearer(headers))
chat_id = _uuid_segment(groups[0])
status, payload = domain.create_message(
user,
chat_id,
headers.get("idempotency-key"),
_json_body(headers, body),
path,
)
return template, status, _json_headers(), _dump(payload)
if name == "list_messages":
user = domain.current_user(_bearer(headers))
payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor"))
return template, 200, _json_headers(), _dump(payload)
if name == "upload":
user = domain.current_user(_bearer(headers))
content_type = media_type(headers.get("content-type", ""))
status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body)
return template, status, _json_headers(), _dump(payload)
if name == "get_attachment":
user = domain.current_user(_bearer(headers))
payload = domain.get_attachment(user, _uuid_segment(groups[0]))
return template, 200, _json_headers(), _dump(payload)
if name == "download":
user = domain.current_user(_bearer(headers))
variant = _single(query, "variant") or "original"
data, content_type = domain.download(user, _uuid_segment(groups[0]), variant)
return template, 200, {"Content-Type": content_type}, data
raise not_found()
def _uuid_segment(value):
from app.codec import parse_uuid
return parse_uuid(value)
def _json_headers():
return {"Content-Type": "application/json; charset=utf-8"}
def _dump(payload):
return json.dumps(payload, ensure_ascii=False).encode("utf-8")
def error_response(exc, request_id):
if isinstance(exc, AppError):
status = exc.status
code = exc.code
message = exc.message
headers = dict(exc.headers)
else:
status = 500
code = "internal_error"
message = "Внутренняя ошибка"
headers = {}
headers["Content-Type"] = "application/json; charset=utf-8"
body = _dump({"error": {"code": code, "message": message, "request_id": request_id}})
return status, headers, body
def request_id_from(headers):
incoming = headers.get("x-request-id", "")
if REQUEST_ID.fullmatch(incoming):
return incoming
import uuid
return uuid.uuid4().hex
def query_from(raw):
if not raw:
return {}
return parse_qs(raw, keep_blank_values=True, strict_parsing=False)
def body_limit(path):
if "/attachments" in path and path.rstrip("/").endswith("attachments"):
return settings.max_upload + 1
return 1_000_000
+62
View File
@@ -0,0 +1,62 @@
import io
from PIL import Image, ImageOps, UnidentifiedImageError
from app.codec import fitted_size
from app.errors import AppError, too_large, unsupported
from app.settings import settings
Image.MAX_IMAGE_PIXELS = settings.max_pixels
def _open(data):
try:
with Image.open(io.BytesIO(data)) as image:
image.verify()
with Image.open(io.BytesIO(data)) as image:
width, height = image.size
if width < 1 or height < 1 or width * height > settings.max_pixels:
raise too_large()
image.load()
return image.copy()
except AppError:
raise
except Image.DecompressionBombError as exc:
raise too_large() from exc
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc:
raise unsupported() from exc
def validate_image(data):
_open(data).close()
def _rgb(image):
if image.mode == "RGB":
return image
if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info):
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return image.convert("RGB")
def thumbnail_jpeg(data):
image = _open(data)
try:
image = ImageOps.exif_transpose(image)
image = _rgb(image)
target = fitted_size(image.width, image.height, 256)
if (image.width, image.height) != target:
image = image.resize(target, Image.Resampling.LANCZOS)
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
blob = buffer.getvalue()
return blob, image.width, image.height
except AppError:
raise
except (UnidentifiedImageError, OSError, ValueError) as exc:
raise unsupported() from exc
finally:
image.close()
+50
View File
@@ -0,0 +1,50 @@
import os
import time
import uvicorn
from app import broker, db, domain, storage, telemetry
from app.asgi import app
from app.poller import start
def _retry(label, action):
last = None
for _ in range(40):
try:
action()
return
except Exception as exc:
last = exc
telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__)
time.sleep(1)
raise last
def main():
telemetry.init()
_retry("postgres", db.init)
db.migrate()
_retry("s3", storage.init)
_retry("broker", broker.init)
domain.init_hasher()
start()
uvicorn.run(
app,
host=os.environ.get("APP_HOST", "0.0.0.0"),
port=int(os.environ.get("APP_PORT", "8080")),
access_log=False,
log_config={
"version": 1,
"disable_existing_loggers": False,
"loggers": {
"uvicorn": {"level": "WARNING"},
"uvicorn.error": {"level": "WARNING"},
"uvicorn.access": {"level": "WARNING"},
},
},
)
if __name__ == "__main__":
main()
+44
View File
@@ -0,0 +1,44 @@
import threading
from app import broker, db, telemetry
def _once():
try:
with db.connection() as conn:
row = conn.execute(
"""
SELECT COUNT(*) AS depth,
EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age
FROM attachments
WHERE status IN ('queued', 'processing')
"""
).fetchone()
depth = int(row["depth"] or 0)
age = float(row["age"] or 0) if depth else 0.0
used, waiting = db.stats()
telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting)
except Exception:
telemetry.update_runtime(db_up=0)
try:
telemetry.update_runtime(worker_consumers=broker.consumer_count())
except Exception:
telemetry.log_event("broker.stats_unavailable", severity="WARNING")
telemetry.sample_process()
try:
with db.connection() as conn:
broker.publish_pending(conn)
except Exception:
telemetry.log_event("job.publish_deferred", severity="WARNING")
def start():
stop = threading.Event()
def loop():
while not stop.wait(2):
_once()
_once()
threading.Thread(target=loop, name="poller", daemon=True).start()
return stop
+108
View File
@@ -0,0 +1,108 @@
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
expires_at timestamptz NOT NULL,
revoked_at timestamptz
);
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position integer NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
sender_id uuid NOT NULL REFERENCES users (id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id);
CREATE TABLE IF NOT EXISTS attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
sha256 text NOT NULL,
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
error_code text,
staging_key text NOT NULL,
original_key text,
pipeline_version integer NOT NULL DEFAULT 1,
attempt integer NOT NULL DEFAULT 0,
lease_until timestamptz,
worker_id text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
ready_at timestamptz
);
CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at)
WHERE status IN ('queued', 'processing');
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id),
name text NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL,
pipeline_version integer NOT NULL,
PRIMARY KEY (attachment_id, name, pipeline_version)
);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
attachment_id uuid NOT NULL REFERENCES attachments (id),
position integer NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
CREATE TABLE IF NOT EXISTS idempotency (
user_id uuid NOT NULL,
method text NOT NULL,
path text NOT NULL,
idem_key text NOT NULL,
body_hash bytea NOT NULL,
status_code integer NOT NULL,
response_json jsonb NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, path, idem_key)
);
CREATE TABLE IF NOT EXISTS outbox (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id),
payload jsonb NOT NULL,
created_at timestamptz NOT NULL,
published_at timestamptz
);
CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at)
WHERE published_at IS NULL;
+45
View File
@@ -0,0 +1,45 @@
import os
from pathlib import Path
def _read(name, file_name=None, default=None):
value = os.environ.get(name)
if value:
return value.strip()
file_env = os.environ.get(file_name) if file_name else None
if file_env:
return Path(file_env).read_text(encoding="utf-8").strip()
if default is not None:
return default
raise RuntimeError("missing configuration " + name)
class Settings:
def __init__(self):
self.service_name = os.environ.get("OTEL_SERVICE_NAME", "messenger-api")
self.service_version = os.environ.get("SERVICE_VERSION", "1.6.0")
self.instance_id = os.environ.get("INSTANCE_ID", "api-local")
self.environment = "lab"
self.database_url = _read("DATABASE_URL", "DATABASE_URL_FILE")
self.session_ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
self.idempotency_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000")
self.s3_region = os.environ.get("S3_REGION", "us-east-1")
self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
self.s3_access_key = _read("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
self.s3_secret_key = _read("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
self.broker_url = _read("BROKER_URL", "BROKER_URL_FILE")
self.worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "2"))
self.job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
self.job_timeout = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
self.staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
self.max_upload = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
self.max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
self.pool_max = int(os.environ.get("DB_POOL_MAX", "8"))
self.otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
self.fault_dir = os.environ.get("FAULT_DIR", "/faults")
self.rabbit_management_url = os.environ.get("RABBITMQ_MANAGEMENT_URL", "http://rabbitmq:15672")
self.role = os.environ.get("APP_ROLE", "api")
settings = Settings()
+92
View File
@@ -0,0 +1,92 @@
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from app import telemetry
from app.settings import settings
_client = None
_lock = threading.Lock()
def init():
global _client
_client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
aws_access_key_id=settings.s3_access_key,
aws_secret_access_key=settings.s3_secret_key,
region_name=settings.s3_region,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
),
)
try:
_client.head_bucket(Bucket=settings.s3_bucket)
return
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"404", "NoSuchBucket", "NotFound"}:
raise
try:
_client.create_bucket(Bucket=settings.s3_bucket)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
_warm()
def _warm():
"""Read one existing object so the first user download is not a cold volume open."""
try:
listed = _client.list_objects_v2(Bucket=settings.s3_bucket, MaxKeys=1)
except (BotoCoreError, ClientError, OSError):
return
contents = listed.get("Contents") or []
if not contents:
return
try:
body = _client.get_object(Bucket=settings.s3_bucket, Key=contents[0]["Key"])["Body"]
body.read()
except (BotoCoreError, ClientError, OSError):
return
def ready():
try:
with _lock:
_client.head_bucket(Bucket=settings.s3_bucket)
return True
except (BotoCoreError, ClientError, OSError):
return False
def put(key, data, content_type, variant):
with telemetry.tracer().start_as_current_span(
"s3.put",
attributes={"s3.operation": "put", "variant": variant},
):
with _lock:
_client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type)
def get(key, variant):
with telemetry.tracer().start_as_current_span(
"s3.get",
attributes={"s3.operation": "get", "variant": variant},
):
with _lock:
response = _client.get_object(Bucket=settings.s3_bucket, Key=key)
return response["Body"].read()
def delete(key):
with _lock:
_client.delete_object(Bucket=settings.s3_bucket, Key=key)
+379
View File
@@ -0,0 +1,379 @@
import json
import logging
import threading
import time
from opentelemetry import context as otel_context
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk._logs.export import LogExportResult
from opentelemetry.sdk.metrics.export import MetricExportResult
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.trace import Status, StatusCode
from app.settings import settings
BANNED_FIELDS = {
"password",
"token",
"authorization",
"secret",
"filename",
"text",
"body",
"database_url",
"broker_url",
"image",
}
_dropped = None
_http_requests = None
_http_duration = None
_jobs = None
_job_duration = None
_ready_duration = None
_db_duration = None
_state = {
"queue_depth": 0,
"queue_age": 0.0,
"db_up": 0,
"pool_used": 0,
"pool_waiting": 0,
"worker_busy": 0,
"worker_capacity": 0,
"worker_consumers": 0,
"cpu": 0.0,
"memory": 0,
}
_cpu_last = None
_lock = threading.Lock()
class _GuardedExporter:
def __init__(self, inner, signal):
self.inner = inner
self.signal = signal
def __getattr__(self, name):
return getattr(self.inner, name)
def export(self, items, **kwargs):
try:
result = self.inner.export(items)
except Exception:
_count_drop(self.signal, len(items))
return self._failure()
if getattr(result, "name", None) != "SUCCESS":
_count_drop(self.signal, len(items))
return result
def _failure(self):
if self.signal == "metrics":
return MetricExportResult.FAILURE
if self.signal == "logs":
return LogExportResult.FAILURE
return SpanExportResult.FAILURE
def shutdown(self, **kwargs):
try:
return self.inner.shutdown(**kwargs)
except Exception:
return None
def force_flush(self, timeout_millis=3000):
try:
return self.inner.force_flush(timeout_millis)
except Exception:
return False
class _RawFormatter(logging.Formatter):
def format(self, record):
return record.getMessage()
def _count_drop(signal, count):
if _dropped is not None and count:
_dropped.add(count, {"signal": signal})
def _resource():
return Resource.create(
{
"service.name": settings.service_name,
"service.version": settings.service_version,
"service.instance.id": settings.instance_id,
"deployment.environment.name": settings.environment,
}
)
def _histogram(name, buckets):
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
def init():
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
resource = _resource()
endpoint = settings.otlp_endpoint
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(
_GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
trace.set_tracer_provider(tracer_provider)
reader = PeriodicExportingMetricReader(
_GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"),
export_interval_millis=5000,
export_timeout_millis=4000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
],
)
from opentelemetry import metrics
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter("messenger", settings.service_version)
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
_http_requests = meter.create_counter("http.server.requests", unit="1")
_http_duration = meter.create_histogram("http.server.duration", unit="s")
_jobs = meter.create_counter("messenger.jobs", unit="1")
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
_GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
root = logging.getLogger()
root.handlers.clear()
root.setLevel(logging.INFO)
stream = logging.StreamHandler()
stream.setFormatter(_RawFormatter())
root.addHandler(stream)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
root.addHandler(otel_handler)
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"):
logging.getLogger(name).setLevel(logging.WARNING)
quiet = logging.getLogger("opentelemetry")
quiet.handlers.clear()
quiet.propagate = False
quiet.setLevel(logging.CRITICAL)
log_event("service.started", role=settings.role)
def tracer():
return trace.get_tracer("messenger", settings.service_version)
def current_ids():
span = trace.get_current_span()
ctx = span.get_span_context()
if not ctx or not ctx.is_valid:
return None, None
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
def log_event(event, severity="INFO", **fields):
trace_id, span_id = current_ids()
payload = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
"severity": severity,
"service.name": settings.service_name,
"event": event,
"trace_id": trace_id,
"span_id": span_id,
}
for key, value in fields.items():
if key in BANNED_FIELDS or value is None:
continue
if isinstance(value, (str, int, float, bool)):
payload[key] = value
level = getattr(logging, severity, logging.INFO)
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
def inject_carrier(carrier=None):
target = {} if carrier is None else carrier
inject(target)
return target
def extract_context(carrier):
clean = {}
for key, value in (carrier or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
clean[str(key).lower()] = value
return extract(clean)
def record_http(method, route, status, duration):
if _http_requests is None:
return
klass = f"{int(status) // 100}xx"
attributes = {
"http.request.method": method,
"http.route": route,
"http.response.status_class": klass,
}
_http_requests.add(1, attributes)
_http_duration.record(duration, attributes)
def record_job(outcome, duration):
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
if _jobs is not None:
_jobs.add(1, attributes)
_job_duration.record(max(duration, 0.0), attributes)
def record_ready(duration):
if _ready_duration is not None:
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
def record_db(duration):
if _db_duration is not None:
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
def update_runtime(**values):
with _lock:
_state.update(values)
def note_worker(busy, capacity):
update_runtime(worker_busy=busy, worker_capacity=capacity)
def sample_process():
global _cpu_last
now_cpu = time.process_time()
now_wall = time.monotonic()
ratio = 0.0
if _cpu_last is not None:
dc = now_cpu - _cpu_last[0]
dw = now_wall - _cpu_last[1]
if dw > 0:
ratio = dc / dw
_cpu_last = (now_cpu, now_wall)
memory = 0
try:
with open("/proc/self/statm", encoding="ascii") as handle:
resident = int(handle.read().split()[1])
memory = resident * os_page()
except (OSError, IndexError, ValueError):
memory = 0
update_runtime(cpu=ratio, memory=memory)
_page = None
def os_page():
global _page
if _page is None:
import os
_page = os.sysconf("SC_PAGE_SIZE")
return _page
def _obs(value, attributes=None):
from opentelemetry.metrics import Observation
return [Observation(value, attributes or {})]
def _queue_depth(_options):
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
def _queue_age(_options):
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
def _pool_used(_options):
return _obs(_state["pool_used"], {"pool": "postgres"})
def _pool_waiting(_options):
return _obs(_state["pool_waiting"], {"pool": "postgres"})
def _worker_busy(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
def _worker_capacity(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
def _worker_consumers(_options):
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
def _db_up(_options):
return _obs(_state["db_up"])
def _cpu(_options):
return _obs(_state["cpu"])
def _memory(_options):
return _obs(_state["memory"])
def mark_error(span):
span.set_status(Status(StatusCode.ERROR))
def attach(ctx):
return otel_context.attach(ctx)
+347
View File
@@ -0,0 +1,347 @@
import datetime as dt
import hashlib
import json
import signal
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pika
from botocore.exceptions import BotoCoreError, ClientError
from opentelemetry.trace import SpanKind
from app import broker, db, storage, telemetry
from app.broker import QUEUE, _params
from app.codec import utcnow
from app.errors import AppError
from app.faults import consume_process_failure
from app.images import thumbnail_jpeg
from app.poller import start
from app.settings import settings
_inflight = 0
_inflight_lock = threading.Lock()
_stop = threading.Event()
class Transient(Exception):
pass
class Permanent(Exception):
pass
@contextmanager
def _busy():
global _inflight
with _inflight_lock:
_inflight += 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
try:
yield
finally:
with _inflight_lock:
_inflight -= 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
def _mark_failed(attachment_id, attempt, code):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(code, utcnow(), attachment_id, attempt),
)
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
def _release(attachment_id, attempt):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(utcnow(), attachment_id, attempt),
)
def process_delivery(body, headers):
started = time.perf_counter()
try:
payload = json.loads(body.decode("utf-8"))
attachment_id = uuid.UUID(str(payload["attachment_id"]))
pipeline = int(payload["pipeline_version"])
except (ValueError, UnicodeError, KeyError, TypeError):
telemetry.log_event("job.poison", severity="ERROR")
return "ack"
carrier = {}
for key, value in (headers or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
carrier[str(key).lower()] = value
if payload.get("traceparent"):
carrier.setdefault("traceparent", payload["traceparent"])
if payload.get("tracestate"):
carrier.setdefault("tracestate", payload["tracestate"])
parent = telemetry.extract_context(carrier)
outcome = None
with telemetry.tracer().start_as_current_span(
"attachment.process consume",
context=parent,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "consume",
},
) as span:
try:
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
except Exception:
telemetry.mark_error(span)
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
action, outcome = "requeue", "retry"
attempt = None
if outcome:
telemetry.record_job(outcome, time.perf_counter() - started)
if attempt is not None:
span.set_attribute("attempt", attempt)
if outcome in {"failed", "retry"}:
telemetry.mark_error(span)
return action
def _consume(payload, attachment_id, pipeline, span):
now = utcnow()
with db.connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not row or int(row["pipeline_version"]) != pipeline:
return "ack", None, None
if row["status"] == "ready":
return "ack", None, int(row["attempt"])
if row["status"] == "failed":
return "ack", None, int(row["attempt"])
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
held_attempt = int(row["attempt"])
staging_key = None
else:
held_attempt = None
attempt = int(row["attempt"]) + 1
if attempt > settings.job_max_attempts:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
WHERE id = %s
""",
(now, attachment_id),
)
return "ack", "failed", attempt
lease = now + dt.timedelta(seconds=settings.job_timeout)
conn.execute(
"""
UPDATE attachments
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
WHERE id = %s
""",
(attempt, lease, settings.instance_id, now, attachment_id),
)
staging_key = row["staging_key"]
content_type = row["content_type"]
expected_sha = row["sha256"]
created_at = row["created_at"]
if staging_key is None:
span.add_event("lease_held")
time.sleep(1)
return "requeue", None, held_attempt
span.set_attribute("attempt", attempt)
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
try:
if consume_process_failure():
span.add_event("retry", {"reason": "injected"})
raise Transient()
data = storage.get(staging_key, "staging")
if hashlib.sha256(data).hexdigest() != expected_sha:
raise Permanent("invalid_image")
thumb, width, height = thumbnail_jpeg(data)
original_key = f"originals/{attachment_id}"
thumb_key = f"variants/{attachment_id}/thumbnail"
storage.put(original_key, data, content_type, "original")
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
except Permanent as exc:
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
return "ack", "failed", attempt
except AppError as exc:
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
_mark_failed(attachment_id, attempt, code)
return "ack", "failed", attempt
except Transient:
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
time.sleep(0.5)
return "requeue", "retry", attempt
except (BotoCoreError, ClientError, OSError, TimeoutError):
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
time.sleep(min(2 ** max(attempt - 1, 0), 8))
return "requeue", "retry", attempt
thumb_sha = hashlib.sha256(thumb).hexdigest()
ready_at = utcnow()
with db.connection() as conn:
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
return "ack", None, attempt
conn.execute(
"""
INSERT INTO attachment_variants (
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
SET content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256,
object_key = EXCLUDED.object_key
""",
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
)
conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
lease_until = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(original_key, ready_at, ready_at, attachment_id, attempt),
)
try:
storage.delete(staging_key)
except (BotoCoreError, ClientError, OSError):
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
telemetry.record_ready((ready_at - created_at).total_seconds())
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
return "ack", "success", attempt
def _serve(connection):
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_qos(prefetch_count=settings.worker_concurrency)
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
def on_message(ch, method, properties, body):
def work():
with _busy():
try:
action = process_delivery(body, properties.headers if properties else None)
except Exception:
action = "requeue"
def finish():
try:
if action == "ack":
ch.basic_ack(method.delivery_tag)
else:
ch.basic_nack(method.delivery_tag, requeue=True)
except Exception:
telemetry.log_event("job.ack_failed", severity="ERROR")
connection.add_callback_threadsafe(finish)
executor.submit(work)
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
def watch():
_stop.wait()
connection.add_callback_threadsafe(channel.stop_consuming)
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
channel.start_consuming()
executor.shutdown(wait=True, cancel_futures=False)
class _Health(BaseHTTPRequestHandler):
def do_GET(self):
if self.path.startswith("/health/live"):
self._send(200)
return
if self.path.startswith("/health/ready"):
try:
with db.connection() as conn:
conn.execute("SELECT 1")
ok = storage.ready() and broker.ready()
except Exception:
ok = False
self._send(200 if ok else 503)
return
self._send(404)
def _send(self, status):
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
return
def _health_server():
server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health)
server.serve_forever()
def _request_stop(signum, _frame):
_stop.set()
def main():
signal.signal(signal.SIGTERM, _request_stop)
signal.signal(signal.SIGINT, _request_stop)
telemetry.init()
from app.main import _retry
_retry("postgres", db.init)
db.migrate()
_retry("s3", storage.init)
_retry("broker", broker.init)
telemetry.note_worker(0, settings.worker_concurrency)
start()
threading.Thread(target=_health_server, name="health", daemon=True).start()
while not _stop.is_set():
connection = None
try:
connection = pika.BlockingConnection(_params())
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
_serve(connection)
except Exception:
if _stop.is_set():
break
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
time.sleep(1)
finally:
if connection is not None and connection.is_open:
try:
connection.close()
except Exception:
pass
if __name__ == "__main__":
main()
+236
View File
@@ -0,0 +1,236 @@
name: mtusi-lab6
x-app-env: &app-env
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_KEY}
BROKER_URL: amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F
RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
DB_POOL_MAX: "8"
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
SERVICE_VERSION: "1.6.0"
FAULT_DIR: /faults
AWS_EC2_METADATA_DISABLED: "true"
services:
postgres:
image: postgres:16.6-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: messenger
volumes:
- pg-data:/var/lib/postgresql/data
- ./app/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
mem_limit: 512m
restart: unless-stopped
s3:
image: chrislusf/seaweedfs:4.51
command: ["server", "-dir=/data", "-ip=0.0.0.0", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0", "-s3.config=/etc/seaweed/s3.json"]
volumes:
- s3-data:/data
- ./infra/seaweed/s3.json:/etc/seaweed/s3.json:ro
healthcheck:
test: ["CMD", "weed", "version"]
interval: 5s
timeout: 5s
retries: 20
start_period: 5s
mem_limit: 512m
restart: unless-stopped
rabbitmq:
image: rabbitmq:3.13.7-management-alpine
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
volumes:
- rabbit-data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
mem_limit: 512m
restart: unless-stopped
api-1: &api
build: .
image: mtusi-lab6-messenger:1.6.0
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-api
INSTANCE_ID: api-1
APP_ROLE: api
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-1
volumes:
- ./faults:/faults
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
interval: 5s
timeout: 4s
retries: 30
start_period: 20s
mem_limit: 512m
restart: unless-stopped
api-2:
<<: *api
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-api
INSTANCE_ID: api-2
APP_ROLE: api
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-2
worker:
build: .
image: mtusi-lab6-messenger:1.6.0
command: ["python", "-m", "app.worker"]
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-worker
INSTANCE_ID: worker-1
APP_ROLE: worker
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=worker-1
volumes:
- ./faults:/faults
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=3)"]
interval: 5s
timeout: 4s
retries: 30
start_period: 20s
mem_limit: 512m
restart: unless-stopped
nginx:
image: nginx:1.27.4-alpine
depends_on:
api-1:
condition: service_healthy
api-2:
condition: service_healthy
ports:
- "127.0.0.1:8080:80"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
mem_limit: 128m
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
command: ["--config=/etc/otelcol/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
depends_on:
- tempo
- loki
- prometheus
mem_limit: 320m
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.55.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=2d
- --web.enable-lifecycle
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro
- prometheus-data:/prometheus
ports:
- "127.0.0.1:9090:9090"
mem_limit: 512m
restart: unless-stopped
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yml"]
volumes:
- ./infra/loki/loki.yml:/etc/loki/loki.yml:ro
- loki-data:/loki
ports:
- "127.0.0.1:3100:3100"
mem_limit: 512m
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.2
command: ["-config.file=/etc/tempo/tempo.yml"]
volumes:
- ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro
- tempo-data:/var/tempo
ports:
- "127.0.0.1:3200:3200"
mem_limit: 512m
restart: unless-stopped
grafana:
image: grafana/grafana:11.5.2
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_SERVER_HTTP_PORT: "3000"
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
depends_on:
- prometheus
- loki
- tempo
mem_limit: 384m
restart: unless-stopped
volumes:
pg-data:
s3-data:
rabbit-data:
prometheus-data:
loki-data:
tempo-data:
grafana-data:
+18
View File
@@ -0,0 +1,18 @@
# API error budget
SLO: 99% of business HTTP responses are not 5xx, measured on routes other than `/health/*`.
4xx is a client outcome and is outside the numerator. The ratio is `5xx / all business requests`.
Windows used on this lab stand:
- fast burn: 1 minute and 5 minutes, both above 14.4% (14.4 times a 1% budget), `for` 30s, and at least 0.2 requests/s
- slow burn: 5 minutes and 15 minutes, both above 6%, `for` 1 minute
- simple page: 5 minute ratio above 5% for 1 minute
A short spike can move the 1 minute window without moving the 5 minute window enough, so the fast burn stays inactive. A sustained burst moves both.
1. Open Prometheus `ALERTS` or the Messenger dashboard panel "Prometheus alert state".
2. If `ApiErrorBudgetFastBurn` or `ApiHighErrorRate` is firing, open a trace for a 5xx request.
3. If the `db.transaction` span is the long or erroring child, follow `docs/runbooks/db-latency.md`.
4. If the server span is long and the database span is short, the handler itself is slow (`handler_slow_ms` fault or application code).
5. Remove the fault file and wait for the next evaluations. The alert leaves `firing` when the expression is false; `for` does not delay resolution.
+15
View File
@@ -0,0 +1,15 @@
# Slow or unavailable database
`messenger_db_up` is 0 when the process cannot run `SELECT 1`. Business calls then return 503 `unavailable`. That increases the API error ratio. Health `/health/live` stays 200. `/health/ready` becomes 503.
A slow database is different from a slow handler:
- slow database: the `db.transaction` span is long and the server span is about the same length
- slow handler: `handler_slow` sits in the server span, and `db.transaction` stays short
`DatabaseUnready` fires when `min(messenger_db_up) == 0` for 30 seconds.
1. Compare one server span with its `db.transaction` child in Tempo.
2. If the database span failed, check PostgreSQL. Grafana being down does not cause this alert.
3. Restore PostgreSQL. `messenger_db_up` returns to 1 and the error-budget alerts resolve after the windows slide.
4. The collector and Grafana are not on the readiness path.
+14
View File
@@ -0,0 +1,14 @@
# Queue lag and worker loss
`messenger_queue_oldest_age_seconds` is the age of the oldest attachment still `queued` or `processing`. It is not the duration of a completed job. `messenger_queue_depth` is how many such rows exist. Both are read from PostgreSQL by the API, so they keep moving when the worker process is stopped.
`messenger_worker_consumers` is the RabbitMQ consumer count. The API polls the management API, so the series becomes 0 instead of going stale.
Alert `QueueLagHigh`: max age above 30 seconds for 1 minute.
Alert `WorkerConsumersZero`: max consumers equals 0 for 1 minute.
1. Confirm consumers are 0 or age is climbing on the Messenger dashboard.
2. Check worker logs in Loki for `job.failed` or `broker.reconnect`. A clean stop has neither new `job.consumed` nor a database error.
3. Start the worker again: `docker compose --env-file .env start worker`.
4. Age returns to 0 after the backlog is acknowledged. Alerts resolve on the next evaluation.
5. Do not use p95 time-to-ready alone: it only observes jobs that reached `ready`.
+8
View File
@@ -0,0 +1,8 @@
# Telemetry backend unavailable
The API and worker export OTLP on a background queue (256 spans or log records, 5 second timeout). The collector limits itself to 192 MiB, batches, and keeps a sending queue of 200 items with retry capped at 30 seconds. When Tempo, Loki, or Prometheus is down, exports fail and `messenger_telemetry_dropped_total` increases. Message acceptance does not wait on those exports.
1. Stop one backend, for example `docker compose --env-file .env stop tempo`.
2. Send a few API requests. `/health/ready` stays 200 and messages still return 201.
3. Watch collector memory. It should stay near its limit, not grow without bound.
4. Start the backend again. New spans appear. Spans dropped during the outage are not replayed from the in-memory queue once it has overflowed.
+11
View File
@@ -0,0 +1,11 @@
# Time to ready
SLO: 95% of attachments that reach `ready` do so within 10 seconds of durable acceptance.
The series is `messenger_attachment_time_to_ready_seconds`, a histogram in seconds.
`AttachmentReadyLatencyHigh` uses `histogram_quantile(0.95, ...)` over 15 minutes and pages after 2 minutes above 10 seconds.
This SLO does not see jobs that never finish. Pair it with queue age.
1. If p95 is high and queue age is low, open a worker trace and compare `s3.put`, `s3.get`, and `db.transaction`.
2. A retry span with `attempt > 1` and event `retry` means the first attempt was transient. The successful attempt should still contain S3 and the ready update.
3. If p95 is quiet but users wait, read queue age. The worker is likely stopped or stuck before completion.
+78
View File
@@ -0,0 +1,78 @@
{
"alerts_before": {
"AttachmentReadyLatencyHigh": "pending"
},
"while_worker_stopped": {
"first_firing": {
"WorkerConsumersZero": 75.5,
"QueueLagHigh": 105.7
},
"samples": [
{
"t": 90.6,
"alerts": {
"QueueLagHigh": "pending",
"AttachmentReadyLatencyHigh": "firing",
"WorkerConsumersZero": "firing"
},
"age": "82.87327",
"api_consumers": "0"
},
{
"t": 95.6,
"alerts": {
"QueueLagHigh": "pending",
"AttachmentReadyLatencyHigh": "firing",
"WorkerConsumersZero": "firing"
},
"age": "88.417685",
"api_consumers": "0"
},
{
"t": 100.7,
"alerts": {
"QueueLagHigh": "pending",
"AttachmentReadyLatencyHigh": "firing",
"WorkerConsumersZero": "firing"
},
"age": "92.908538",
"api_consumers": "0"
},
{
"t": 105.7,
"alerts": {
"QueueLagHigh": "firing",
"AttachmentReadyLatencyHigh": "firing",
"WorkerConsumersZero": "firing"
},
"age": "98.453736",
"api_consumers": "0"
}
]
},
"after_worker_start": {
"cleared_after_seconds": {
"QueueLagHigh": 20.1,
"WorkerConsumersZero": 20.1
},
"alerts": {
"AttachmentReadyLatencyHigh": "firing"
}
},
"tempo_down_40s": {
"api_results": {
"200": 76
},
"failed_before": [
"9"
],
"failed_after": [
"24"
],
"queue_size": [
"0",
"22"
],
"collector_memory": "68.8MiB / 320MiB"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.078,
"concurrency": 2,
"successful_requests": 1247,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 82.703,
"successful_latency_ms": {
"p50": 31.0,
"p95": 47.0,
"p99": 47.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+16
View File
@@ -0,0 +1,16 @@
{
"trace_id": "cf5025ad9d9a2a248da8335a0c6caee2",
"log": {
"job.retry": {"attempt": 1, "same_trace": true},
"job.ready": {"attempt": 2, "same_trace": true}
},
"chain": [
"POST /api/v1/chats/{chat_id}/attachments",
"attachment.process publish parent=POST",
"attachment.process consume (attempt 1, injected transient) parent=publish",
"attachment.process consume (attempt 2) parent=publish",
"s3.get and s3.put parent=second consume",
"db.transaction parent=POST and parent=consume"
],
"note": "Both consumer spans share the producer span as parent. No message text, filename, or object key is stored here."
}
+89
View File
@@ -0,0 +1,89 @@
{
"started": "2026-11-06T21:40:04Z",
"retry_upload": {
"final_status": "ready",
"log": {
"event": "job.ready",
"attempt": 1,
"trace_id": "40c8e289c22994a3c786030354799554",
"span_id": "606017e0d8c6f6b5",
"service": "messenger-worker"
},
"consume_spans": 1,
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"db.transaction",
"db.transaction",
"s3.put",
"db.transaction",
"db.transaction",
"attachment.process publish",
"attachment.process consume",
"db.transaction",
"s3.get",
"s3.put",
"s3.put",
"db.transaction"
],
"parents_linked": true
},
"warmup": {
"200": 180
},
"alerts_after_warmup": {},
"short_spike": {
"500": 20
},
"alerts_after_short_spike": {
"ApiHighErrorRate": "pending",
"ApiErrorBudgetSlowBurn": "pending"
},
"recovery_after_spike": {
"200": 40
},
"fast_burn_detected_after_seconds": 51.6,
"alerts_when_fast_burn_fired": {
"ApiHighErrorRate": "firing",
"ApiErrorBudgetSlowBurn": "firing",
"ApiErrorBudgetFastBurn": "firing"
},
"sustained_counts_until_detection_or_timeout": {
"500": 104
},
"fast_burn_resolved_after_seconds": 70.6,
"alerts_after_resolve": {
"ApiHighErrorRate": "firing",
"ApiErrorBudgetSlowBurn": "firing"
},
"worker_stopped": {
"consumers_firing_after_seconds": null,
"queue_lag_firing_after_seconds": 0.0,
"alerts": {
"ApiHighErrorRate": "firing",
"ApiErrorBudgetSlowBurn": "firing",
"QueueLagHigh": "firing"
}
},
"worker_restored": {
"consumers_cleared_after_seconds": 0.0,
"alerts": {
"ApiHighErrorRate": "firing",
"ApiErrorBudgetSlowBurn": "firing",
"QueueLagHigh": "firing"
}
},
"latency_note": "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span",
"tempo_stopped": {
"api_results": {
"200": 16
},
"failed_spans_before": [
"0"
],
"failed_spans_after": [
"0"
],
"collector_memory": "67.75MiB / 320MiB"
},
"finished": "2026-11-06T21:48:01Z"
}
+11
View File
@@ -0,0 +1,11 @@
Local fault-injection files. They are not present by default, so the API behaves normally.
Write one of these names in this directory while the stack is running:
- `http_error_ratio` — float 0..1, fraction of `/api/` requests that return 500
- `db_slow_ms` — delay added inside the `db.transaction` span
- `db_fail` — `1` makes database calls return 503
- `handler_slow_ms` — delay inside the HTTP server span and outside the database span
- `process_fail_remaining` — positive integer, worker fails that many processing attempts once each
Do not put secrets here. The worker must be able to update `process_fail_remaining`.
+18 -4
View File
@@ -1,7 +1,21 @@
# Инфраструктура студента
# Инфраструктура
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Топология — корневой `compose.yaml`. `compose.example.yaml` остаётся worksheet курса и не запускается.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
| Роль | Сервис | Образ | Порт на хосте | Том |
| --- | --- | --- | --- | --- |
| PostgreSQL | postgres | postgres:16.6-alpine | нет | pg-data |
| S3 | s3 | chrislusf/seaweedfs:4.51 | нет | s3-data, плюс gitignored `infra/seaweed/s3.json` |
| Брокер | rabbitmq | rabbitmq:3.13.7-management-alpine | нет | rabbit-data |
| API | api-1, api-2 | сборка из Dockerfile | нет | `./faults` |
| Worker | worker | та же сборка | нет | `./faults` |
| Балансировщик | nginx | nginx:1.27.4-alpine | 127.0.0.1:8080 | — |
| Collector | otel-collector | otel/opentelemetry-collector-contrib:0.120.0 | нет | — |
| Метрики | prometheus | prom/prometheus:v2.55.1 | 127.0.0.1:9090 | prometheus-data |
| Логи | loki | grafana/loki:3.3.2 | 127.0.0.1:3100 | loki-data |
| Трассы | tempo | grafana/tempo:2.7.2 | 127.0.0.1:3200 | tempo-data |
| UI | grafana | grafana/grafana:11.5.2 | 127.0.0.1:3000 | grafana-data |
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
Секреты приходят из `.env` (БД, S3, RabbitMQ, пароль Grafana). Идентичность SeaweedFS генерируется `python scripts/render-s3-config.py` и не коммитится. Схема БД — `app/schema.sql`. Бакет и очередь создают приложения при старте.
Grafana не хранит метрики, логи и трассы. Datasources и дашборд `messenger-overview` импортируются из `infra/grafana/provisioning` и `infra/grafana/dashboards`. Правила алертов — `infra/prometheus/rules.yml`.
+187
View File
@@ -0,0 +1,187 @@
{
"uid": "messenger-overview",
"title": "Messenger observability",
"timezone": "utc",
"schemaVersion": 39,
"version": 1,
"refresh": "10s",
"tags": ["messenger", "lab6"],
"time": {"from": "now-1h", "to": "now"},
"panels": [
{
"id": 1,
"type": "text",
"title": "Signal stores",
"gridPos": {"h": 4, "w": 24, "x": 0, "y": 0},
"options": {
"mode": "markdown",
"content": "Prometheus stores metrics (rates, histograms, gauges). Loki stores structured logs. Tempo stores traces. The collector is not a database: it receives OTLP, redacts, batches, and exports. Grafana only queries those stores. Percentiles use `histogram_quantile` on cumulative buckets, summed across API replicas before the quantile."
}
},
{
"id": 2,
"type": "timeseries",
"title": "API RPS by route",
"description": "Unit: requests per second. Sum of rates across replicas.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
"fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []},
"targets": [
{
"refId": "A",
"expr": "sum by (http_route) (rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
"legendFormat": "{{http_route}}"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "API error ratio (5xx / business)",
"description": "Unit: fraction 0-1. Numerator is status class 5xx. Denominator is business routes, excluding /health and unmatched. 4xx is not an error for this SLI.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
"targets": [
{"refId": "A", "expr": "messenger:api_error_ratio:rate5m", "legendFormat": "5m"},
{"refId": "B", "expr": "messenger:api_error_ratio:rate1m", "legendFormat": "1m"}
]
},
{
"id": 4,
"type": "timeseries",
"title": "API duration p50/p95/p99",
"description": "Unit: seconds. histogram_quantile over the sum of bucket rates. Not an average of per-replica quantiles.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p50"},
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p95"},
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p99"}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Jobs by outcome",
"description": "Unit: jobs per second. outcome is success, failed, or retry. job_type is attachment.process.v1.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", "legendFormat": "{{outcome}}"}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Worker job duration p95",
"description": "Unit: seconds, from the job histogram.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 20},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Queue depth and oldest age",
"description": "Depth is a count of queued+processing attachments. Age is seconds since the oldest of those rows was accepted. max() avoids double-counting the two API replicas.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 20},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "age"}, "properties": [{"id": "unit", "value": "s"}]}]},
"targets": [
{"refId": "A", "expr": "max(messenger_queue_depth{queue=\"attachment.process.v1\"})", "legendFormat": "depth"},
{"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachment.process.v1\"})", "legendFormat": "age"}
]
},
{
"id": 8,
"type": "timeseries",
"title": "Time to ready p95",
"description": "Unit: seconds from durable acceptance to ready. Completed jobs only.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 20},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m])))", "legendFormat": "p95"}
]
},
{
"id": 9,
"type": "timeseries",
"title": "DB pool used and waiting",
"description": "Unit: connections. Summed across processes. waiting is requests blocked on the pool.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 28},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "sum(messenger_db_pool_used)", "legendFormat": "used"},
{"refId": "B", "expr": "sum(messenger_db_pool_waiting)", "legendFormat": "waiting"}
]
},
{
"id": 10,
"type": "timeseries",
"title": "Worker saturation and consumers",
"description": "busy/capacity are counts inside the worker. consumers is the RabbitMQ consumer count observed by the API.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 28},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"},
{"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"},
{"refId": "C", "expr": "max(messenger_worker_consumers{service_name=\"messenger-api\"})", "legendFormat": "consumers"}
]
},
{
"id": 11,
"type": "timeseries",
"title": "Process CPU and memory",
"description": "CPU unit is process CPU-seconds per wall second (can exceed 1). Memory unit is bytes of RSS.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 28},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "max by (service_name) (messenger_process_cpu_utilization)", "legendFormat": "cpu {{service_name}}"},
{"refId": "B", "expr": "max by (service_name) (messenger_process_memory_bytes)", "legendFormat": "rss {{service_name}}"}
]
},
{
"id": 12,
"type": "timeseries",
"title": "DB duration p95",
"description": "Unit: seconds. Includes an injected db_slow fault. Compare with the server span to separate a slow database from a slow handler.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_db_duration_seconds_bucket[5m])))", "legendFormat": "db p95"}
]
},
{
"id": 13,
"type": "table",
"title": "Prometheus alert state",
"description": "Local firing/pending state. No external notification channel is configured.",
"datasource": {"type": "prometheus", "uid": "prometheus"},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
"targets": [
{"refId": "A", "expr": "ALERTS{alertstate=~\"firing|pending\"}", "instant": true, "format": "table"}
]
},
{
"id": 14,
"type": "logs",
"title": "API and worker logs",
"datasource": {"type": "loki", "uid": "loki"},
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 44},
"targets": [
{"refId": "A", "expr": "{service_name=~\"messenger-api|messenger-worker\"}"}
]
}
]
}
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 30
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,41 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherRegex: "\"trace_id\":\"([0-9a-f]+)\""
url: "$${__value.raw}"
datasourceUid: tempo
urlDisplayLabel: View trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
lokiSearch:
datasourceUid: loki
+39
View File
@@ -0,0 +1,39 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
volume_enabled: true
reject_old_samples: false
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
max_query_series: 500
analytics:
reporting_enabled: false
+22
View File
@@ -0,0 +1,22 @@
upstream messenger_api {
server api-1:8080 max_fails=1 fail_timeout=5s;
server api-2:8080 max_fails=1 fail_timeout=5s;
}
server {
listen 80;
client_max_body_size 12m;
proxy_http_version 1.1;
proxy_connect_timeout 1s;
proxy_send_timeout 30s;
proxy_read_timeout 30s;
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
proxy_next_upstream_tries 2;
location / {
proxy_pass http://messenger_api;
proxy_set_header Host $host;
proxy_set_header Connection "";
}
}
+12 -4
View File
@@ -1,7 +1,15 @@
# Конфигурация наблюдаемости — реализовать
# Collector
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
`collector.yaml` принимает OTLP gRPC `:4317` и HTTP `:4318` только внутри сети Compose.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
Процессоры на всех трёх pipeline: memory limiter (192 MiB, spike 48 MiB), удаление URL, SQL и заголовков Authorization/Cookie, batch.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Экспорт:
- трассы — OTLP gRPC в Tempo, очередь 200, retry не дольше 30 с;
- логи — OTLP HTTP в Loki `/otlp/v1/logs`, та же очередь и retry;
- метрики — Prometheus exporter `:8889`, его скрейпит Prometheus.
Внутренние метрики Collector — `:8888`. Health — `:13133`. Эти порты наружу не опубликованы.
Если Tempo или Loki недоступны, очередь exporter ограничена 200, retry обрывается через 30 с, memory limiter не даёт процессу расти до лимита контейнера 320 MiB. Приложение при этом отвечает: экспорт не блокирует приём сообщений.
+95
View File
@@ -0,0 +1,95 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
attributes/redact:
actions:
- key: url.full
action: delete
- key: url.query
action: delete
- key: url.path
action: delete
- key: http.url
action: delete
- key: http.target
action: delete
- key: db.statement
action: delete
- key: db.query.text
action: delete
- key: http.request.header.authorization
action: delete
- key: http.request.header.cookie
action: delete
batch:
timeout: 2s
send_batch_size: 256
send_batch_max_size: 512
exporters:
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
timeout: 5s
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 15s
max_elapsed_time: 30s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
otlphttp/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
tls:
insecure: true
timeout: 5s
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 15s
max_elapsed_time: 30s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
metric_expiration: 180s
service:
extensions: [health_check]
telemetry:
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/redact, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, attributes/redact, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/redact, batch]
exporters: [otlphttp/loki]
+15
View File
@@ -0,0 +1,15 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_timeout: 4s
rule_files:
- /etc/prometheus/rules.yml
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8889"]
- job_name: otel-collector-internal
static_configs:
- targets: ["otel-collector:8888"]
+96
View File
@@ -0,0 +1,96 @@
groups:
- name: messenger-sli
interval: 5s
rules:
- record: messenger:api_requests:rate1m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m]))
- record: messenger:api_5xx:rate1m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m]))
- record: messenger:api_error_ratio:rate1m
expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9)
- record: messenger:api_requests:rate5m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m]))
- record: messenger:api_5xx:rate5m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m]))
- record: messenger:api_error_ratio:rate5m
expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9)
- record: messenger:api_requests:rate15m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m]))
- record: messenger:api_5xx:rate15m
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m]))
- record: messenger:api_error_ratio:rate15m
expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9)
- name: messenger-alerts
interval: 5s
rules:
- alert: ApiHighErrorRate
expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2
for: 1m
labels:
severity: page
slo: api-availability
annotations:
summary: API business 5xx ratio is above 5%
description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m.
runbook: docs/runbooks/api-error-budget.md
- alert: ApiErrorBudgetFastBurn
expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2
for: 30s
labels:
severity: page
slo: api-availability
burn: fast
annotations:
summary: Fast multi-window burn of the 1% API error budget
description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present.
runbook: docs/runbooks/api-error-budget.md
- alert: ApiErrorBudgetSlowBurn
expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2
for: 1m
labels:
severity: ticket
slo: api-availability
burn: slow
annotations:
summary: Slow multi-window burn of the API error budget
description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss.
runbook: docs/runbooks/api-error-budget.md
- alert: QueueLagHigh
expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30
for: 1m
labels:
severity: page
slo: time-to-ready
annotations:
summary: Oldest attachment job is older than 30s
description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds.
runbook: docs/runbooks/queue-lag.md
- alert: WorkerConsumersZero
expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0
for: 1m
labels:
severity: page
annotations:
summary: RabbitMQ has no attachment consumers
description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops.
runbook: docs/runbooks/queue-lag.md
- alert: AttachmentReadyLatencyHigh
expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10
for: 2m
labels:
severity: ticket
slo: time-to-ready
annotations:
summary: p95 time to ready is above 10s
description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that.
runbook: docs/runbooks/time-to-ready.md
- alert: DatabaseUnready
expr: min(messenger_db_up) == 0
for: 30s
labels:
severity: page
annotations:
summary: An API or worker process cannot use PostgreSQL
description: Distinguish this from a slow handler by comparing the db.transaction span with the server span.
runbook: docs/runbooks/db-latency.md
+9
View File
@@ -0,0 +1,9 @@
# SeaweedFS S3 identity
`s3.json` is generated locally and is not committed:
```
python scripts/render-s3-config.py
```
The script reads `S3_ACCESS_KEY` and `S3_SECRET_KEY` from `.env` and writes this directory's identity file. Compose mounts it read-only at `/etc/seaweed/s3.json`.
+34
View File
@@ -0,0 +1,34 @@
stream_over_http_enabled: true
server:
http_listen_port: 3200
grpc_listen_port: 9096
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
trace_idle_period: 10s
max_block_duration: 5m
compactor:
compaction:
block_retention: 48h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/traces
querier:
max_concurrent_queries: 4
+10
View File
@@ -0,0 +1,10 @@
starlette==0.45.3
uvicorn==0.34.0
psycopg[binary,pool]==3.2.6
argon2-cffi==23.1.0
boto3==1.37.38
pika==1.3.2
Pillow==11.1.0
opentelemetry-api==1.29.0
opentelemetry-sdk==1.29.0
opentelemetry-exporter-otlp-proto-http==1.29.0
+109
View File
@@ -0,0 +1,109 @@
"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
import json
import subprocess
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
OUT = ROOT / "evidence" / "incidents.json"
def prom(expr):
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)["data"]["result"]
def alerts():
return {
item["metric"].get("alertname"): item["metric"].get("alertstate")
for item in prom("ALERTS")
}
def reload_rules():
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
with urllib.request.urlopen(request, timeout=10) as response:
return response.status
def main():
print("reload", reload_rules(), flush=True)
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
report = {"alerts_before": alerts()}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
started = time.monotonic()
seen = {}
samples = []
while time.monotonic() - started < 130:
state = alerts()
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
sample = {
"t": round(time.monotonic() - started, 1),
"alerts": state,
"age": age[0]["value"][1] if age else None,
"api_consumers": consumers[0]["value"][1] if consumers else None,
}
samples.append(sample)
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in seen and state.get(name) == "firing":
seen[name] = sample["t"]
if len(seen) == 2:
break
time.sleep(5)
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
cleared = {}
restore_started = time.monotonic()
while time.monotonic() - restore_started < 90:
state = alerts()
for name in ("QueueLagHigh", "WorkerConsumersZero"):
if name not in cleared and name not in state:
cleared[name] = round(time.monotonic() - restore_started, 1)
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
break
time.sleep(5)
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
before_failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
counts = {}
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
time.sleep(0.5)
time.sleep(5)
memory = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
queue = prom("otelcol_exporter_queue_size")
failed = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_down_40s"] = {
"api_results": counts,
"failed_before": [item["value"][1] for item in before_failed],
"failed_after": [item["value"][1] for item in failed],
"queue_size": [item["value"][1] for item in queue],
"collector_memory": memory,
}
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
import json
import urllib.parse
import urllib.request
query = '{service_name="messenger-api"} |= "http.request.completed" |= "content"'
url = "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=15"
data = json.load(urllib.request.urlopen(url, timeout=10))
values = []
for stream in data["data"]["result"]:
values.extend(stream["values"])
print("n", len(values))
for _, line in values[-8:]:
print(line[:500])
+31
View File
@@ -0,0 +1,31 @@
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
import json
import urllib.parse
import urllib.request
def lines(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = json.load(
urllib.request.urlopen(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=20",
timeout=10,
)
)
found = []
for stream in payload["data"]["result"]:
for ts, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
found.sort()
return found[-5:]
print("retry", lines("job.retry"))
print("ready", lines("job.ready"))
+47
View File
@@ -0,0 +1,47 @@
"""Print a short, secret-free snapshot of metrics, one log, and recent traces."""
import json
import urllib.parse
import urllib.request
def get(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)
def query(expr):
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
names = get("http://127.0.0.1:9090/api/v1/label/__name__/values")["data"]
app = [name for name in names if name.startswith("http_") or name.startswith("messenger_")]
print("NAMES")
print("\n".join(app))
for expr in (
"messenger_jobs_total",
"messenger_queue_depth",
"messenger_db_up",
"messenger_worker_busy",
"messenger_worker_consumers",
):
print(expr, query(expr))
log_query = '{service_name="messenger-worker"}'
logs = get(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(log_query)
+ "&limit=2"
)
streams = logs["data"]["result"]
print("LOG_STREAMS", len(streams))
if streams:
print(streams[0]["values"][-1][1][:600])
tempo_query = '{resource.service.name="messenger-api"}'
traces = get(
"http://127.0.0.1:3200/api/search?q=" + urllib.parse.quote(tempo_query) + "&limit=5"
)
print("TRACES")
for item in traces.get("traces", []):
print(item.get("rootServiceName"), item.get("rootTraceName"), item.get("traceID"), item.get("durationMs"))
+31
View File
@@ -0,0 +1,31 @@
"""Summarize one Tempo trace by id. Prints span names and parent links only."""
import json
import sys
import urllib.request
trace_id = sys.argv[1]
payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:3200/api/traces/{trace_id}", timeout=10))
batches = payload.get("batches") or payload.get("resourceSpans") or []
# Tempo may return OTLP JSON under batches or the classic trace format.
if "trace" in payload:
spans = payload["trace"].get("spans") or []
for span in spans:
print(span.get("name") or span.get("operationName"), span.get("spanID"), span.get("parentSpanID"))
raise SystemExit
def walk(node):
if isinstance(node, dict):
if "name" in node and ("spanId" in node or "span_id" in node):
print(
node.get("name"),
"id=" + str(node.get("spanId") or node.get("span_id")),
"parent=" + str(node.get("parentSpanId") or node.get("parent_span_id") or "-"),
)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
+32
View File
@@ -0,0 +1,32 @@
"""Write the local SeaweedFS identity file from .env. The file is gitignored."""
import json
from pathlib import Path
root = Path(__file__).resolve().parents[1]
env = {}
for line in (root / ".env").read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
env[key] = value
out = root / "infra" / "seaweed" / "s3.json"
out.parent.mkdir(parents=True, exist_ok=True)
document = {
"identities": [
{
"name": "app",
"credentials": [
{
"accessKey": env["S3_ACCESS_KEY"],
"secretKey": env["S3_SECRET_KEY"],
}
],
"actions": ["Admin", "Read", "List", "Write", "Tagging"],
}
]
}
out.write_text(json.dumps(document), encoding="utf-8")
print("wrote infra/seaweed/s3.json")
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
wait_healthy() {
i=0
while [ "$i" -lt 40 ]; do
ok=1
for svc in "$@"; do
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
if [ "$health" != "healthy" ]; then
ok=0
fi
done
if [ "$ok" -eq 1 ]; then
return 0
fi
i=$((i + 1))
sleep 1
done
echo "timed out waiting for healthy: $*" >&2
return 1
}
docker compose --env-file .env restart -t 5 postgres s3 rabbitmq
wait_healthy postgres s3 rabbitmq
docker compose --env-file .env restart -t 5 api-1 api-2 worker
wait_healthy api-1 api-2 worker
docker compose --env-file .env restart -t 2 nginx
+25
View File
@@ -0,0 +1,25 @@
"""Run the public resilience test on Windows by executing the shell hook with Git Bash.
The public test launches the hook directly. That works on Linux. This runner does not
change tests/resilience.py; it only prefixes the same script with bash.
"""
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
import resilience # noqa: E402
BASH = r"C:\Program Files\Git\bin\bash.exe"
_popen = resilience.subprocess.Popen
def popen(args, **kwargs):
if args and str(args[0]).endswith(".sh"):
args = [BASH, str(args[0])]
return _popen(args, **kwargs)
resilience.subprocess.Popen = popen
resilience.run(resilience.main)
+270
View File
@@ -0,0 +1,270 @@
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
import json
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
FAULTS = ROOT / "faults"
OUT = ROOT / "evidence" / "slo-track2.json"
def get(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)
def prom(expr):
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
def alert_states():
found = {}
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
name = item["metric"].get("alertname")
found[name] = item["metric"].get("alertstate")
return found
def write_fault(name, value):
path = FAULTS / name
if value is None:
path.unlink(missing_ok=True)
else:
path.write_text(str(value), encoding="utf-8")
def clear_faults():
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
write_fault(name, None)
def drive(client, actor, seconds, rps):
interval = 1.0 / rps
deadline = time.monotonic() + seconds
counts = {}
while time.monotonic() < deadline:
started = time.monotonic()
try:
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
status = result["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
return counts
def wait_state(name, wanted, timeout):
started = time.monotonic()
last = {}
while time.monotonic() - started < timeout:
last = alert_states()
if last.get(name) == wanted or (wanted is None and name not in last):
return round(time.monotonic() - started, 1), last
time.sleep(5)
return None, last
def span_summary(trace_id):
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
spans = []
def walk(node):
if isinstance(node, dict):
if "name" in node and "spanId" in node:
start = node.get("startTimeUnixNano")
end = node.get("endTimeUnixNano")
duration = None
try:
duration = round((int(end) - int(start)) / 1e6, 1)
except (TypeError, ValueError):
duration = None
spans.append(
{
"name": node.get("name"),
"span": node.get("spanId"),
"parent": node.get("parentSpanId") or "",
"ms": duration,
}
)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return spans
def latest_worker_trace(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = get(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=5"
)
lines = []
for stream in payload["data"]["result"]:
for _, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
lines.append(body)
if not lines:
return None
body = lines[-1]
return {
"event": body.get("event"),
"attempt": body.get("attempt"),
"trace_id": body.get("trace_id"),
"span_id": body.get("span_id"),
"service": body.get("service.name"),
}
def main():
clear_faults()
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
print("bootstrap ok", flush=True)
write_fault("process_fail_remaining", "1")
attachment, _content = client.attachment(alice, chat)
ready = client.wait_ready(alice, attachment, wait_seconds=40)
time.sleep(3)
log = latest_worker_trace("job.ready")
spans = []
if log and log.get("trace_id"):
try:
spans = span_summary(log["trace_id"])
except Exception as exc:
report["retry_trace_error"] = type(exc).__name__
report["retry_upload"] = {
"final_status": ready["status"],
"log": log,
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
"span_names": [span["name"] for span in spans],
"parents_linked": any(span["parent"] for span in spans),
}
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
print("warmup", flush=True)
report["warmup"] = drive(client, alice, 90, 2)
report["alerts_after_warmup"] = alert_states()
write_fault("http_error_ratio", "1")
print("short spike", flush=True)
report["short_spike"] = drive(client, alice, 10, 2)
time.sleep(35)
report["alerts_after_short_spike"] = alert_states()
clear_faults()
report["recovery_after_spike"] = drive(client, alice, 20, 2)
write_fault("http_error_ratio", "1")
print("sustained", flush=True)
sustained_started = time.monotonic()
seen = None
counts = {}
deadline = time.monotonic() + 100
interval = 0.5
while time.monotonic() < deadline:
started = time.monotonic()
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
states = alert_states()
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
seen = round(time.monotonic() - sustained_started, 1)
report["fast_burn_detected_after_seconds"] = seen
report["alerts_when_fast_burn_fired"] = states
break
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
report["sustained_counts_until_detection_or_timeout"] = counts
if seen is None:
report["fast_burn_detected_after_seconds"] = None
report["alerts_after_sustained"] = alert_states()
clear_faults()
print("resolve", flush=True)
resolve_started = time.monotonic()
resolved = None
resolve_deadline = time.monotonic() + 180
while time.monotonic() < resolve_deadline:
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
if "ApiErrorBudgetFastBurn" not in alert_states():
resolved = round(time.monotonic() - resolve_started, 1)
break
time.sleep(5)
report["fast_burn_resolved_after_seconds"] = resolved
report["alerts_after_resolve"] = alert_states()
print("worker stop", flush=True)
import subprocess
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
report["worker_stopped"] = {
"consumers_firing_after_seconds": consumer_wait,
"queue_lag_firing_after_seconds": queue_wait,
"alerts": queue_state or consumer_state,
}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
print("latency split", flush=True)
write_fault("db_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
time.sleep(3)
write_fault("db_slow_ms", None)
write_fault("handler_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
time.sleep(3)
write_fault("handler_slow_ms", None)
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
print("tempo down", flush=True)
before = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
tempo_counts = drive(client, alice, 8, 2)
time.sleep(6)
after = prom("otelcol_exporter_send_failed_spans")
stats = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_stopped"] = {
"api_results": tempo_counts,
"failed_spans_before": [item["value"][1] for item in before],
"failed_spans_after": [item["value"][1] for item in after],
"collector_memory": stats,
}
clear_faults()
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
try:
main()
finally:
clear_faults()
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
api-1|api-2) ;;
*) echo "unexpected instance" >&2; exit 1 ;;
esac
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"