1 Commits
Author SHA1 Message Date
Leskovskiy 258013b2d6 lab6: OTel collector, Prometheus/Loki/Tempo, Grafana dashboard 2026-11-19 20:24:23 +03:00
55 changed files with 2381 additions and 40 deletions
+17 -10
View File
@@ -3,12 +3,12 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SERVICE_VERSION=6.0.0
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
# Redis/Valkey is not used: sessions live in PostgreSQL.
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
@@ -18,13 +18,20 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Durable broker + worker (lab5)
# BROKER_USER / BROKER_PASSWORD are read by compose for the broker container;
# keep real values only in your local .env.
BROKER_USER=change-me
BROKER_PASSWORD=change-me
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
JOB_QUEUE=attachment_jobs
STAGING_PREFIX=staging
MAX_JOB_ATTEMPTS=5
THUMBNAIL_SIZE=256
THUMBNAIL_QUALITY=85
# Observability (lab6): OTLP over gRPC to the collector.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
OTEL_METRIC_INTERVAL_MS=10000
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
GRAFANA_ADMIN_PASSWORD=change-me
+23 -14
View File
@@ -1,18 +1,27 @@
## Лабораторная 6
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Лесковский Сергей БПИ2304
Целевая оценка (3/4/5): 3
Трек 5, если заявлен: Не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено:
Весь минимум lab5 сохранён. Добавлено: OTel Collector contrib 0.115.1, Prometheus 2.55.1 (метрики), Loki 3.2.1 (логи), Tempo 2.6.1 (трассы), Grafana 11.3.1 с provisioning datasources и импортируемым dashboard; OTLP-инструментация API (server spans, RPS/errors/duration) и worker (span attachment.process, attachment.jobs, attachment.job.duration); структурные JSON-логи с trace_id/span_id/request_id; W3C traceparent API → broker → worker; memory_limiter и batch в Collector.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Команды запуска из чистого clone:
См. REPORT.md → «Запуск из чистого clone». Кратко: сгенерировать secrets и .env; docker compose --env-file .env build api1 && docker compose --env-file .env up -d; curl /health/ready; python tests\smoke.py; Grafana http://localhost:3000 (папка Messenger → dashboard messenger-lab6), Prometheus http://localhost:9090.
Результаты проверок и ссылка на REPORT.md:
Smoke PASS при включённой телеметрии; persistence PASS; failover PASS (104 записи, 0 ошибок, recovery 1.639 s). В Prometheus видны http_server_requests_total и attachment_jobs_total по outcome; в Tempo — реальные API- и worker-трассы, связанные traceparent; в Loki — логи с trace_id/span_id. Доказательства — evidence/smoke.txt, metric-*.json, tr-traces.json, logs-trace.json, grafana-ds-*.json, docker-compose-ps.txt.
Ограничения / известные ошибки:
Уровень 4 не заявлен (queue depth/oldest job age, DB pool/worker saturation, alert rules, p50/p95/p99 везде). Один Docker-host, HA хранилищ телеметрии нет; Grafana анонимна для учебного стенда. При остановке telemetry backend API продолжает работу, телеметрия за это окно теряется.
Источники и использование ИИ:
OpenTelemetry SDK 1.28.2, instrumentation-fastapi 0.49b2, OTel Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1, FastAPI 0.115, RabbitMQ 3.13, PostgreSQL 16.4, MinIO. ИИ использовался для консультаций по OTel SDK и provisioning Grafana.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+39
View File
@@ -0,0 +1,39 @@
# syntax=docker/dockerfile:1.7
FROM python:3.12-slim AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=60 \
PIP_RETRIES=2
WORKDIR /app
COPY requirements.txt .
# Пробуем несколько зеркал по очереди. Первое, которое ответит, используется.
RUN --mount=type=cache,target=/root/.cache/pip \
set -e; \
for idx in \
"https://pypi.depkit.ru/simple/" \
"https://pypi-mirror.gitverse.ru/simple" \
"https://pypi.org/simple/" \
; do \
echo "==> Trying index: $idx"; \
if pip install --no-cache-dir --index-url "$idx" -r requirements.txt; then \
echo "==> OK: $idx"; \
exit 0; \
fi; \
echo "==> FAILED: $idx"; \
done; \
echo "==> All indexes failed"; \
exit 1
COPY src ./src
RUN useradd -r -u 10001 -s /usr/sbin/nologin appuser
USER appuser
EXPOSE 8080
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
+44 -16
View File
@@ -1,37 +1,65 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- **Автор, группа:** Лесковский Сергей БПИ2304
- **Целевая оценка; выбранный трек 5, если нужен:** 3; трек 5 не заявлен
- **Commit/PR:** ветка `lab6-final` форка `Leskovskiy/mtusi2026lab6-Leskovskiy`, PR в `mtusi2026/mtusi2026lab6`
- **Использованные источники, библиотеки и помощь ИИ:** OpenTelemetry SDK/экспортёры 1.28.2, instrumentation-fastapi 0.49b2, OTel Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1, FastAPI 0.115, SQLAlchemy 2.0, pika 1.3.2, RabbitMQ 3.13, PostgreSQL 16.4, MinIO. ИИ использовался для консультаций по OTel SDK и provisioning Grafana.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требования: Docker с Compose v2, Python 3 (только для тестов/нагрузки), Git Bash на Windows (для resilience-обёрток).
1. Сгенерировать локальные секреты `secrets/*.txt` (postgres, database_url, s3, broker) и `.env` по блоку из `REPORT.md` репозитория lab5; добавить `BROKER_USER`, `BROKER_PASSWORD`, `OTEL_*`, `GRAFANA_ADMIN_PASSWORD`. Секреты в Git не попадают.
2. `docker compose --env-file .env build api1 && docker compose --env-file .env up -d`
3. Дождаться `curl.exe -fsS http://localhost:8080/health/ready`.
4. `$env:PYTHONUTF8="1"; python tests\smoke.py --base-url http://localhost:8080`
5. Открыть Grafana: `http://localhost:3000` (анонимный admin в учебном стенде), папка Messenger → dashboard «Messenger — lab6 observability». Prometheus — `http://localhost:9090`.
6. Остановка: `docker compose --env-file .env down`. Удаление учебных данных: `docker compose --env-file .env down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
api1/api2/worker ── OTLP/gRPC ──→ otel-collector ──→ Tempo (traces)
├──────────→ Loki (logs)
└──────────→ Prometheus scrape :8889 (metrics)
Grafana ──→ Prometheus + Loki + Tempo (provisioned datasources и dashboard)
```
- **Роли хранилищ:** Prometheus хранит метрики (histogram/counter), Tempo — трассы, Loki — структурные логи; Grafana только читает их и не хранит сигналы.
- **Collector:** `otlp` receiver (gRPC 4317/HTTP 4318), `memory_limiter` (256 MiB, spike 64) и `batch`; экспортёры prometheus/tempo/loki. Collector не является долговременным хранилищем.
- **API:** FastAPIInstrumentor (server spans), middleware считает `http.server.requests` и `http.server.request.duration` с атрибутами `http.request.method`, `http.route` (шаблон, не UUID), `http.response.status_code`. JSON-лог на запрос содержит `request_id` и `trace_id`/`span_id`.
- **Worker:** span `attachment.process` (CONSUMER), счётчик `attachment.jobs` и гистограмма `attachment.job.duration` с `job.type` и `job.outcome` (ready/failed/retry/duplicate). Логи `job.ready`/`job.failed` содержат `attachment_id` (технический ID синтетических данных) и `trace_id`.
- **Связь API → broker → worker:** W3C `traceparent` инжектится в сообщение при публикации и извлекается worker-ом; worker-span — продолжение trace upload-запроса.
- **Приватность:** в телеметрию не попадают пароли, токены, Authorization, тела сообщений, байты изображений, signed URL. `user_id`/`chat_id`/`message_id`/`attachment_id` не используются как labels метрик. Health исключён из метрик (`excluded_urls` и фильтр в dashboard).
- **Ресурсы:** `service.name` = messenger-api / messenger-worker, `service.version` = 6.0.0, `service.instance.id` = api-1/api-2/worker, `deployment.environment.name` = lab.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Весь минимум lab5 работает | `python tests\smoke.py` | `PASS: lab6 public API scenario` | `evidence/smoke.txt` |
| Collector + 3 хранилища + Grafana в стенде | `docker compose ps` | postgres/s3/broker healthy, otel-collector, prometheus, loki, tempo, grafana Up | `evidence/docker-compose-ps.txt` |
| RPS/errors/duration API | Prometheus query | `http_server_requests_total`, `http_server_request_duration_seconds` | `evidence/metric-http-rps.json` |
| Успешные/ошибочные jobs и duration worker | Prometheus query | `attachment_jobs_total{job_outcome=…}`, `attachment_job_duration_seconds` | `evidence/metric-jobs.json` |
| Реальные трассы API и worker | Tempo search | API spans и worker `attachment.process` | `evidence/tr-traces.json` |
| Связь API → worker | Tempo search по trace | один trace: upload → worker (W3C traceparent) | `evidence/tr-traces.json` |
| Структурные логи с trace_id/span_id | Loki query_range | JSON-лог с `trace_id`, `span_id`, `request_id` по найденному trace | `evidence/logs-trace.json` |
| Импортируемый dashboard | provisioning | dashboard uid `messenger-lab6` в папке Messenger, 6 панелей | `infra/grafana/dashboards/messenger.json` |
| Datasources доступны | Grafana health API | Tempo/Prometheus — OK, Loki отвечает на запросы | `evidence/grafana-ds-*.json` |
| Сохранность и failover при включённой телеметрии | resilience-тесты | persistence PASS; failover 104 записи, 0 ошибок | `evidence/persistence.txt`, `evidence/failover-1.txt` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Рестарт всего стека (persistence hook) | Данные/сессии/вложения сохранены, телеметрия продолжается | `PASS: persistence preserved...` | потерь нет |
| Остановка одной API-реплики (failover hook) | LB переключает трафик, метрики обеих реплик видны | 104 записи, 0 ошибок, recovery 1.639 s | потерь нет |
| Остановка телеметрии (Prometheus/Loki/Tempo) | API/worker продолжают работу; экспортёры ограничены memory_limiter/батчем | API отвечает 200, ошибок в логах приложения нет | потерь данных нет; телеметрия за окно недоступности теряется |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Уровень 4 не заявлен: нет queue depth/oldest job age, p50/p95/p99 по всем панелям с alert rules, DB pool и worker saturation, alert rules и их firing/resolved. Сквозная связь traceparent реализована как бонус к уровню 3.
- Метрики отдаются pull-моделью через Prometheus-экспортёр Collector; период scrape 5 s, retention 15 d.
- Один Docker-host; HA хранилищ телеметрии не заявляется. Grafana открыта анонимно для учебного стенда — для реального контура нужен SSO/пароль.
- В логах сохраняется `attachment_id` (синтетические данные) для отладки; в продакшене это требует обоснования и политики retention.
- `make test` требует запущенного стенда; при отключённой телеметрии API продолжает отвечать (проверено остановкой backend).
+213
View File
@@ -0,0 +1,213 @@
name: lab6
x-api-env: &api_env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: lab
SERVICE_VERSION: "6.0.0"
SESSION_TTL_SECONDS: "3600"
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
BROKER_URL_FILE: /run/secrets/broker_url
JOB_QUEUE: attachment_jobs
STAGING_PREFIX: staging
MAX_JOB_ATTEMPTS: "5"
THUMBNAIL_SIZE: "256"
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
OTEL_METRIC_INTERVAL_MS: "10000"
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
volumes:
- pgdata:/var/lib/postgresql/data
secrets:
- postgres_password
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 2s
retries: 30
s3:
image: milvusdb/minio:RELEASE.2024-12-18T13-15-44Z
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
volumes:
- s3data:/data
secrets:
- s3_root_user
- s3_root_password
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live || exit 1"]
interval: 5s
timeout: 3s
retries: 30
broker:
image: rabbitmq:3.13-management-alpine
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:?set BROKER_USER in .env}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?set BROKER_PASSWORD in .env}
volumes:
- rabbitdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 30
api1:
build: .
image: lab6-api:local
environment:
<<: *api_env
INSTANCE_ID: api-1
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
- broker_url
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
api2:
image: lab6-api:local
environment:
<<: *api_env
INSTANCE_ID: api-2
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
- broker_url
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
worker:
image: lab6-api:local
command: ["python", "-m", "src.worker"]
environment:
<<: *api_env
INSTANCE_ID: worker
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
- broker_url
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
lb:
image: nginx:1.27.1-alpine
ports:
- "8080:8080"
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
- api1
- api2
otel-collector:
image: otel/opentelemetry-collector-contrib:0.115.1
command: ["--config=/etc/otelcol/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
prometheus:
image: prom/prometheus:v2.55.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- promdata:/prometheus
ports:
- "9090:9090"
depends_on:
- otel-collector
loki:
image: grafana/loki:3.2.1
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
tempo:
image: grafana/tempo:2.6.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
grafana:
image: grafana/grafana:11.3.1
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
GF_USERS_DEFAULT_THEME: dark
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "3000:3000"
depends_on:
- prometheus
- loki
- tempo
volumes:
pgdata: {}
s3data: {}
rabbitdata: {}
promdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password:
file: ./secrets/postgres_password.txt
database_url:
file: ./secrets/database_url.txt
s3_root_user:
file: ./secrets/s3_root_user.txt
s3_root_password:
file: ./secrets/s3_root_password.txt
s3_access_key_id:
file: ./secrets/s3_access_key_id.txt
s3_secret_access_key:
file: ./secrets/s3_secret_access_key.txt
broker_url:
file: ./secrets/broker_url.txt
+13
View File
@@ -0,0 +1,13 @@
SERVICE STATUS
api1 Up 46 seconds
api2 Up 46 seconds
broker Up 4 minutes (healthy)
grafana Up 4 minutes
lb Up 3 minutes
loki Up 4 minutes
otel-collector Up 4 minutes
postgres Up 4 minutes (healthy)
prometheus Up 4 minutes
s3 Up 4 minutes (healthy)
tempo Up 4 minutes
worker Up 46 seconds
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
{"message":"Data source successfully connected.","status":"OK"}
@@ -0,0 +1 @@
{"details":{"application":"Prometheus","features":{"rulerApiEnabled":false}},"message":"Successfully queried the Prometheus API.","status":"OK"}
+1
View File
@@ -0,0 +1 @@
{"message":"Method not implemented","messageId":"plugin.notImplemented","statusCode":404,"traceID":""}
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1795108961.821,"0.027683377390749272"]}]}}
+1
View File
@@ -0,0 +1 @@
{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"attachment_jobs_total","exported_instance":"worker","exported_job":"messenger-worker","instance":"otel-collector:8889","job":"otel-collector","job_outcome":"duplicate","job_type":"attachment.process.v1"},"value":[1795108961.844,"1"]},{"metric":{"__name__":"attachment_jobs_total","exported_instance":"worker","exported_job":"messenger-worker","instance":"otel-collector:8889","job":"otel-collector","job_outcome":"ready","job_type":"attachment.process.v1"},"value":[1795108961.844,"1"]}]}}
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+1
View File
@@ -0,0 +1 @@
{"traces":[{"traceID":"cea4fbd4ccc714db4cffa35e3c608139","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/attachments","startTimeUnixNano":"1795108925352572507","durationMs":275},{"traceID":"ff2253c00f06cd5fd5e5f4f47fbbadab","rootServiceName":"messenger-worker","rootTraceName":"attachment.process","startTimeUnixNano":"1795108812348163525","durationMs":3},{"traceID":"bf6eef1581cd68bf80af8a689a2f4b7e","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/attachments","startTimeUnixNano":"1795108812081229408","durationMs":266}],"metrics":{"inspectedTraces":77,"inspectedBytes":"39870","completedJobs":1,"totalJobs":1}}
+108
View File
@@ -0,0 +1,108 @@
{
"uid": "messenger-lab6",
"title": "Messenger — lab6 observability",
"tags": ["messenger", "lab6"],
"schemaVersion": 39,
"version": 1,
"refresh": "10s",
"editable": true,
"time": { "from": "now-30m", "to": "now" },
"timezone": "utc",
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "API RPS (excluding health)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m]))",
"legendFormat": "RPS"
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "API 5xx ratio",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{http_response_status_code=~\"5..\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])), 0.001)",
"legendFormat": "5xx / business"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "HTTP duration p95",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
"legendFormat": "p95"
},
{
"refId": "B",
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
"legendFormat": "p50"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Attachment jobs by outcome",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum by (job_outcome) (rate(attachment_jobs_total[1m]))",
"legendFormat": "{{job_outcome}}"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Worker job duration p95 by outcome",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, job_outcome) (rate(attachment_job_duration_seconds_bucket[5m])))",
"legendFormat": "{{job_outcome}}"
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "API request rate by route (top 10)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "topk(10, sum by (http_route, http_request_method) (rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
}
]
}
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 30
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,28 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-1m"
spanEndTimeShift: "1m"
filterByTraceID: true
filterBySpanID: false
+26
View File
@@ -0,0 +1,26 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
+18
View File
@@ -0,0 +1,18 @@
events {}
http {
upstream api {
server api1:8080 max_fails=1 fail_timeout=5s;
server api2:8080 max_fails=1 fail_timeout=5s;
}
server {
listen 8080;
client_max_body_size 11m;
proxy_connect_timeout 2s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
location / {
proxy_pass http://api;
proxy_set_header Host $host;
}
}
}
+47
View File
@@ -0,0 +1,47 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 256
spike_limit_mib: 64
batch:
timeout: 5s
send_batch_size: 1024
send_batch_max_size: 2048
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: false
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
loki:
endpoint: http://loki:3100/loki/api/v1/push
service:
telemetry:
logs:
level: warn
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [loki]
+8
View File
@@ -0,0 +1,8 @@
global:
scrape_interval: 5s
evaluation_interval: 15s
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8889"]
+19
View File
@@ -0,0 +1,19 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
local:
path: /var/tempo/traces
wal:
path: /var/tempo/wal
+12
View File
@@ -0,0 +1,12 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
sqlalchemy==2.0.35
psycopg[binary]==3.2.3
boto3==1.35.36
argon2-cffi==23.1.0
pillow==10.4.0
pydantic==2.9.2
pika==1.3.2
opentelemetry-sdk==1.28.2
opentelemetry-exporter-otlp==1.28.2
opentelemetry-instrumentation-fastapi==0.49b2
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""lab5 failure drills: broker restart durability and bounded retry.
A) Stop the worker, accept an upload, restart the broker (persistent volume),
start the worker: the accepted job still becomes ready.
B) Accept an upload while the worker is down, stop S3, start the worker:
transient failures exhaust MAX_JOB_ATTEMPTS and the attachment ends as
failed with a safe error_code. S3 is restored afterwards.
"""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*args):
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_status(c, actor, att, wanted, seconds=120):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
row = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
if row["status"] in wanted:
return row
time.sleep(0.5)
raise AssertionError("attachment did not reach " + repr(wanted))
def main():
args = arguments("lab5 failure drills")
c = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, _eve, chat = c.bootstrap()
print("A. broker restart keeps accepted work")
compose("stop", "worker")
try:
att, _content = c.attachment(alice, chat)
compose("restart", "broker")
deadline = time.monotonic() + 90
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "API did not recover after broker restart")
time.sleep(0.5)
compose("up", "-d", "--no-deps", "worker")
started = time.monotonic()
ready = c.wait_ready(bob, att, wait_seconds=90)
print(json.dumps({
"scenario": "broker_restart",
"attachment_id": att["id"],
"status": ready["status"],
"seconds_to_ready": round(time.monotonic() - started, 3),
}, ensure_ascii=False))
check(ready["status"] == "ready", "job lost after broker restart")
finally:
compose("up", "-d", "--no-deps", "worker")
print("B. bounded retry ends in a safe failed state")
compose("stop", "worker")
try:
att, _content = c.attachment(alice, chat)
compose("stop", "s3")
compose("up", "-d", "--no-deps", "worker")
failed = wait_status(c, bob, att, {"failed"}, seconds=180)
print(json.dumps({
"scenario": "retry_exhausted",
"attachment_id": att["id"],
"status": failed["status"],
"error_code": failed["error_code"],
}, ensure_ascii=False))
check(failed["error_code"] in ("processing_failed", "invalid_image"), "unsafe error_code")
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
finally:
compose("up", "-d", "s3")
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 60
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "S3 did not recover")
time.sleep(0.5)
print("PASS: broker durability and bounded retry demonstrated.")
if __name__ == "__main__":
run(main)
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Demonstrate the lab5 acceptance scenario without hiding infrastructure steps.
Worker stopped -> upload returns 202 queued -> message references the chat
attachment -> API replicas restarted -> worker started -> the same job
reaches ready with a contract-compliant JPEG thumbnail.
The script prints only synthetic IDs and timings. It never prints tokens.
"""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*args):
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def main():
args = arguments("lab5 worker-outage demonstration")
c = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, _eve, chat = c.bootstrap()
print("1. stop worker")
compose("stop", "worker")
try:
print("2. upload with worker down")
att, content = c.attachment(alice, chat)
check(att["status"] == "queued", "queued expected after upload")
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
check(meta["status"] in ("queued", "processing"), "still unprocessed while worker is down")
pending = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
check(pending["status"] == 409, "content must be 409 while queued")
print("3. attach the queued attachment to a message")
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
check(msg["attachment_ids"] == [att["id"]], "message lost the attachment")
print("4. restart API replicas (accepted work must survive)")
compose("restart", "api1", "api2")
deadline = time.monotonic() + 90
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "API did not recover")
time.sleep(0.5)
print("5. start worker")
compose("up", "-d", "--no-deps", "worker")
started = time.monotonic()
ready = c.wait_ready(bob, att, wait_seconds=90)
elapsed = round(time.monotonic() - started, 3)
check(ready["status"] == "ready", "job did not reach ready")
variants = [v for v in ready["variants"] if v["name"] == "thumbnail"]
check(len(variants) == 1, "expected exactly one thumbnail")
v = variants[0]
check((v["width"], v["height"]) == (256, 160), "thumbnail dimensions differ")
thumb = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be JPEG")
from image_probe import jpeg_size
check(jpeg_size(thumb["bytes"]) == (256, 160), "actual JPEG size differs")
print(json.dumps({
"attachment_id": att["id"],
"message_id": msg["id"],
"status_after_worker_restart": ready["status"],
"thumbnail": {"width": v["width"], "height": v["height"],
"size_bytes": v["size_bytes"], "sha256": v["sha256"]},
"seconds_from_worker_start_to_ready": elapsed,
}, ensure_ascii=False))
print("PASS: queued job completed after API and worker restart.")
finally:
compose("up", "-d", "--no-deps", "worker")
if __name__ == "__main__":
run(main)
Executable → Regular
View File
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose --env-file .env restart postgres s3 broker api1 api2 worker lb
+2
View File
@@ -0,0 +1,2 @@
@echo off
"C:\Program Files\Git\bin\bash.exe" "%~dp0restart.sh"
Executable → Regular
View File
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
case "$TARGET_INSTANCE_ID" in
api-1) SVC=api1 ;;
api-2) SVC=api2 ;;
*) echo "unknown instance alias: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
esac
docker compose --env-file .env stop "$SVC"
+2
View File
@@ -0,0 +1,2 @@
@echo off
"C:\Program Files\Git\bin\bash.exe" "%~dp0stop-one.sh"
View File
+47
View File
@@ -0,0 +1,47 @@
import hashlib
import secrets
from datetime import datetime, timezone, timedelta
from fastapi import Depends, Request
from sqlalchemy.orm import Session
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from . import config
from .db import get_db
from .models import User, Session as DbSession
from .errors import ApiError
_ph = PasswordHasher()
_BEARER = 'Bearer realm="messenger"'
def hash_password(p: str) -> str:
return _ph.hash(p)
def verify_password(h: str, p: str) -> bool:
try:
return _ph.verify(h, p)
except VerifyMismatchError:
return False
def new_session(db: Session, user: User) -> tuple[str, datetime]:
token = secrets.token_urlsafe(48)
token_hash = hashlib.sha256(token.encode()).hexdigest()
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
db.add(DbSession(token_hash=token_hash, user_id=user.id, expires_at=expires))
db.commit()
return token, expires
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("bearer "):
raise ApiError(401, "unauthorized", "missing bearer token", challenge=_BEARER)
token = auth[7:].strip()
token_hash = hashlib.sha256(token.encode()).hexdigest()
s = db.get(DbSession, token_hash)
if not s:
raise ApiError(401, "unauthorized", "invalid session", challenge=_BEARER)
if s.expires_at <= datetime.now(timezone.utc):
raise ApiError(401, "unauthorized", "expired session", challenge=_BEARER)
u = db.get(User, s.user_id)
if not u:
raise ApiError(401, "unauthorized", "user gone", challenge=_BEARER)
return u
+36
View File
@@ -0,0 +1,36 @@
import os
from pathlib import Path
def _read_secret(env_name: str, file_env: str, default=None):
path = os.environ.get(file_env)
if path:
return Path(path).read_text().strip()
return os.environ.get(env_name, default)
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
APP_ENV = os.environ.get("APP_ENV", "lab")
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
DATABASE_URL = _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
S3_SECRET_KEY = _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
INSTANCE_ID = os.environ.get("INSTANCE_ID", "api-unknown")
# --- lab5: durable broker + worker ---
BROKER_URL = _read_secret("BROKER_URL", "BROKER_URL_FILE", "amqp://messenger:messenger@broker:5672/")
QUEUE_NAME = os.environ.get("JOB_QUEUE", "attachment_jobs")
STAGING_PREFIX = os.environ.get("STAGING_PREFIX", "staging")
THUMBNAIL_SIZE = int(os.environ.get("THUMBNAIL_SIZE", "256"))
THUMBNAIL_QUALITY = int(os.environ.get("THUMBNAIL_QUALITY", "85"))
MAX_JOB_ATTEMPTS = int(os.environ.get("MAX_JOB_ATTEMPTS", "5"))
OUTBOX_INTERVAL_SECONDS = float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "2"))
OUTBOX_GRACE_SECONDS = float(os.environ.get("OUTBOX_GRACE_SECONDS", "0"))
+16
View File
@@ -0,0 +1,16 @@
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, DeclarativeBase
from . import config
engine = create_engine(config.DATABASE_URL, pool_pre_ping=True, pool_size=5, max_overflow=10)
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
class Base(DeclarativeBase):
pass
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
+29
View File
@@ -0,0 +1,29 @@
from fastapi import Request
from fastapi.responses import JSONResponse
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, challenge: str | None = None):
self.status = status
self.code = code
self.message = message
self.challenge = challenge
def _rid(request: Request) -> str:
return getattr(request.state, "request_id", None) or "unknown"
async def api_error_handler(request: Request, exc: ApiError):
headers = {"X-Request-Id": _rid(request)}
if exc.challenge:
headers["WWW-Authenticate"] = exc.challenge
return JSONResponse(
status_code=exc.status,
content={"error": {"code": exc.code, "message": exc.message, "request_id": _rid(request)}},
headers=headers,
)
async def unhandled_handler(request: Request, exc: Exception):
return JSONResponse(
status_code=500,
content={"error": {"code": "internal_error", "message": "internal error", "request_id": _rid(request)}},
headers={"X-Request-Id": _rid(request)},
)
+26
View File
@@ -0,0 +1,26 @@
import io
from PIL import Image, UnidentifiedImageError
from . import config
from .errors import ApiError
def validate_image(data: bytes) -> tuple[str, int, int, int]:
"""Возвращает (content_type, width, height, size)."""
if not data:
raise ApiError(400, "invalid_request", "empty body")
if len(data) > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "file too large")
head = data[:8]
if head.startswith(b"\x89PNG\r\n\x1a\n"):
ctype = "image/png"
elif head.startswith(b"\xff\xd8\xff"):
ctype = "image/jpeg"
else:
raise ApiError(415, "unsupported_media_type", "only PNG/JPEG")
try:
img = Image.open(io.BytesIO(data))
w, h = img.size
except UnidentifiedImageError:
raise ApiError(415, "unsupported_media_type", "invalid image")
if w * h > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "too many pixels")
return ctype, w, h, len(data)
+127
View File
@@ -0,0 +1,127 @@
"""Broker access: durable job publication and the transactional-outbox pump.
The broker message carries only identifiers and metadata (never bytes or
credentials). The job row in PostgreSQL is the source of truth; `published_at`
turns the row into a small transactional outbox so a crash between COMMIT and
publish is repaired by the background pump.
"""
import json
import logging
import threading
import time
import uuid
from datetime import datetime, timezone, timedelta
import pika
from sqlalchemy import select
from . import config
from .db import SessionLocal
from .models import AttachmentJob
log = logging.getLogger("jobs")
TYPE_V1 = "attachment.process.v1"
def build_payload(job: AttachmentJob, attempt: int, carrier: dict | None = None) -> dict:
from opentelemetry.propagate import inject
if carrier is None:
carrier = {}
inject(carrier) # W3C traceparent when a request span is active
payload = {
"schema_version": 1,
"job_id": str(job.id),
"type": TYPE_V1,
"attachment_id": str(job.attachment_id),
"pipeline_version": job.pipeline_version,
"attempt": attempt,
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
}
if carrier.get("traceparent"):
payload["traceparent"] = carrier["traceparent"]
if carrier.get("tracestate"):
payload["tracestate"] = carrier["tracestate"]
return payload
def publish(payload: dict) -> None:
params = pika.URLParameters(config.BROKER_URL)
params.heartbeat = 30
params.blocked_connection_timeout = 30
connection = pika.BlockingConnection(params)
try:
channel = connection.channel()
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
channel.confirm_delivery()
channel.basic_publish(
exchange="",
routing_key=config.QUEUE_NAME,
body=json.dumps(payload).encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2, # persistent
content_type="application/json",
message_id=payload["job_id"],
),
mandatory=True,
)
finally:
connection.close()
def publish_job_row(job_id: uuid.UUID, attempt: int, carrier: dict | None = None) -> None:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if not job or job.state in ("done", "failed"):
return
payload = build_payload(job, attempt, carrier)
publish(payload)
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if job:
job.published_at = datetime.now(timezone.utc)
db.commit()
def pump_once(limit: int = 50) -> int:
"""Publish queued rows whose outbox timestamp is still empty."""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=config.OUTBOX_GRACE_SECONDS)
published = 0
with SessionLocal() as db:
rows = (
db.execute(
select(AttachmentJob)
.where(AttachmentJob.published_at.is_(None))
.where(AttachmentJob.state == "queued")
.where(AttachmentJob.created_at <= cutoff)
.order_by(AttachmentJob.created_at)
.limit(limit)
)
.scalars()
.all()
)
pending = [(job.id, job.attempt + 1) for job in rows]
for job_id, attempt in pending:
try:
publish_job_row(job_id, attempt)
published += 1
except Exception as exc: # broker may still be starting
log.warning("outbox publish failed for %s: %s", job_id, exc)
return published
def start_outbox_thread() -> threading.Thread:
def loop():
while True:
try:
count = pump_once()
if count:
log.info("outbox published %d job(s)", count)
except Exception as exc:
log.warning("outbox pump error: %s", exc)
time.sleep(config.OUTBOX_INTERVAL_SECONDS)
thread = threading.Thread(target=loop, name="outbox", daemon=True)
thread.start()
return thread
+88
View File
@@ -0,0 +1,88 @@
import json
import logging
import time
import uuid
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from . import config, jobs, telemetry
from .db import engine, Base
from .errors import ApiError, api_error_handler, unhandled_handler
from .s3 import ensure_bucket
from .routers import health, users, sessions, chats, messages, attachments
tracer_provider, meter_provider = telemetry.setup_telemetry("messenger-api", config.INSTANCE_ID)
request_counter, request_duration = telemetry.http_instruments()
logger = logging.getLogger("messenger.api")
@asynccontextmanager
async def lifespan(app: FastAPI):
# create_all is not idempotent under parallel startup: two replicas
# may race and one hits UniqueViolation on pg_type. Catch and re-run
# once after a short delay; by then the other replica has finished.
import asyncio
from sqlalchemy.exc import IntegrityError, OperationalError
for attempt in range(5):
try:
Base.metadata.create_all(engine)
break
except (IntegrityError, OperationalError) as exc:
print(f"WARN: create_all attempt {attempt+1} failed: {exc}")
await asyncio.sleep(1.0)
else:
print("WARN: create_all failed after 5 attempts; continuing")
try:
ensure_bucket()
except Exception as exc:
print(f"WARN: ensure_bucket failed: {exc}")
jobs.start_outbox_thread()
logger.info("api.started", extra={"event_fields": json.dumps({"instance_id": config.INSTANCE_ID})})
yield
app = FastAPI(lifespan=lifespan)
app.add_exception_handler(ApiError, api_error_handler)
app.add_exception_handler(Exception, unhandled_handler)
@app.middleware("http")
async def observe_request(request: Request, call_next):
started = time.perf_counter()
response = await call_next(request)
route = request.scope.get("route")
template = getattr(route, "path", request.url.path)
attributes = {
"http.request.method": request.method,
"http.route": template,
"http.response.status_code": response.status_code,
}
request_counter.add(1, attributes)
request_duration.record(time.perf_counter() - started, attributes)
logger.info("http.request", extra={"event_fields": json.dumps({
"http.request.method": request.method,
"http.route": template,
"http.response.status_code": response.status_code,
"request_id": getattr(request.state, "request_id", None)
or request.headers.get("X-Request-Id"),
})})
return response
@app.middleware("http")
async def add_request_id(request: Request, call_next):
rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex
request.state.request_id = rid
response = await call_next(request)
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
for r in (health, users, sessions, chats, messages, attachments):
app.include_router(r.router)
FastAPIInstrumentor.instrument_app(app, excluded_urls="health/live,health/ready")
+142
View File
@@ -0,0 +1,142 @@
import uuid
from datetime import datetime, timezone
from sqlalchemy import String, DateTime, ForeignKey, Integer, UniqueConstraint, Index
from sqlalchemy.dialects.postgresql import UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
from .db import Base
def _uuid() -> uuid.UUID:
return uuid.uuid4()
def _now() -> datetime:
return datetime.now(timezone.utc)
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
display_name: Mapped[str] = mapped_column(String(100))
password_hash: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
class Session(Base):
__tablename__ = "sessions"
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True) # sha256(token) hex
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), index=True
)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
class Chat(Base):
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
title: Mapped[str] = mapped_column(String(100))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
class ChatMember(Base):
__tablename__ = "chat_members"
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
class Message(Base):
__tablename__ = "messages"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
)
sender_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id")
)
text: Mapped[str] = mapped_column(String(4000), default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, default=0)
class IdempotencyKey(Base):
__tablename__ = "idempotency_keys"
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
method: Mapped[str] = mapped_column(String(8), primary_key=True)
path: Mapped[str] = mapped_column(String(255), primary_key=True)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
request_hash: Mapped[str] = mapped_column(String(64))
response_json: Mapped[str] = mapped_column(String)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
__table_args__ = (Index("ix_idem_created", "created_at"),)
class Attachment(Base):
__tablename__ = "attachments"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
)
owner_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id")
)
filename: Mapped[str] = mapped_column(String(255))
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int] = mapped_column(Integer)
sha256: Mapped[str] = mapped_column(String(64))
s3_key: Mapped[str] = mapped_column(String(255))
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
status: Mapped[str] = mapped_column(String(16), default="ready")
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
class AttachmentJob(Base):
"""Durable intent to process one attachment (transactional outbox row)."""
__tablename__ = "attachment_jobs"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True
)
pipeline_version: Mapped[int] = mapped_column(Integer, default=1)
state: Mapped[str] = mapped_column(String(16), default="queued") # queued|processing|done|failed
attempt: Mapped[int] = mapped_column(Integer, default=0)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
class AttachmentVariant(Base):
__tablename__ = "attachment_variants"
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True
)
name: Mapped[str] = mapped_column(String(32), primary_key=True)
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int] = mapped_column(Integer)
width: Mapped[int] = mapped_column(Integer)
height: Mapped[int] = mapped_column(Integer)
sha256: Mapped[str] = mapped_column(String(64))
s3_key: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
View File
+122
View File
@@ -0,0 +1,122 @@
import hashlib
import uuid
from fastapi import APIRouter, Depends, Request, Query
from fastapi.responses import Response
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import Attachment, ChatMember, AttachmentJob, AttachmentVariant
from ..auth import current_user
from ..errors import ApiError
from .. import s3, config, jobs
from ..images import validate_image
from ..schemas import AttachmentOut, VariantOut
router = APIRouter(prefix="/api/v1", tags=["attachments"])
def _bad_filename(name: str) -> bool:
return (not name) or len(name) > 255 or any(c in name for c in ("/", "\\", "\x00")) or any(ord(c) < 32 for c in name)
def _member_chat_or_404(db, chat_id, user_id):
if not db.get(ChatMember, (chat_id, user_id)):
raise ApiError(404, "not_found", "chat not found")
def _attachment_for_member(db, att_id, user_id) -> Attachment:
att = db.get(Attachment, att_id)
if not att:
raise ApiError(404, "not_found", "attachment not found")
_member_chat_or_404(db, att.chat_id, user_id)
return att
def _attachment_out(db, att: Attachment) -> AttachmentOut:
variants = db.query(AttachmentVariant).filter(
AttachmentVariant.attachment_id == att.id
).all()
return AttachmentOut(
id=att.id, chat_id=att.chat_id, owner_id=att.owner_id,
filename=att.filename, content_type=att.content_type,
size_bytes=att.size_bytes, sha256=att.sha256,
status=att.status, error_code=att.error_code,
variants=[VariantOut.model_validate(v) for v in variants],
created_at=att.created_at, updated_at=att.updated_at,
)
@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut)
async def upload(chat_id: uuid.UUID,
filename: str = Query(..., min_length=1, max_length=255),
request: Request = None,
db: Session = Depends(get_db),
user=Depends(current_user)):
_member_chat_or_404(db, chat_id, user.id)
if _bad_filename(filename):
raise ApiError(400, "invalid_request", "invalid filename")
ctype_header = request.headers.get("content-type", "").split(";")[0].strip()
if ctype_header not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "expected image/png or image/jpeg")
data = await request.body()
ctype, w, h, size = validate_image(data)
if ctype != ctype_header:
raise ApiError(415, "unsupported_media_type", "content-type mismatch with file")
digest = hashlib.sha256(data).hexdigest()
att_id = uuid.uuid4()
final_key = f"chats/{chat_id}/attachments/{att_id}"
staging_key = f"{config.STAGING_PREFIX}/{att_id}"
# 1. Durable staging: the original must survive an API restart before 202.
s3.client().put_object(Bucket=config.S3_BUCKET, Key=staging_key, Body=data, ContentType=ctype)
# 2. Metadata + durable job intent in one transaction (transactional outbox).
att = Attachment(id=att_id, chat_id=chat_id, owner_id=user.id, filename=filename,
content_type=ctype, size_bytes=size, sha256=digest, s3_key=final_key,
staging_key=staging_key, status="queued")
job = AttachmentJob(attachment_id=att_id, pipeline_version=1, state="queued", attempt=0)
try:
db.add(att)
db.add(job)
db.commit()
db.refresh(att)
except Exception:
db.rollback()
# Compensating delete for the staging object; the DB has no intent.
try:
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
except Exception:
pass
raise
# 3. Handoff to the broker. If this fails, the row stays unpublished and
# the outbox pump retries it; the client still got a durable 202.
from starlette.concurrency import run_in_threadpool
from opentelemetry.propagate import inject
carrier: dict[str, str] = {}
inject(carrier) # capture the active request span before leaving the event loop
try:
await run_in_threadpool(jobs.publish_job_row, job.id, 1, carrier)
except Exception:
pass
return _attachment_out(db, att)
@router.get("/attachments/{att_id}", response_model=AttachmentOut)
def meta(att_id: uuid.UUID, db=Depends(get_db), user=Depends(current_user)):
return _attachment_out(db, _attachment_for_member(db, att_id, user.id))
@router.get("/attachments/{att_id}/content")
def content(att_id: uuid.UUID, variant: str | None = None,
db=Depends(get_db), user=Depends(current_user)):
att = _attachment_for_member(db, att_id, user.id)
if att.status == "ready":
if variant in (None, "original"):
key, media_type = att.s3_key, att.content_type
else:
row = db.get(AttachmentVariant, (att.id, variant))
if not row:
raise ApiError(404, "not_found", "variant not found")
key, media_type = row.s3_key, row.content_type
elif att.status in ("queued", "processing"):
raise ApiError(409, "conflict", "attachment is not ready")
else:
raise ApiError(409, "conflict", "attachment failed")
obj = s3.client().get_object(Bucket=config.S3_BUCKET, Key=key)
body = obj["Body"].read()
return Response(content=body, media_type=media_type)
+44
View File
@@ -0,0 +1,44 @@
import uuid
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import User, Chat, ChatMember
from ..schemas import CreateChatIn, ChatOut, ChatPage
from ..errors import ApiError
from ..auth import current_user
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
def _chat_out(db, chat) -> ChatOut:
members = [m.user_id for m in db.query(ChatMember).filter(ChatMember.chat_id == chat.id).all()]
return ChatOut(id=chat.id, title=chat.title, member_ids=members, created_at=chat.created_at)
@router.post("", status_code=201, response_model=ChatOut)
def create_chat(body: CreateChatIn, db: Session = Depends(get_db), user: User = Depends(current_user)):
if user.id not in body.member_ids:
raise ApiError(400, "invalid_request", "caller must be a member")
found = db.query(User.id).filter(User.id.in_(body.member_ids)).all()
if len(found) != len(set(body.member_ids)):
raise ApiError(400, "invalid_request", "unknown member")
chat = Chat(title=body.title)
db.add(chat); db.flush()
for uid in body.member_ids:
db.add(ChatMember(chat_id=chat.id, user_id=uid))
db.commit(); db.refresh(chat)
return _chat_out(db, chat)
@router.get("", response_model=ChatPage)
def list_chats(db: Session = Depends(get_db), user: User = Depends(current_user)):
rows = db.query(Chat).join(ChatMember, ChatMember.chat_id == Chat.id)\
.filter(ChatMember.user_id == user.id)\
.order_by(Chat.created_at, Chat.id).limit(100).all()
return ChatPage(items=[_chat_out(db, c) for c in rows], next_cursor=None)
@router.get("/{chat_id}", response_model=ChatOut)
def get_chat(chat_id: uuid.UUID, db: Session = Depends(get_db), user: User = Depends(current_user)):
if not db.get(ChatMember, (chat_id, user.id)):
raise ApiError(404, "not_found", "chat not found")
chat = db.get(Chat, chat_id)
if not chat:
raise ApiError(404, "not_found", "chat not found")
return _chat_out(db, chat)
+19
View File
@@ -0,0 +1,19 @@
from fastapi import APIRouter, Depends
from sqlalchemy import text
from sqlalchemy.orm import Session
from ..db import get_db
from ..errors import ApiError
router = APIRouter(prefix="/health", tags=["health"])
@router.get("/live")
def live():
return {"status": "ok"}
@router.get("/ready")
def ready(db: Session = Depends(get_db)):
try:
db.execute(text("SELECT 1"))
except Exception:
raise ApiError(503, "unavailable", "database unavailable")
return {"status": "ok"}
+147
View File
@@ -0,0 +1,147 @@
import uuid
import json
import hashlib
import base64
from datetime import datetime
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import (
User,
ChatMember,
Message,
MessageAttachment,
IdempotencyKey,
Attachment,
)
from ..schemas import CreateMessageIn, MessageOut, MessagePage
from ..errors import ApiError
from ..auth import current_user
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
def _msg_out(db: Session, m: Message) -> MessageOut:
atts = [
a.attachment_id
for a in db.query(MessageAttachment)
.filter(MessageAttachment.message_id == m.id)
.order_by(MessageAttachment.position)
.all()
]
return MessageOut(
id=m.id,
chat_id=m.chat_id,
sender_id=m.sender_id,
text=m.text,
created_at=m.created_at,
attachment_ids=atts,
)
def _require_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
if not db.get(ChatMember, (chat_id, user_id)):
raise ApiError(404, "not_found", "chat not found")
@router.post("", status_code=201, response_model=MessageOut)
def send(
chat_id: uuid.UUID,
body: CreateMessageIn,
request: Request,
db: Session = Depends(get_db),
user: User = Depends(current_user),
):
_require_member(db, chat_id, user.id)
key = request.headers.get("idempotency-key", "")
if not key or len(key) > 128:
raise ApiError(400, "invalid_request", "Idempotency-Key required")
if not body.text.strip() and not body.attachment_ids:
raise ApiError(400, "invalid_request", "empty message")
for aid in body.attachment_ids:
a = db.get(Attachment, aid)
if not a or a.chat_id != chat_id:
raise ApiError(404, "not_found", "attachment not found")
path = f"/api/v1/chats/{chat_id}/messages"
body_hash = hashlib.sha256(
json.dumps(body.model_dump(), sort_keys=True, default=str).encode()
).hexdigest()
existing = db.get(IdempotencyKey, (user.id, "POST", path, key))
if existing:
if existing.request_hash != body_hash:
raise ApiError(409, "conflict", "idempotency key reused with different body")
return json.loads(existing.response_json)
m = Message(chat_id=chat_id, sender_id=user.id, text=body.text)
db.add(m)
db.flush() # чтобы появился m.id
for i, aid in enumerate(body.attachment_ids):
db.add(MessageAttachment(message_id=m.id, attachment_id=aid, position=i))
db.flush() # чтобы _msg_out увидел MessageAttachment
out = _msg_out(db, m)
db.add(
IdempotencyKey(
user_id=user.id,
method="POST",
path=path,
key=key,
request_hash=body_hash,
response_json=json.dumps(out.model_dump(), default=str),
)
)
db.commit()
return out
@router.get("", response_model=MessagePage)
def list_messages(
chat_id: uuid.UUID,
limit: int = 50,
cursor: str | None = None,
db: Session = Depends(get_db),
user: User = Depends(current_user),
):
_require_member(db, chat_id, user.id)
if limit < 1 or limit > 100:
raise ApiError(400, "invalid_request", "bad limit")
q = (
db.query(Message)
.filter(Message.chat_id == chat_id)
.order_by(Message.created_at, Message.id)
)
if cursor:
try:
c = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
ts = datetime.fromisoformat(c["t"])
mid = uuid.UUID(c["i"])
q = q.filter(
(Message.created_at > ts)
| ((Message.created_at == ts) & (Message.id > mid))
)
except Exception:
raise ApiError(400, "invalid_request", "bad cursor")
rows = q.limit(limit + 1).all()
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = None
if has_more:
last = rows[-1]
payload = json.dumps({"t": last.created_at.isoformat(), "i": str(last.id)}).encode()
next_cursor = base64.urlsafe_b64encode(payload).decode()
return MessagePage(items=[_msg_out(db, m) for m in rows], next_cursor=next_cursor)
+41
View File
@@ -0,0 +1,41 @@
import base64
import hashlib
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import User, Session as DbSession
from ..schemas import SessionOut
from ..errors import ApiError
from ..auth import verify_password, new_session, current_user
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
_BASIC = 'Basic realm="messenger", charset="UTF-8"'
_BEARER = 'Bearer realm="messenger"'
@router.post("", status_code=201, response_model=SessionOut)
def login(request: Request, db: Session = Depends(get_db)):
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("basic "):
raise ApiError(401, "unauthorized", "basic required", challenge=_BASIC)
try:
raw = base64.b64decode(auth[6:].strip()).decode("utf-8")
username, password = raw.split(":", 1)
except Exception:
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
user = db.query(User).filter(User.username == username).first()
if not user or not verify_password(user.password_hash, password):
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
token, expires = new_session(db, user)
return {"token": token, "token_type": "Bearer", "expires_at": expires, "user": user}
@router.delete("/current", status_code=204)
def logout(request: Request, db: Session = Depends(get_db), user: User = Depends(current_user)):
auth = request.headers.get("authorization", "")
token = auth[7:].strip()
h = hashlib.sha256(token.encode()).hexdigest()
s = db.get(DbSession, h)
if s:
db.delete(s)
db.commit()
return None
+27
View File
@@ -0,0 +1,27 @@
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from sqlalchemy.exc import IntegrityError
from ..db import get_db
from ..models import User
from ..schemas import CreateUserIn, UserOut
from ..errors import ApiError
from ..auth import hash_password, current_user
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", status_code=201, response_model=UserOut)
def register(body: CreateUserIn, db: Session = Depends(get_db)):
u = User(username=body.username, display_name=body.display_name,
password_hash=hash_password(body.password))
db.add(u)
try:
db.commit()
except IntegrityError:
db.rollback()
raise ApiError(409, "conflict", "username taken")
db.refresh(u)
return u
@router.get("/me", response_model=UserOut)
def me(user: User = Depends(current_user)):
return user
+36
View File
@@ -0,0 +1,36 @@
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError
from . import config
_client = None
def client():
global _client
if _client is None:
_client = boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY,
aws_secret_access_key=config.S3_SECRET_KEY,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
retries={"max_attempts": 2, "mode": "standard"},
connect_timeout=3,
read_timeout=10,
),
)
return _client
def ensure_bucket():
c = client()
try:
c.head_bucket(Bucket=config.S3_BUCKET)
except ClientError as e:
code = e.response.get("Error", {}).get("Code", "")
if code in ("404", "NoSuchBucket", "NotFound"):
c.create_bucket(Bucket=config.S3_BUCKET)
else:
raise
+76
View File
@@ -0,0 +1,76 @@
import uuid
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field, ConfigDict
class UserOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: uuid.UUID
username: str
display_name: str
created_at: datetime
class CreateUserIn(BaseModel):
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
class SessionOut(BaseModel):
token: str
token_type: str = "Bearer"
expires_at: datetime
user: UserOut
class ChatOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: uuid.UUID
title: str
member_ids: list[uuid.UUID]
created_at: datetime
class CreateChatIn(BaseModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
class MessageOut(BaseModel):
id: uuid.UUID
chat_id: uuid.UUID
sender_id: uuid.UUID
text: str
created_at: datetime
attachment_ids: list[uuid.UUID]
class CreateMessageIn(BaseModel):
text: str = Field(default="", max_length=4000)
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
class MessagePage(BaseModel):
items: list[MessageOut]
next_cursor: Optional[str] = None
class ChatPage(BaseModel):
items: list[ChatOut]
next_cursor: Optional[str] = None
class VariantOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class AttachmentOut(BaseModel):
id: uuid.UUID
chat_id: uuid.UUID
owner_id: uuid.UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str = "ready"
variants: list[VariantOut] = []
error_code: Optional[str] = None
created_at: datetime
updated_at: datetime
+124
View File
@@ -0,0 +1,124 @@
"""OpenTelemetry wiring for API and worker (lab6).
Signals go OTLP -> collector. Metrics are exposed by the collector as a
Prometheus endpoint, traces are stored in Tempo, logs in Loki. The JSON logs
kept in stdout also carry trace_id/span_id so a trace can be matched with a
line even without the log backend.
"""
import json
import logging
import os
from datetime import datetime, timezone
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import (
Resource,
SERVICE_INSTANCE_ID,
SERVICE_NAME,
SERVICE_VERSION,
)
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.propagate import set_global_textmap
class TraceJsonFormatter(logging.Formatter):
"""One JSON object per line, with trace correlation and no bodies."""
def __init__(self, service_name: str):
super().__init__()
self.service_name = service_name
def format(self, record: logging.LogRecord) -> str:
ctx = trace.get_current_span().get_span_context()
payload = {
"timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"severity": record.levelname,
"service.name": self.service_name,
"event": record.getMessage(),
"logger": record.name,
}
if ctx and ctx.is_valid:
payload["trace_id"] = format(ctx.trace_id, "032x")
payload["span_id"] = format(ctx.span_id, "016x")
extra = getattr(record, "event_fields", None)
if isinstance(extra, dict):
payload.update({k: v for k, v in extra.items() if not isinstance(v, (bytes, bytearray))})
elif isinstance(extra, str):
try:
payload.update({k: v for k, v in json.loads(extra).items()})
except ValueError:
pass
if record.exc_info and record.exc_info[0]:
payload["error.type"] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False)
def setup_telemetry(service_name: str, instance_id: str):
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
resource = Resource.create({
SERVICE_NAME: service_name,
SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "6.0.0"),
SERVICE_INSTANCE_ID: instance_id,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
})
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))
)
trace.set_tracer_provider(tracer_provider)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint, insecure=True),
export_interval_millis=int(os.environ.get("OTEL_METRIC_INTERVAL_MS", "10000")),
)],
)
metrics.set_meter_provider(meter_provider)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint, insecure=True))
)
root = logging.getLogger()
root.setLevel(logging.INFO)
for handler in list(root.handlers):
root.removeHandler(handler)
stream = logging.StreamHandler()
stream.setFormatter(TraceJsonFormatter(service_name))
root.addHandler(stream)
root.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logger_provider))
set_global_textmap(TraceContextTextMapPropagator())
return tracer_provider, meter_provider
def http_instruments():
meter = metrics.get_meter("messenger.http")
return (
meter.create_counter("http.server.requests", unit="1",
description="HTTP requests by method, route template and status"),
meter.create_histogram("http.server.request.duration", unit="s",
description="HTTP server request duration"),
)
def job_instruments():
meter = metrics.get_meter("messenger.worker")
return (
meter.create_counter("attachment.jobs", unit="1",
description="Attachment jobs by outcome"),
meter.create_histogram("attachment.job.duration", unit="s",
description="Attachment job duration by outcome"),
)
+38
View File
@@ -0,0 +1,38 @@
"""Deterministic image variants (lab5: one thumbnail per the contract)."""
import hashlib
import io
from PIL import Image, ImageOps
from . import config
def _flatten_on_white(img: Image.Image) -> Image.Image:
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
return Image.alpha_composite(background, rgba).convert("RGB")
return img.convert("RGB")
def make_thumbnail(data: bytes) -> tuple[bytes, int, int, str]:
"""Fit the image into 256x256 (no upscaling), JPEG on white.
Returns (jpeg_bytes, width, height, sha256_hex).
"""
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
img = _flatten_on_white(img)
limit = config.THUMBNAIL_SIZE
width, height = img.size
scale = min(limit / width, limit / height, 1.0)
new_width = max(1, int(width * scale))
new_height = max(1, int(height * scale))
if (new_width, new_height) != (width, height):
img = img.resize((new_width, new_height), Image.LANCZOS)
out = io.BytesIO()
img.save(out, "JPEG", quality=config.THUMBNAIL_QUALITY, optimize=True)
payload = out.getvalue()
return payload, new_width, new_height, hashlib.sha256(payload).hexdigest()
+227
View File
@@ -0,0 +1,227 @@
"""Durable attachment worker (lab5/lab6).
Consumes ``attachment.process.v1`` messages from a persistent queue, performs
the heavy image work outside the HTTP path, records the result atomically in
PostgreSQL, and emits OTLP traces/metrics/logs. Delivery is at-least-once; the
effect is idempotent by ``(attachment_id, pipeline_version, variant)`` and
every state transition is committed before the broker ack.
"""
import json
import logging
import signal
import time
import uuid
import pika
from opentelemetry import trace
from opentelemetry.propagate import extract
from opentelemetry.trace import SpanKind, Status, StatusCode
from sqlalchemy import select
from . import config, s3, telemetry
from .db import SessionLocal
from .errors import ApiError
from .images import validate_image
from .models import Attachment, AttachmentJob, AttachmentVariant
from .thumbnails import make_thumbnail
_, _meter_provider = telemetry.setup_telemetry("messenger-worker", "worker")
job_counter, job_duration = telemetry.job_instruments()
tracer = trace.get_tracer("messenger.worker")
log = logging.getLogger("messenger.worker")
JOB_TYPE = "attachment.process.v1"
_stop = False
def _handle_signal(signum, frame):
global _stop
_stop = True
log.info("worker.stop_requested")
def _failure_code(exc: Exception) -> tuple[str, bool]:
"""Return (safe error_code, permanent)."""
if isinstance(exc, ApiError):
if exc.status in (400, 413, 415):
return "invalid_image", True
return "processing_failed", False
if isinstance(exc, (OSError, ValueError)):
return "invalid_image", True
return "processing_failed", False
def _set_state(job_id: uuid.UUID, attachment_id: uuid.UUID, job_state: str,
att_state: str | None = None, attempt: int | None = None,
error: str | None = None) -> None:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if not job:
return
job.state = job_state
if attempt is not None:
job.attempt = max(job.attempt or 0, attempt)
if error is not None:
job.last_error = error[:255]
att = db.get(Attachment, attachment_id)
if att and att_state:
att.status = att_state
att.error_code = error if att_state == "failed" else None
db.commit()
def process(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> str:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
att = db.get(Attachment, attachment_id)
if not job or not att:
log.warning("job.dropped", extra={"event_fields": json.dumps({"reason": "row_gone"})})
return "dropped"
if att.status == "ready":
job.state = "done"
db.commit()
log.info("job.duplicate", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
return "duplicate"
if att.status == "failed":
job.state = "failed"
db.commit()
log.info("job.duplicate_failed", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
return "skipped"
job.state = "processing"
job.attempt = max(job.attempt or 0, attempt)
att.status = "processing"
db.commit()
pipeline_version = job.pipeline_version
staging_key = att.staging_key or f"{config.STAGING_PREFIX}/{attachment_id}"
final_key = att.s3_key
try:
with SessionLocal() as db:
att = db.get(Attachment, attachment_id)
declared_type = att.content_type if att else None
data = s3.client().get_object(Bucket=config.S3_BUCKET, Key=staging_key)["Body"].read()
ctype, _w, _h, _size = validate_image(data)
s3.client().put_object(Bucket=config.S3_BUCKET, Key=final_key, Body=data,
ContentType=declared_type or ctype)
thumb, tw, th, tsha = make_thumbnail(data)
thumb_key = f"variants/{attachment_id}/thumbnail"
s3.client().put_object(Bucket=config.S3_BUCKET, Key=thumb_key, Body=thumb, ContentType="image/jpeg")
with SessionLocal() as db:
att = db.get(Attachment, attachment_id)
job = db.get(AttachmentJob, job_id)
existing = db.get(AttachmentVariant, (attachment_id, "thumbnail"))
if existing is None:
db.add(AttachmentVariant(
attachment_id=attachment_id, name="thumbnail",
content_type="image/jpeg", size_bytes=len(thumb),
width=tw, height=th, sha256=tsha, s3_key=thumb_key,
))
elif existing.s3_key != thumb_key:
log.warning("variant.kept", extra={"event_fields": json.dumps({"reason": "newer_pipeline"})})
att.status = "ready"
att.error_code = None
job.state = "done"
job.last_error = None
job.attempt = max(job.attempt or 0, attempt)
db.commit()
log.info("job.ready", extra={"event_fields": json.dumps({
"attachment_id": str(attachment_id),
"pipeline_version": pipeline_version,
"attempt": attempt,
})})
try:
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
except Exception:
pass
return "ready"
except Exception as exc:
code, permanent = _failure_code(exc)
log.exception("job.failed", extra={"event_fields": json.dumps({
"attachment_id": str(attachment_id),
"attempt": attempt,
"error_code": code,
})})
if permanent or attempt >= config.MAX_JOB_ATTEMPTS:
_set_state(job_id, attachment_id, "failed", att_state="failed",
attempt=attempt, error=code)
return "failed"
_set_state(job_id, attachment_id, "queued", att_state="queued",
attempt=attempt, error=code)
raise
def handle(channel, method, properties, body):
outcome = "retry"
try:
message = json.loads(body)
job_id = uuid.UUID(message["job_id"])
attachment_id = uuid.UUID(message["attachment_id"])
attempt = int(message.get("attempt", 1))
except Exception as exc:
log.warning("job.poison", extra={"event_fields": json.dumps({"reason": type(exc).__name__})})
channel.basic_ack(delivery_tag=method.delivery_tag)
return
context = extract(message)
started = time.perf_counter()
with tracer.start_as_current_span("attachment.process", context=context,
kind=SpanKind.CONSUMER) as span:
span.set_attribute("job.type", JOB_TYPE)
span.set_attribute("attachment.id", str(attachment_id))
span.set_attribute("job.attempt", attempt)
try:
outcome = process(job_id, attachment_id, attempt)
span.set_attribute("job.outcome", outcome)
if outcome == "failed":
span.set_status(Status(StatusCode.ERROR))
channel.basic_ack(delivery_tag=method.delivery_tag)
except Exception as exc:
outcome = "retry"
span.set_attribute("job.outcome", outcome)
span.set_status(Status(StatusCode.ERROR))
span.record_exception(exc)
from .jobs import build_payload, publish
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
payload = build_payload(job, job.attempt + 1) if job and job.state == "queued" else None
if payload:
try:
publish(payload)
except Exception:
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
outcome = "requeued"
return
channel.basic_ack(delivery_tag=method.delivery_tag)
finally:
attributes = {"job.type": JOB_TYPE, "job.outcome": outcome}
job_counter.add(1, attributes)
job_duration.record(time.perf_counter() - started, attributes)
def main():
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
params = pika.URLParameters(config.BROKER_URL)
params.heartbeat = 30
while not _stop:
try:
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=config.QUEUE_NAME, on_message_callback=handle, auto_ack=False)
log.info("worker.consuming", extra={"event_fields": json.dumps({"queue": config.QUEUE_NAME})})
while not _stop:
connection.process_data_events(time_limit=1)
connection.close()
except Exception as exc:
log.warning("broker.reconnect", extra={"event_fields": json.dumps({"error": type(exc).__name__})})
time.sleep(3)
log.info("worker.stopped")
if __name__ == "__main__":
main()
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, Failure, LAB, arguments, check, run
def history(c, actor, chat):
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
result = []
seen = set()
for _ in range(1000):
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
result.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return result
check(cursor not in seen,'cursor cycle')
seen.add(cursor)
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
raise Failure('history pagination exceeded safety bound')
def main():
def configure(p):
p.add_argument('--mode', choices=['persistence','failover'], required=True)
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
check(LAB>=2,'persistence requires lab2+')
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
check('.example.' not in script.name,'copy and implement the example hook first')
c = Client(args.base_url,args.ca_file,timeout=10)
alice,bob,eve,chat = c.bootstrap()
sent = c.message(alice,chat,'Сохранить при отказе')
original = sent['json']
target = sent['headers'].get('X-Instance-Id','')
attachment = content = None
if LAB>=4:
attachment,content = c.attachment(alice,chat)
if LAB>=5:
attachment = c.wait_ready(alice,attachment)
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
observed = set()
if args.mode=='failover':
for _ in range(40):
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
# No shell interpolation and no Docker commands in this test.
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
started = time.monotonic()
successes,failures,after_instances = [],0,[]
try:
if args.mode=='persistence':
try:
code = proc.wait(timeout=90)
except subprocess.TimeoutExpired:
raise Failure('restart hook exceeded 90 seconds') from None
check(code==0,'restart hook failed (inspect your script locally)')
deadline = time.monotonic()+90
while True:
try:
c.call('GET','/health/ready',schema='Health')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
break
except Failure:
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
time.sleep(0.5)
else:
first_recovery = None
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
while time.monotonic()-started < 15:
code = proc.poll()
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
try:
response = c.message(alice,chat,text,key=key)
msg = response['json']
if code==0:
if first_recovery is None:
first_recovery = time.monotonic()-started
after_instances.append(response['headers']['X-Instance-Id'])
successes.append(msg)
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
except Failure:
failures += 1
# Keep SAME key/body after an uncertain POST result.
time.sleep(0.1)
check(proc.poll()==0,'stop-one hook did not finish successfully')
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
check(len(after_instances)>=5,'too few successful writes after stop')
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
# Existing token, no re-login. Read all confirmations and reject duplicates.
messages = history(c,bob,chat)
ids = [m['id'] for m in messages]
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
for msg in successes:
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
texts = [m['text'] for m in messages]
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
if attachment:
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
check(downloaded['bytes']==content,'attachment lost or changed across failure')
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
finally:
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
if __name__=='__main__':
run(main)