ЛР 6 — БВТ2302 — Вотчицева — на 4 #39
+7
-26
@@ -1,30 +1,11 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
POSTGRES_PASSWORD=lab6-local-postgres-change-me
|
||||
S3_ROOT_USER=lab6root
|
||||
S3_ROOT_PASSWORD=lab6-local-s3-admin-change-me
|
||||
S3_ACCESS_KEY_ID=lab6api
|
||||
S3_SECRET_ACCESS_KEY=lab6-local-s3-api-change-me
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
SESSION_TTL_SECONDS=3600
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
GRAFANA_ADMIN_PASSWORD=lab6-local-grafana-change-me
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
# Pinned base image. Use a digest for reproducibility if you have one available:
|
||||
# FROM python:3.12.7-slim-bookworm@sha256:<digest>
|
||||
FROM python:3.12.7-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY src/ ./src/
|
||||
|
||||
RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \
|
||||
&& chown -R appuser:appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=2s --timeout=3s --start-period=5s --retries=2 \
|
||||
CMD python -c "import sys,urllib.request; \
|
||||
sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"
|
||||
|
||||
CMD ["python", "-m", "src"]
|
||||
@@ -1,37 +1,80 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Вотчицева Вероника, БВТ2302
|
||||
- **Целевая оценка:** 4 (трек на 5 не заявлялся).
|
||||
- **Commit/PR:**
|
||||
- **Использованные источники и ИИ:**
|
||||
- Python 3.12, FastAPI, SQLAlchemy, RabbitMQ (aio-pika), SeaweedFS (boto3).
|
||||
- OpenTelemetry SDK (api, sdk, exporter-otlp-proto-http).
|
||||
- Стек мониторинга: OTel Collector, Prometheus, Loki, Tempo, Grafana.
|
||||
- При подготовке конфигураций и кода использовался ассистент (Claude).
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Prerequisites:** Docker Desktop с поддержкой `--wait`, Python 3.9+ для скриптов проверки.
|
||||
|
||||
```powershell
|
||||
# 1. Подготовка окружения
|
||||
Copy-Item .env.example .env
|
||||
# (Заполнить пароли POSTGRES_PASSWORD и GRAFANA_ADMIN_PASSWORD)
|
||||
|
||||
# 2. Запуск всего стека (включая Grafana, Prometheus, Loki, Tempo)
|
||||
docker compose up -d --build --wait --wait-timeout 180
|
||||
|
||||
# 3. Проверка API
|
||||
$env:PYTHONUTF8="1"
|
||||
python tests/smoke.py
|
||||
|
||||
# 4. Запуск нагрузки (для появления графиков)
|
||||
python tests/load.py --duration 60 --concurrency 4 --output evidence/load-lab6.json
|
||||
```
|
||||
|
||||
**Остановка:** `docker compose down`. **Удаление данных (включая дашборды и базы):** `docker compose down -v`.
|
||||
|
||||
Интерфейсы:
|
||||
- API (Swagger): `http://localhost:8080/docs`
|
||||
- Grafana: `http://localhost:3000` (admin / пароль из `.env`)
|
||||
- Prometheus: `http://localhost:9090`
|
||||
- Jaeger/Tempo UI: `http://localhost:16686` (если настроен порт)
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
В систему добавлен полный стек наблюдаемости (Observability) на базе OpenTelemetry. Схема сбора данных: `API/Worker → OTLP → OTel Collector → (Prometheus, Loki, Tempo) → Grafana`.
|
||||
|
||||
**Три сигнала (Three Pillars):**
|
||||
1. **Метрики (Prometheus):** Собираются через OTel Collector. Настроены дашборды для RPS, p50/p95/p99, доли ошибок, глубины очереди (Queue Depth) и возраста старейшей задачи.
|
||||
2. **Логи (Loki):** Отправляются напрямую через OTLP HTTP. Логи структурированные (JSON), содержат `trace_id` и `span_id` для связки с трассами.
|
||||
3. **Трассы (Tempo):** Позволяют отследить путь запроса. Реализована передача контекста W3C Trace Context (`traceparent`) через RabbitMQ. API создает задачу, сохраняет контекст в Outbox, Worker извлекает его и продолжает трассу.
|
||||
|
||||
**Защита от перегрузки:**
|
||||
В конфигурации OTel Collector настроен `memory_limiter` (лимит 96 MiB). В SDK настроены ограниченные очереди и таймауты. Если Tempo или Loki упадут, это не заблокирует основной поток обработки API.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `python tests/smoke.py` | PASS | — |
|
||||
| Импортируемый Dashboard | Grafana → MTUCI Lab 6 | Все панели отрисовываются | `dashboard.json` в Git |
|
||||
| Сбор метрик | `tests/load.py` | RPS и Latency видны в Prometheus | Скриншот Grafana |
|
||||
| Связь Trace → Log | Tempo → клик по Trace ID | Переход в Loki с тем же trace_id | Скриншот Tempo/Loki |
|
||||
| Сквозная трассировка | Tempo: `messenger-api` → `messenger-worker` | Единый трейс от HTTP до записи в S3 | — |
|
||||
| Алерт на ошибки 5xx | Остановка БД под нагрузкой | RPS падает, HAProxy отдает 503 | Скриншот Prometheus |
|
||||
| Алерт на Queue Lag | `incident_worker.py --hold 80` | Загорается `MessengerQueueLagGrowing` | Скриншот Prometheus |
|
||||
| Отказ телеметрии | `stop loki tempo` + `smoke.py` | API продолжает работать (PASS) | — |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Инцидент 1: Остановка воркера (Queue Lag).**
|
||||
Скрипт `scripts/incident_worker.py --hold 80` остановил контейнер `worker` и загрузил тестовый PNG. API вернул `202 Accepted` (статус `queued`), файл застрял в очереди. Через 35 секунд в Prometheus сработал алерт `MessengerQueueLagGrowing` (возраст старейшей задачи превысил 20 секунд, удерживался >15с). В Grafana на графике «Очередь» наблюдался рост. После автоматического запуска воркера задача обработалась, статус сменился на `ready`, алерт перешел в `resolved` (Inactive).
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
**Инцидент 2: Отказ PostgreSQL.**
|
||||
Во время нагрузочного теста я остановила базу данных (`docker compose stop db`). Эндпоинт `/health/ready` начал возвращать `503`, из-за чего **HAProxy исключил реплики API из балансировки** и начал отдавать свой `503` на уровне LB. Из-за этого ошибки 5xx не попали в метрику API (ограничение архитектуры SLI), однако в Grafana график RPS упал до нуля. После команды `docker compose start db` сервис автоматически восстановился.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
**Инцидент 3: Отказ Telemetry Backend.**
|
||||
При остановке контейнеров `loki` и `tempo` бизнес-логика приложения не пострадала. Публичный тест `smoke.py` успешно прошел (PASS). В Grafana перестали появляться новые логи и трассы, так как Collector не мог их экспортировать. Это доказывает, что падение системы мониторинга не ломает основной сервис (graceful degradation телеметрии).
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Ограничения
|
||||
|
||||
- **Single-node:** Prometheus, Loki, Tempo и Grafana запущены в одном экземпляре. При падении всего Docker-хоста данные мониторинга будут потеряны (хотя volumes сохранят историю при перезапуске).
|
||||
- **Ограничение метрик HAProxy:** Так как HAProxy отсекает трафик при падении API, ошибки 5xx не всегда попадают в метрики FastAPI. Для полной картины нужен экспортер метрик самого HAProxy (например, `haproxy_exporter`).
|
||||
- **Кардинальность:** В метках (labels) нет пользовательских UUID, поэтому кардинальность метрик низкая. Все детали запросов (кто именно делал) ищутся через логи и трассы.
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
x-api: &api
|
||||
build: .
|
||||
image: mtusi-lab5-votchitseva-api:local
|
||||
environment: &api_env
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: amqp://guest:guest@broker:5672/
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
networks: [front, data]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
stop_grace_period: 15s
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4).close()"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 16
|
||||
start_period: 35s
|
||||
|
||||
services:
|
||||
api1:
|
||||
<<: *api
|
||||
api2:
|
||||
<<: *api
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
command: ["python", "-m", "src.worker"]
|
||||
environment:
|
||||
<<: *api_env
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
replicas: 1
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13-alpine
|
||||
hostname: rabbitmq
|
||||
entrypoint: ["/bin/sh", "-c", "rm -f /var/lib/rabbitmq/.erlang.cookie && exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server"]
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: guest
|
||||
RABBITMQ_DEFAULT_PASS: guest
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
networks: [data]
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
|
||||
s3_config:
|
||||
image: python:3.12.7-slim-bookworm
|
||||
command: ["python", "/bootstrap/generate-s3-config.py"]
|
||||
environment:
|
||||
S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER in .env}
|
||||
S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD in .env}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro
|
||||
- s3_config_data:/s3-config
|
||||
networks: [data]
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.35
|
||||
command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"]
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- s3_config_data:/s3-config:ro
|
||||
depends_on:
|
||||
s3_config: {condition: service_completed_successfully}
|
||||
networks: [data]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8333/status || exit 1"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-lab5-votchitseva-api:local
|
||||
command: ["python", "-m", "src.migrate"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
networks: [data]
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
ports: ["127.0.0.1:8080:8080"]
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
networks: [front]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
db:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Copy .env.example to .env}
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
networks: [data]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U user -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
volumes: [redisdata:/data]
|
||||
networks: [data]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.138.0
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
networks: [data]
|
||||
restart: unless-stopped
|
||||
mem_limit: 192m
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.6.0
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.enable-lifecycle"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports: ["127.0.0.1:9090:9090"]
|
||||
networks: [data, front]
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.5.5
|
||||
user: "0"
|
||||
command: ["-config.file=/etc/loki/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/loki/config.yaml:/etc/loki/config.yaml:ro
|
||||
- loki_data:/loki
|
||||
networks: [data]
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.8.2
|
||||
user: "0"
|
||||
command: ["-config.file=/etc/tempo/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/tempo/config.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
networks: [data]
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.2.0
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-lab6-local-change-me}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports: ["127.0.0.1:3000:3000"]
|
||||
networks: [data, front]
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
s3_data:
|
||||
s3_config_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
broker_data:
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 31.27,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 60,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1.919,
|
||||
"successful_latency_ms": {
|
||||
"p50": 2063.889,
|
||||
"p95": 2124.196,
|
||||
"p99": 2185.684
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 7.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 29 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.0 KiB |
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"scenario": "stop worker -> 202 queued -> restart -> ready",
|
||||
"attachment_id": "792f6182-62e6-4f06-9b9b-7c4830ff8fcf",
|
||||
"status": "ready",
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"queue_alert_before_worker_start": "firing",
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Bootstraps a private S3 identity configuration from non-committed .env.
|
||||
|
||||
The server reads it from a shared internal-only configuration volume.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately"
|
||||
config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "lab4-admin",
|
||||
"credentials": [{"accessKey": os.environ["S3_ROOT_USER"],
|
||||
"secretKey": os.environ["S3_ROOT_PASSWORD"]}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"],
|
||||
},
|
||||
{
|
||||
"name": "lab4-api",
|
||||
"credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"],
|
||||
"secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"],
|
||||
},
|
||||
],
|
||||
}
|
||||
target = Path("/s3-config/s3.json")
|
||||
target.write_text(json.dumps(config, indent=2), encoding="utf-8")
|
||||
target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images.
|
||||
print("PASS: private S3 identity config generated; credentials not logged")
|
||||
@@ -0,0 +1,38 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 512
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
timeout connect 3s
|
||||
timeout client 75s
|
||||
timeout server 75s
|
||||
timeout check 2s
|
||||
retries 1
|
||||
option redispatch
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
|
||||
server api1 api1:8080
|
||||
server api2 api2:8080
|
||||
@@ -0,0 +1,62 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 2s
|
||||
limit_mib: 96
|
||||
spike_limit_mib: 24
|
||||
batch:
|
||||
timeout: 3s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
service:
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger Lab 6
|
||||
orgId: 1
|
||||
folder: MTUCI
|
||||
type: file
|
||||
disableDeletion: false
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards/json
|
||||
@@ -0,0 +1,481 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "mtuci-votchitseva-lab6",
|
||||
"title": "МТУСИ · ЛР6 · Мессенджер",
|
||||
"description": "API + RabbitMQ worker, OpenTelemetry Collector, Prometheus, Loki and Tempo. All metric labels are low-cardinality.",
|
||||
"tags": [
|
||||
"mtuci",
|
||||
"lab6",
|
||||
"observability"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 40,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API — запросов в секунду",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!=\"/health\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "req/s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API — доля ошибок 5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!=\"/health\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!=\"/health\"}[1m])), 0.000001)",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.50, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket{route!=\"/health\"}[5m])))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Очередь — ожидают обработки",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Очередь — возраст старейшей",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_age_seconds)",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Worker — jobs/sec",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(outcome)(rate(messenger_jobs_total[1m]))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 24
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Worker — p95 обработки",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_job_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Время до ready — p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool — занято",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_db_pool_used{service_name=\"messenger-api\"})",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 40
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Worker — загрузка",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_worker_busy{service_name=\"messenger-worker\"}) / clamp_min(max(messenger_worker_capacity{service_name=\"messenger-worker\"}), 1)",
|
||||
"legendFormat": "{{service_name}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 40
|
||||
}
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0,211,255,1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace_id=([0-9a-f]{32})'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
@@ -0,0 +1,27 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,23 @@
|
||||
groups:
|
||||
- name: lab6-signals
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: |
|
||||
(sum(rate(messenger_http_requests_total{route!="/health",status_class="5xx"}[1m]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!="/health"}[1m])), 0.000001)) > 0.10
|
||||
and sum(rate(messenger_http_requests_total{route!="/health"}[1m])) > 0.05
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: '5xx responses exceed 10% of business requests'
|
||||
runbook: 'Check API error rate, Loki errors, Tempo DB spans and postgres health'
|
||||
- alert: MessengerQueueLagGrowing
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 20
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 'Oldest queued attachment is older than 20 seconds'
|
||||
runbook: 'Check worker status, RabbitMQ queue and job failure spans in Tempo'
|
||||
@@ -0,0 +1,12 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
scrape_configs:
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
@@ -0,0 +1,18 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
@@ -0,0 +1,14 @@
|
||||
fastapi==0.115.4
|
||||
uvicorn==0.32.0
|
||||
sqlalchemy==2.0.36
|
||||
asyncpg==0.30.0
|
||||
redis==5.2.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
bcrypt==4.0.1
|
||||
python-dotenv==1.0.1
|
||||
boto3==1.40.67
|
||||
Pillow==11.3.0
|
||||
aio-pika==9.4.3
|
||||
opentelemetry-api==1.39.1
|
||||
opentelemetry-sdk==1.39.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.39.1
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Convenience runner. Does not fabricate evidence or control Docker itself."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
for command in (
|
||||
[sys.executable, str(ROOT / "tests/smoke.py")],
|
||||
[sys.executable, str(ROOT / "student-tests/run.py")],
|
||||
[sys.executable, str(ROOT / "student-tests/test_attachments.py")],
|
||||
):
|
||||
print("RUN:", " ".join(command), flush=True)
|
||||
subprocess.run(command, cwd=ROOT, check=True)
|
||||
print("PASS: public smoke + lab3 concurrency + grade4 negative cases; run extra resilience and memory experiment separately")
|
||||
@@ -0,0 +1,71 @@
|
||||
"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics)."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check, arguments, run
|
||||
|
||||
|
||||
def mib(value):
|
||||
value = value.strip()
|
||||
for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)):
|
||||
if value.endswith(suffix):
|
||||
return float(value[:-len(suffix)]) * multiplier
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Concurrent uploads + observed API container memory')
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
file = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
results, memory = [], []
|
||||
stop = threading.Event()
|
||||
def observe():
|
||||
while not stop.is_set():
|
||||
r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0:
|
||||
for line in r.stdout.splitlines():
|
||||
if '|api1-' in line or '|api2-' in line:
|
||||
pass
|
||||
parts = line.split('|')
|
||||
if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]):
|
||||
amount = mib(parts[1].split('/')[0])
|
||||
if amount is not None:
|
||||
memory.append({'container': parts[0], 'mem_mib': round(amount,3)})
|
||||
stop.wait(0.2)
|
||||
monitor = threading.Thread(target=observe, daemon=True)
|
||||
monitor.start()
|
||||
try:
|
||||
def worker(i):
|
||||
local = Client(args.base_url,args.ca_file,timeout=30)
|
||||
start=time.perf_counter()
|
||||
resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png',
|
||||
202, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment')
|
||||
return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures=[executor.submit(worker,i) for i in range(10)]
|
||||
results=[f.result() for f in concurrent.futures.as_completed(futures)]
|
||||
finally:
|
||||
stop.set(); monitor.join(timeout=12)
|
||||
output={'concurrency':10,'file_bytes':len(file),'completed':len(results),
|
||||
'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2),
|
||||
'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None),
|
||||
'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))}
|
||||
target=ROOT / 'evidence' / 'uploads-10.json'
|
||||
target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
check(len(results)==10, 'not all uploads completed')
|
||||
print('PASS: 10 parallel uploads; metrics written:',target)
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
if not memory:
|
||||
print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment')
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,95 @@
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import secrets
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
BASE = 'http://localhost:8080'
|
||||
|
||||
|
||||
def call(method, url, body=None, headers=None):
|
||||
request = urllib.request.Request(url, data=body, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
raw = response.read()
|
||||
return response.status, json.loads(raw) if raw else {}
|
||||
except urllib.error.HTTPError as error:
|
||||
raw = error.read()
|
||||
return error.code, json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def json_request(method, path, data, auth=None):
|
||||
headers = {'Content-Type':'application/json'}
|
||||
if auth:
|
||||
headers['Authorization'] = 'Bearer '+auth
|
||||
return call(method, BASE+path, json.dumps(data).encode(), headers)
|
||||
|
||||
|
||||
def docker(*args):
|
||||
subprocess.run(['docker', 'compose', *args], check=True)
|
||||
|
||||
|
||||
def alerts():
|
||||
try:
|
||||
code, result = call('GET', 'http://localhost:9090/api/v1/alerts')
|
||||
if code == 200:
|
||||
return {a['labels'].get('alertname'):a['state'] for a in result['data']['alerts']}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def main():
|
||||
cli = argparse.ArgumentParser()
|
||||
cli.add_argument('--hold', type=int, default=50)
|
||||
opts = cli.parse_args()
|
||||
username = 'obs'+secrets.token_hex(6)
|
||||
password = secrets.token_urlsafe(24)
|
||||
status, user = json_request('POST','/api/v1/users',{'username':username,'password':password,'display_name':'Lab6 Test'})
|
||||
assert status == 201, user
|
||||
basic = base64.b64encode((username+':'+password).encode()).decode()
|
||||
status, auth = call('POST',BASE+'/api/v1/auth/sessions',b'',{'Authorization':'Basic '+basic})
|
||||
assert status == 201, auth
|
||||
token = auth['token']
|
||||
status, chat = json_request('POST','/api/v1/chats',{'title':'Lab6 Incident','member_ids':[user['id']]},token)
|
||||
assert status == 201, chat
|
||||
docker('stop','worker')
|
||||
ready = False
|
||||
try:
|
||||
sample = Path('tests/fixtures/sample.png').read_bytes()
|
||||
status, attachment = call('POST', BASE+f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
|
||||
sample, {'Content-Type':'image/png','Authorization':'Bearer '+token})
|
||||
assert status == 202 and attachment['status'] == 'queued', (status,attachment)
|
||||
attachment_id = attachment['id']
|
||||
status, _ = call('GET',BASE+f'/api/v1/attachments/{attachment_id}/content',None,{'Authorization':'Bearer '+token})
|
||||
assert status == 409, status
|
||||
print('202 queued; content 409; worker stopped')
|
||||
time.sleep(opts.hold)
|
||||
before = alerts().get('MessengerQueueLagGrowing','not_observed')
|
||||
print('QueueLag alert:',before)
|
||||
finally:
|
||||
docker('start','worker')
|
||||
deadline = time.monotonic()+100
|
||||
while time.monotonic()<deadline:
|
||||
status, attachment = call('GET',BASE+f'/api/v1/attachments/{attachment_id}',None,{'Authorization':'Bearer '+token})
|
||||
if status == 200 and attachment['status'] == 'ready':
|
||||
ready = True
|
||||
break
|
||||
time.sleep(2)
|
||||
assert ready, 'Attachment did not become ready after restoring worker'
|
||||
names = sorted(v['name'] for v in attachment['variants'])
|
||||
assert names == ['crop','optimized','thumbnail'], names
|
||||
result = {'scenario':'stop worker -> 202 queued -> restart -> ready',
|
||||
'attachment_id':attachment_id,'status':'ready','variants':names,
|
||||
'queue_alert_before_worker_start':before,'passed':True}
|
||||
Path('evidence').mkdir(exist_ok=True)
|
||||
Path('evidence/worker-incident.json').write_text(json.dumps(result,indent=2),encoding='utf-8')
|
||||
print('PASS worker recovery:',json.dumps(result,ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator actions for the local Compose stand; volumes are kept."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ("api1", "api2")
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True,
|
||||
text=True, stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def state(service):
|
||||
cid = compose("ps", "--all", "--quiet", service, capture=True)
|
||||
if not cid:
|
||||
raise RuntimeError("No container for " + service)
|
||||
result = subprocess.check_output(
|
||||
["docker", "inspect", "--format",
|
||||
'{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True)
|
||||
hostname, status = result.strip().split()
|
||||
return {"service": service, "container": cid[:12],
|
||||
"hostname": json.loads(hostname), "status": json.loads(status)}
|
||||
|
||||
|
||||
def wait_instances(count=2, timeout=40):
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r:
|
||||
instance = r.headers.get("X-Instance-Id")
|
||||
if instance:
|
||||
seen.add(instance)
|
||||
if len(seen) >= count:
|
||||
return seen
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError("Not enough ready API instances through LB")
|
||||
|
||||
|
||||
def stop_one(service=None):
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0]
|
||||
if target:
|
||||
service = next((s for s in SERVICES if state(s)["hostname"] == target), None)
|
||||
if not service:
|
||||
raise RuntimeError("TARGET_INSTANCE_ID is not an API container")
|
||||
service = service or "api1"
|
||||
if service not in SERVICES:
|
||||
raise RuntimeError("Only API replicas may be stopped")
|
||||
compose("kill", "--signal", "SIGKILL", service)
|
||||
current = state(service)
|
||||
if current["status"] != "exited":
|
||||
raise RuntimeError("API did not stop")
|
||||
print(json.dumps(current))
|
||||
|
||||
|
||||
def restore():
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES)
|
||||
wait_instances()
|
||||
|
||||
|
||||
def restart():
|
||||
compose("down")
|
||||
compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60")
|
||||
wait_instances()
|
||||
|
||||
|
||||
def rolling_update():
|
||||
phases = []
|
||||
compose("build", "api1")
|
||||
# This migration only adds tables and is compatible with the old API.
|
||||
compose("run", "--rm", "--no-deps", "migrate")
|
||||
for service in SERVICES:
|
||||
before = state(service)
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--force-recreate",
|
||||
"--wait", "--wait-timeout", "40", service)
|
||||
after = state(service)
|
||||
if before["container"] == after["container"]:
|
||||
raise RuntimeError("Container was not replaced")
|
||||
seen = wait_instances()
|
||||
if not any(i.split(":", 1)[0] == after["hostname"] for i in seen):
|
||||
raise RuntimeError("LB has not discovered the replacement")
|
||||
phases.append({"service": service, "started": started, "finished": time.monotonic()})
|
||||
print(json.dumps({"replaced": service, "before": before, "after": after}))
|
||||
return phases
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"])
|
||||
parser.add_argument("--service", choices=SERVICES)
|
||||
args = parser.parse_args()
|
||||
if args.action == "stop-one":
|
||||
stop_one(args.service)
|
||||
else:
|
||||
{"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\ops.py restart
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/ops.py restart
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\ops.py stop-one
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/ops.py stop-one
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Entry point: uvicorn as PID 1 with env-driven host/port and graceful shutdown."""
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .main import app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
port = int(os.environ.get("APP_PORT", "8080"))
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=host,
|
||||
port=port,
|
||||
log_level="info",
|
||||
access_log=True,
|
||||
timeout_keep_alive=2,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import secrets
|
||||
import redis.asyncio as redis
|
||||
from passlib.context import CryptContext
|
||||
|
||||
# bcrypt_sha256 avoids bcrypt's 72-byte limit. Old lab2 hashes still verify.
|
||||
pwd_context = CryptContext(schemes=["bcrypt_sha256", "bcrypt"], deprecated="auto")
|
||||
CACHE_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0")
|
||||
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
redis_client = redis.from_url(CACHE_URL, decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=1)
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
return pwd_context.verify(plain_password, hashed_password)
|
||||
|
||||
async def create_session(user_id: str) -> str:
|
||||
token = secrets.token_urlsafe(32)
|
||||
await redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, user_id)
|
||||
return token
|
||||
|
||||
async def get_user_id_by_token(token: str) -> str | None:
|
||||
return await redis_client.get(f"session:{token}")
|
||||
|
||||
async def delete_session(token: str) -> None:
|
||||
await redis_client.delete(f"session:{token}")
|
||||
@@ -0,0 +1,112 @@
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import List
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import String, ForeignKey, DateTime, Index, Integer, BigInteger
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql+asyncpg://user:password@localhost:5432/messenger")
|
||||
engine = create_async_engine(
|
||||
DATABASE_URL, echo=False, pool_pre_ping=True, pool_size=5, max_overflow=5,
|
||||
pool_timeout=2, connect_args={"timeout": 1, "command_timeout": 3,
|
||||
"server_settings": {"statement_timeout": "2500", "lock_timeout": "2000"}},
|
||||
)
|
||||
async_session = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100))
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
title: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||
members: Mapped[List["ChatMember"]] = relationship(back_populates="chat", lazy="selectin")
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True)
|
||||
user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
|
||||
chat: Mapped["Chat"] = relationship(back_populates="members")
|
||||
user: Mapped["User"] = relationship()
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True)
|
||||
sender_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"))
|
||||
text: Mapped[str] = mapped_column(String(4000))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index('ix_messages_chat_created_id', 'chat_id', 'created_at', 'id'),
|
||||
)
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
__tablename__ = "idempotency_keys"
|
||||
user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
request_text: Mapped[str] = mapped_column(String(4000))
|
||||
response: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
|
||||
request_attachment_ids: Mapped[list] = mapped_column(JSONB, default=list, nullable=False)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
class SchemaVersion(Base):
|
||||
__tablename__ = "schema_versions"
|
||||
version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), index=True, nullable=False)
|
||||
owner_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(String(250), unique=True, nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), default="queued", nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False)
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
message_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True)
|
||||
attachment_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
|
||||
# --- НОВЫЕ ТАБЛИЦЫ ДЛЯ 5-Й ЛАБЫ ---
|
||||
class AttachmentVariant(Base):
|
||||
__tablename__ = "attachment_variants"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
attachment_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), index=True)
|
||||
name: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(String(250), unique=True, nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
|
||||
class Outbox(Base):
|
||||
__tablename__ = "outbox"
|
||||
id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
|
||||
processed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Validate compressed image bytes without trusting filename or HTTP Content-Type."""
|
||||
import os
|
||||
import threading
|
||||
import struct
|
||||
import zlib
|
||||
import io
|
||||
import hashlib
|
||||
from PIL import Image, ImageFile, UnidentifiedImageError, ImageOps
|
||||
from .storage import MAX_IMAGE_PIXELS
|
||||
|
||||
# Prevent unrestricted decompression of malicious image dimensions.
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
_IMAGE_LOCK = threading.BoundedSemaphore(1)
|
||||
|
||||
class InvalidImage(Exception):
|
||||
pass
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def validate_image_file(file, declared_type: str) -> None:
|
||||
"""Проверяет, что файл является валидным PNG/JPEG и не превышает лимиты."""
|
||||
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}.get(declared_type)
|
||||
if not expected:
|
||||
raise InvalidImage("unsupported image type")
|
||||
with _IMAGE_LOCK:
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as probe:
|
||||
if probe.format != expected:
|
||||
raise InvalidImage("image MIME does not match its content")
|
||||
width, height = probe.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise InvalidImage("image dimensions are invalid")
|
||||
if width * height > MAX_IMAGE_PIXELS:
|
||||
raise TooManyPixels("image dimensions exceed limit")
|
||||
probe.verify() # verify compressed structure
|
||||
file.seek(0)
|
||||
with Image.open(file) as loaded:
|
||||
loaded.load() # detect damaged truncated payloads
|
||||
except Image.DecompressionBombError as e:
|
||||
raise TooManyPixels("image dimensions exceed limit") from e
|
||||
except Image.DecompressionBombWarning as e:
|
||||
raise TooManyPixels("image dimensions exceed limit") from e
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError, struct.error, zlib.error) as e:
|
||||
raise InvalidImage("invalid or truncated PNG/JPEG") from e
|
||||
finally:
|
||||
file.seek(0)
|
||||
|
||||
|
||||
def make_thumbnail(file_bytes: bytes) -> tuple[bytes, int, int, str]:
|
||||
"""
|
||||
Создает JPEG-превью (вписанное в 256x256) из сырых байтов.
|
||||
Возвращает: (байты_превью, ширина, высота, sha256_превью).
|
||||
"""
|
||||
with _IMAGE_LOCK:
|
||||
with Image.open(io.BytesIO(file_bytes)) as img:
|
||||
# 1. Применяем EXIF orientation (если картинку сфоткали на телефон боком)
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
# 2. Компонуем прозрачность (RGBA, LA) на белый фон
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
bg = Image.new("RGB", img.size, (255, 255, 255))
|
||||
bg.paste(img, mask=img.convert("RGBA").split()[3])
|
||||
img = bg
|
||||
elif img.mode != "RGB":
|
||||
# На всякий случай конвертируем все остальные режимы (например, CMYK) в RGB
|
||||
img = img.convert("RGB")
|
||||
|
||||
# 3. Вписываем в квадрат 256x256 (метод thumbnail сохраняет пропорции и не увеличивает маленькие картинки)
|
||||
img.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
|
||||
width, height = img.size
|
||||
|
||||
# 4. Сохраняем в JPEG
|
||||
out_buf = io.BytesIO()
|
||||
img.save(out_buf, format="JPEG", quality=85)
|
||||
data = out_buf.getvalue()
|
||||
|
||||
# 5. Считаем хэш для метаданных
|
||||
sha256 = hashlib.sha256(data).hexdigest()
|
||||
|
||||
return data, width, height, sha256
|
||||
|
||||
def make_variants(data):
|
||||
import io
|
||||
import hashlib
|
||||
from PIL import Image, ImageOps
|
||||
thumb_bytes, tw, th, tsha = make_thumbnail(data)
|
||||
outputs = [('thumbnail', thumb_bytes, 'image/jpeg', tw, th, tsha)]
|
||||
with _IMAGE_LOCK:
|
||||
with Image.open(io.BytesIO(data)) as original:
|
||||
image = ImageOps.exif_transpose(original)
|
||||
if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
|
||||
rgba = image.convert('RGBA')
|
||||
canvas = Image.new('RGB', image.size, 'white')
|
||||
canvas.paste(rgba, mask=rgba.getchannel('A'))
|
||||
image = canvas
|
||||
else:
|
||||
image = image.convert('RGB')
|
||||
crop = ImageOps.fit(image, (128, 128), Image.Resampling.LANCZOS, centering=(0.5,0.5))
|
||||
optimized = image.copy()
|
||||
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
for name, picture, fmt, quality in [('crop',crop,'JPEG',85), ('optimized',optimized,'WEBP',80)]:
|
||||
stream = io.BytesIO()
|
||||
picture.save(stream, format=fmt, quality=quality)
|
||||
raw = stream.getvalue()
|
||||
outputs.append((name,raw,'image/webp' if fmt=='WEBP' else 'image/jpeg',
|
||||
picture.width,picture.height,hashlib.sha256(raw).hexdigest()))
|
||||
return outputs
|
||||
+600
@@ -0,0 +1,600 @@
|
||||
from __future__ import annotations
|
||||
import base64
|
||||
import hashlib
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
import asyncio
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import aio_pika
|
||||
from fastapi import Depends, FastAPI, Request, Query
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from sqlalchemy import select, and_, or_, text
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from asyncpg import PostgresError
|
||||
from .db import engine, async_session, User, Chat, ChatMember, Message, IdempotencyKey, Attachment, MessageAttachment, Outbox, AttachmentVariant
|
||||
from .auth import hash_password, verify_password, create_session, get_user_id_by_token, delete_session, redis_client, SESSION_TTL_SECONDS
|
||||
from .storage import s3, bucket, check_storage, upload_original, delete_original, download_original, StorageFailure, MAX_UPLOAD_BYTES
|
||||
from .images import validate_image_file, InvalidImage, TooManyPixels
|
||||
from .telemetry import telemetry, trace, inject, extract, SpanKind, Status, StatusCode
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}", re.ASCII)
|
||||
INSTANCE_ID = socket.gethostname() + ":" + str(os.getpid())
|
||||
BROKER_URL = os.getenv("BROKER_URL", "amqp://guest:guest@broker:5672/")
|
||||
QUEUE_NAME = "attachment.process.v1"
|
||||
|
||||
def now_iso(dt: datetime = None) -> str:
|
||||
if not dt:
|
||||
dt = datetime.now(timezone.utc)
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
def error_payload(request_id: str, code: str, message: str) -> dict:
|
||||
return {"error": {"code": code, "message": message, "request_id": request_id}}
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
super().__init__(message)
|
||||
|
||||
# --- Курсоры для пагинации ---
|
||||
def _encode_cursor(scope: str, created_at: datetime, obj_id: uuid.UUID) -> str:
|
||||
raw = f"{scope}|{created_at.isoformat()}|{obj_id}".encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[str, datetime, uuid.UUID]:
|
||||
try:
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
raw = base64.urlsafe_b64decode(padded).decode("utf-8")
|
||||
scope, dt_str, obj_id_str = raw.split("|", 2)
|
||||
dt = datetime.fromisoformat(dt_str)
|
||||
if dt.tzinfo is None:
|
||||
raise ValueError("timezone required")
|
||||
return scope, dt, uuid.UUID(obj_id_str)
|
||||
except Exception:
|
||||
raise ApiError(400, "invalid_request", "invalid cursor")
|
||||
|
||||
# --- Фоновая задача: Relayer для Outbox ---
|
||||
async def observe_queue():
|
||||
while True:
|
||||
try:
|
||||
async with async_session() as session:
|
||||
row = (await session.execute(text("""
|
||||
SELECT count(*) FILTER (WHERE status='queued') AS queued,
|
||||
coalesce(extract(epoch from now() - min(created_at)
|
||||
FILTER (WHERE status='queued')), 0) AS age
|
||||
FROM attachments
|
||||
"""))).one()
|
||||
telemetry.queue_depth = int(row.queued)
|
||||
telemetry.queue_age = max(0.0, float(row.age))
|
||||
telemetry.db_used = int(engine.pool.checkedout())
|
||||
try:
|
||||
telemetry.db_waiting = len(engine.pool._pool._queue._getters)
|
||||
except AttributeError:
|
||||
telemetry.db_waiting = 0
|
||||
except Exception:
|
||||
telemetry.event('queue_metrics_unavailable')
|
||||
await asyncio.sleep(5)
|
||||
|
||||
async def outbox_relayer():
|
||||
while True:
|
||||
try:
|
||||
connection = await aio_pika.connect_robust(BROKER_URL)
|
||||
channel = await connection.channel(publisher_confirms=True)
|
||||
await channel.declare_queue(QUEUE_NAME, durable=True)
|
||||
break
|
||||
except Exception:
|
||||
telemetry.event('broker_connection_retry')
|
||||
await asyncio.sleep(5)
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
async with async_session() as session:
|
||||
records = (await session.scalars(
|
||||
select(Outbox).where(Outbox.processed_at == None, Outbox.available_at <= datetime.now(timezone.utc))
|
||||
.with_for_update(skip_locked=True).limit(10)
|
||||
)).all()
|
||||
for record in records:
|
||||
parent = extract(record.payload)
|
||||
with telemetry.tracer.start_as_current_span(
|
||||
'rabbitmq publish attachment', context=parent, kind=SpanKind.PRODUCER):
|
||||
carrier = {}
|
||||
inject(carrier)
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=json.dumps({'attachment_id': record.payload['attachment_id']}).encode(),
|
||||
headers=carrier,
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT),
|
||||
routing_key=QUEUE_NAME, mandatory=True)
|
||||
telemetry.event('job_published', job_type='attachment')
|
||||
record.processed_at = datetime.now(timezone.utc)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
telemetry.event('outbox_publish_retry')
|
||||
await asyncio.sleep(2)
|
||||
await asyncio.sleep(0.5)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
tasks = [asyncio.create_task(outbox_relayer()), asyncio.create_task(observe_queue())]
|
||||
yield
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
await redis_client.aclose()
|
||||
await engine.dispose()
|
||||
telemetry.shutdown()
|
||||
|
||||
app = FastAPI(title="Messenger", version="1.6.0", lifespan=lifespan)
|
||||
security_bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
request_id = request.headers.get('X-Request-Id') or str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
start = __import__('time').monotonic()
|
||||
remote_context = extract(request.headers)
|
||||
method = request.method
|
||||
with telemetry.tracer.start_as_current_span(
|
||||
'HTTP ' + method, context=remote_context, kind=SpanKind.SERVER) as span:
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
response = JSONResponse(status_code=500, content=error_payload(request_id, 'internal_error', 'internal error'))
|
||||
response.headers['X-Request-Id'] = request_id
|
||||
response.headers['X-Instance-Id'] = INSTANCE_ID
|
||||
route_obj = request.scope.get('route')
|
||||
route = getattr(route_obj, 'path', '/unmatched')
|
||||
if route.startswith('/health/'):
|
||||
route = '/health'
|
||||
status_class = str(response.status_code // 100) + 'xx'
|
||||
attrs = {'method':method,'route':route,'status_class':status_class}
|
||||
span.set_attribute('http.request.method', method)
|
||||
span.set_attribute('http.route', route)
|
||||
span.set_attribute('http.response.status_code', response.status_code)
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
telemetry.requests.add(1, attrs)
|
||||
telemetry.http_duration.record(__import__('time').monotonic()-start, attrs)
|
||||
telemetry.event('http_request', method=method, route=route,
|
||||
status=response.status_code, request_id=request_id)
|
||||
return response
|
||||
|
||||
def _rid(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", None) or str(uuid.uuid4())
|
||||
|
||||
# --- Обработчики ошибок ---
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
headers = {}
|
||||
if exc.status == 401:
|
||||
if request.url.path == "/api/v1/auth/sessions":
|
||||
headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"'
|
||||
else:
|
||||
headers["WWW-Authenticate"] = 'Bearer realm="messenger"'
|
||||
return JSONResponse(status_code=exc.status, content=error_payload(_rid(request), exc.code, exc.message), headers=headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
return JSONResponse(status_code=400, content=error_payload(_rid(request), "invalid_request", "invalid request"))
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||
return JSONResponse(status_code=exc.status_code, content=error_payload(_rid(request), "not_found", str(exc.detail)))
|
||||
|
||||
@app.exception_handler(InvalidImage)
|
||||
async def invalid_image_handler(request: Request, exc: InvalidImage):
|
||||
return JSONResponse(status_code=415, content=error_payload(_rid(request), "unsupported_media_type", str(exc)))
|
||||
|
||||
@app.exception_handler(TooManyPixels)
|
||||
async def too_large_image_handler(request: Request, exc: TooManyPixels):
|
||||
return JSONResponse(status_code=413, content=error_payload(_rid(request), "payload_too_large", str(exc)))
|
||||
|
||||
@app.exception_handler(RedisError)
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
@app.exception_handler(PostgresError)
|
||||
@app.exception_handler(OSError)
|
||||
@app.exception_handler(TimeoutError)
|
||||
@app.exception_handler(ClientError)
|
||||
@app.exception_handler(BotoCoreError)
|
||||
@app.exception_handler(StorageFailure)
|
||||
async def dependency_error_handler(request: Request, exc: Exception):
|
||||
return JSONResponse(status_code=503, content=error_payload(_rid(request), "unavailable", "dependency unavailable"))
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def generic_exception_handler(request: Request, exc: Exception):
|
||||
return JSONResponse(status_code=500, content=error_payload(_rid(request), "internal_error", "internal error"))
|
||||
|
||||
# --- Схемы ---
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
username: str = Field(strict=True)
|
||||
display_name: str = Field(strict=True, min_length=1, max_length=100)
|
||||
password: str = Field(strict=True, min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def not_blank(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("blank value")
|
||||
return value
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
title: str = Field(strict=True, min_length=1, max_length=100)
|
||||
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def not_blank(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("blank title")
|
||||
return value
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str = Field(strict=True, max_length=4000)
|
||||
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def unique_attachments(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("duplicate attachment id")
|
||||
return value
|
||||
|
||||
async def get_actor(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)) -> dict:
|
||||
if not credentials:
|
||||
raise ApiError(401, "unauthorized", "missing token")
|
||||
|
||||
user_id_str = await get_user_id_by_token(credentials.credentials)
|
||||
if not user_id_str:
|
||||
raise ApiError(401, "unauthorized", "invalid or expired token")
|
||||
|
||||
async with async_session() as session:
|
||||
user = await session.get(User, uuid.UUID(user_id_str))
|
||||
if not user:
|
||||
raise ApiError(401, "unauthorized", "user not found")
|
||||
return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)}
|
||||
|
||||
async def require_chat_member(session, chat_id: uuid.UUID, user_id: str):
|
||||
member = await session.scalar(select(ChatMember).where(and_(
|
||||
ChatMember.chat_id == chat_id, ChatMember.user_id == uuid.UUID(user_id))))
|
||||
if not member:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
@app.get("/health/live")
|
||||
def liveness():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def readiness():
|
||||
async with asyncio.timeout(1.5):
|
||||
async with async_session() as session:
|
||||
version = await session.scalar(text("SELECT version FROM schema_versions WHERE version = 6"))
|
||||
if version != 6:
|
||||
raise ApiError(503, "unavailable", "migration required")
|
||||
await redis_client.ping()
|
||||
await run_in_threadpool(check_storage)
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def register_user(payload: CreateUser):
|
||||
if not USERNAME_RE.fullmatch(payload.username):
|
||||
raise ApiError(400, "invalid_request", "invalid username")
|
||||
async with async_session() as session:
|
||||
existing = await session.scalar(select(User).where(User.username == payload.username))
|
||||
if existing:
|
||||
raise ApiError(409, "conflict", "username already exists")
|
||||
password_hash = await run_in_threadpool(hash_password, payload.password)
|
||||
user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash)
|
||||
session.add(user)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
raise ApiError(409, "conflict", "username already exists")
|
||||
return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)}
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def create_session_endpoint(request: Request):
|
||||
try:
|
||||
scheme, encoded = request.headers.get("Authorization", "").split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError("Basic required")
|
||||
username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials")
|
||||
|
||||
async with async_session() as session:
|
||||
user = await session.scalar(select(User).where(User.username == username))
|
||||
if not user or not await run_in_threadpool(verify_password, password, user.password_hash):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials")
|
||||
token = await create_session(str(user.id))
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
return {
|
||||
"token": token, "token_type": "Bearer", "expires_at": now_iso(expires_at),
|
||||
"user": {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)}
|
||||
}
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def logout(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)):
|
||||
if not credentials:
|
||||
raise ApiError(401, "unauthorized", "missing token")
|
||||
user_id = await get_user_id_by_token(credentials.credentials)
|
||||
if not user_id:
|
||||
raise ApiError(401, "unauthorized", "invalid or expired token")
|
||||
await delete_session(credentials.credentials)
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def current_user(actor: dict = Depends(get_actor)):
|
||||
return actor
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(payload: CreateChat, actor: dict = Depends(get_actor)):
|
||||
member_ids = [str(u) for u in payload.member_ids]
|
||||
if actor["id"] not in member_ids:
|
||||
raise ApiError(400, "invalid_request", "creator must be a member")
|
||||
if len(set(member_ids)) != len(member_ids):
|
||||
raise ApiError(400, "invalid_request", "duplicate members")
|
||||
async with async_session() as session:
|
||||
users = await session.scalars(select(User.id).where(User.id.in_([uuid.UUID(m) for m in member_ids])))
|
||||
if len(list(users)) != len(set(member_ids)):
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
chat = Chat(title=payload.title)
|
||||
session.add(chat)
|
||||
await session.flush()
|
||||
for mid in member_ids:
|
||||
session.add(ChatMember(chat_id=chat.id, user_id=uuid.UUID(mid)))
|
||||
await session.commit()
|
||||
return {"id": str(chat.id), "title": chat.title, "member_ids": member_ids, "created_at": now_iso(chat.created_at)}
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)):
|
||||
async with async_session() as session:
|
||||
stmt = select(Chat).join(ChatMember).where(ChatMember.user_id == uuid.UUID(actor["id"]))
|
||||
if cursor:
|
||||
scope, c_dt, c_id = _decode_cursor(cursor)
|
||||
if scope != "chats":
|
||||
raise ApiError(400, "invalid_request", "invalid cursor scope")
|
||||
stmt = stmt.where(or_(Chat.created_at > c_dt, and_(Chat.created_at == c_dt, Chat.id > c_id)))
|
||||
stmt = stmt.order_by(Chat.created_at.asc(), Chat.id.asc()).limit(limit + 1)
|
||||
chats = (await session.scalars(stmt)).unique().all()
|
||||
has_more = len(chats) > limit
|
||||
page = chats[:limit]
|
||||
next_cursor = _encode_cursor("chats", page[-1].created_at, page[-1].id) if has_more and page else None
|
||||
items = [{"id": str(c.id), "title": c.title, "created_at": now_iso(c.created_at), "member_ids": [str(m.user_id) for m in c.members]} for c in page]
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: uuid.UUID, actor: dict = Depends(get_actor)):
|
||||
async with async_session() as session:
|
||||
chat = await session.scalar(select(Chat).join(ChatMember).where(Chat.id == chat_id, ChatMember.user_id == uuid.UUID(actor["id"])))
|
||||
if not chat:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return {"id": str(chat.id), "title": chat.title, "member_ids": [str(m.user_id) for m in chat.members], "created_at": now_iso(chat.created_at)}
|
||||
|
||||
def attachment_json(item: Attachment, variants: list[AttachmentVariant] = None) -> dict:
|
||||
return {
|
||||
"id": str(item.id), "chat_id": str(item.chat_id),
|
||||
"owner_id": str(item.owner_id), "filename": item.filename,
|
||||
"content_type": item.content_type, "size_bytes": item.size_bytes,
|
||||
"sha256": item.sha256, "status": item.status,
|
||||
"variants": [{
|
||||
"name": v.name, "content_type": v.content_type, "size_bytes": v.size_bytes,
|
||||
"width": v.width, "height": v.height, "sha256": v.sha256
|
||||
} for v in (variants or [])],
|
||||
"error_code": item.error_code,
|
||||
"created_at": now_iso(item.created_at), "updated_at": now_iso(item.updated_at),
|
||||
}
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(
|
||||
chat_id: uuid.UUID, request: Request, filename: str = Query(...),
|
||||
actor: dict = Depends(get_actor),
|
||||
):
|
||||
async with async_session() as session:
|
||||
await require_chat_member(session, chat_id, actor["id"])
|
||||
|
||||
if not (1 <= len(filename) <= 255) or any(c in filename for c in ("/", "\\")) or any(ord(c) < 32 or ord(c) == 127 for c in filename):
|
||||
raise ApiError(400, "invalid_request", "invalid filename")
|
||||
|
||||
declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if declared_type not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "only PNG and JPEG are accepted")
|
||||
|
||||
hasher = hashlib.sha256()
|
||||
size = 0
|
||||
with SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as original:
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "image exceeds 10 MiB")
|
||||
hasher.update(chunk)
|
||||
original.write(chunk)
|
||||
if size == 0:
|
||||
raise ApiError(415, "unsupported_media_type", "empty image")
|
||||
await run_in_threadpool(validate_image_file, original, declared_type)
|
||||
|
||||
item_id = uuid.uuid4()
|
||||
key = f"originals/{item_id.hex}/{uuid.uuid4().hex}"
|
||||
with telemetry.tracer.start_as_current_span('s3 stage original', kind=SpanKind.CLIENT):
|
||||
await run_in_threadpool(upload_original, original, key, size, declared_type)
|
||||
try:
|
||||
async with async_session() as session:
|
||||
await require_chat_member(session, chat_id, actor["id"])
|
||||
item = Attachment(
|
||||
id=item_id, chat_id=chat_id, owner_id=uuid.UUID(actor["id"]),
|
||||
filename=filename, object_key=key, content_type=declared_type,
|
||||
size_bytes=size, sha256=hasher.hexdigest(), status="queued",
|
||||
)
|
||||
session.add(item)
|
||||
# Добавляем задачу в Outbox
|
||||
carrier = {}
|
||||
inject(carrier)
|
||||
payload = {'attachment_id': str(item_id), **carrier}
|
||||
session.add(Outbox(event_type='attachment.process.v1', payload=payload))
|
||||
with telemetry.tracer.start_as_current_span('db commit accepted job', kind=SpanKind.CLIENT):
|
||||
await session.commit()
|
||||
telemetry.event('attachment_accepted', job_type='attachment')
|
||||
return attachment_json(item)
|
||||
except Exception:
|
||||
try:
|
||||
await run_in_threadpool(delete_original, key)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning("S3 compensation failed; reconcile needed")
|
||||
raise
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def attachment_metadata(attachment_id: uuid.UUID, actor: dict = Depends(get_actor)):
|
||||
async with async_session() as session:
|
||||
item = await session.get(Attachment, attachment_id)
|
||||
if item is None:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
await require_chat_member(session, item.chat_id, actor["id"])
|
||||
|
||||
variants = (await session.scalars(select(AttachmentVariant).where(AttachmentVariant.attachment_id == item.id))).all()
|
||||
return attachment_json(item, variants)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def attachment_bytes(
|
||||
attachment_id: uuid.UUID, variant: str = "original", actor: dict = Depends(get_actor)
|
||||
):
|
||||
async with async_session() as session:
|
||||
item = await session.get(Attachment, attachment_id)
|
||||
if item is None:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
await require_chat_member(session, item.chat_id, actor["id"])
|
||||
|
||||
if item.status != "ready":
|
||||
raise ApiError(409, "conflict", "attachment is not ready")
|
||||
|
||||
if variant == "original":
|
||||
key, content_type, size = item.object_key, item.content_type, item.size_bytes
|
||||
else:
|
||||
var = await session.scalar(select(AttachmentVariant).where(
|
||||
AttachmentVariant.attachment_id == item.id, AttachmentVariant.name == variant
|
||||
))
|
||||
if not var:
|
||||
raise ApiError(404, "not_found", "variant not found")
|
||||
key, content_type, size = var.object_key, var.content_type, var.size_bytes
|
||||
|
||||
body = await run_in_threadpool(download_original, key)
|
||||
def chunks():
|
||||
try:
|
||||
yield from body.iter_chunks(chunk_size=64 * 1024)
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(chunks(), media_type=content_type,
|
||||
headers={"Content-Length": str(size), "Cache-Control": "private, no-store"})
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def post_message(chat_id: uuid.UUID, payload: CreateMessage, request: Request, actor: dict = Depends(get_actor)):
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not KEY_RE.fullmatch(key):
|
||||
raise ApiError(400, "invalid_request", "invalid Idempotency-Key")
|
||||
if not payload.text.strip() and not payload.attachment_ids:
|
||||
raise ApiError(400, "invalid_request", "empty message")
|
||||
attachment_ids = [str(a) for a in payload.attachment_ids]
|
||||
|
||||
async with async_session() as session:
|
||||
await require_chat_member(session, chat_id, actor["id"])
|
||||
|
||||
if attachment_ids:
|
||||
attachments = (await session.scalars(select(Attachment).where(
|
||||
Attachment.id.in_(payload.attachment_ids), Attachment.chat_id == chat_id
|
||||
))).all()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
# В 5-й лабе можно прикреплять файлы в статусе queued/processing, но не failed
|
||||
if any(a.status == "failed" for a in attachments):
|
||||
raise ApiError(409, "conflict", "attachment failed and cannot be attached")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
scope = {"user_id": uuid.UUID(actor["id"]), "chat_id": chat_id, "key": key}
|
||||
await session.execute(insert(IdempotencyKey).values(
|
||||
**scope, request_text=payload.text, request_attachment_ids=attachment_ids,
|
||||
expires_at=now + timedelta(hours=24)
|
||||
).on_conflict_do_nothing())
|
||||
record = await session.scalar(select(IdempotencyKey).where(
|
||||
IdempotencyKey.user_id == scope["user_id"],
|
||||
IdempotencyKey.chat_id == chat_id, IdempotencyKey.key == key
|
||||
).with_for_update())
|
||||
|
||||
if record.response is not None and record.expires_at > now:
|
||||
if record.request_text != payload.text or record.request_attachment_ids != attachment_ids:
|
||||
raise ApiError(409, "conflict", "key used with another message")
|
||||
return record.response
|
||||
|
||||
msg = Message(chat_id=chat_id, sender_id=uuid.UUID(actor["id"]), text=payload.text)
|
||||
session.add(msg)
|
||||
await session.flush()
|
||||
for pos, a_id in enumerate(payload.attachment_ids):
|
||||
session.add(MessageAttachment(message_id=msg.id, attachment_id=a_id, position=pos))
|
||||
|
||||
result = {
|
||||
"id": str(msg.id), "chat_id": str(msg.chat_id), "sender_id": str(msg.sender_id),
|
||||
"text": msg.text, "created_at": now_iso(msg.created_at),
|
||||
"attachment_ids": attachment_ids,
|
||||
}
|
||||
record.request_text = payload.text
|
||||
record.request_attachment_ids = attachment_ids
|
||||
record.expires_at = now + timedelta(hours=24)
|
||||
record.response = result
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def get_messages(chat_id: uuid.UUID, limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)):
|
||||
async with async_session() as session:
|
||||
await require_chat_member(session, chat_id, actor["id"])
|
||||
stmt = select(Message).where(Message.chat_id == chat_id)
|
||||
if cursor:
|
||||
scope, c_dt, c_id = _decode_cursor(cursor)
|
||||
if scope != str(chat_id):
|
||||
raise ApiError(400, "invalid_request", "invalid cursor scope")
|
||||
stmt = stmt.where(or_(Message.created_at > c_dt, and_(Message.created_at == c_dt, Message.id > c_id)))
|
||||
stmt = stmt.order_by(Message.created_at.asc(), Message.id.asc()).limit(limit + 1)
|
||||
messages = (await session.scalars(stmt)).all()
|
||||
has_more = len(messages) > limit
|
||||
page = messages[:limit]
|
||||
next_cursor = _encode_cursor(str(chat_id), page[-1].created_at, page[-1].id) if has_more and page else None
|
||||
|
||||
msg_ids = [m.id for m in page]
|
||||
links = []
|
||||
if msg_ids:
|
||||
links = (await session.execute(select(MessageAttachment).where(
|
||||
MessageAttachment.message_id.in_(msg_ids)
|
||||
).order_by(MessageAttachment.message_id, MessageAttachment.position))).scalars().all()
|
||||
linked: dict[uuid.UUID, list[str]] = {m.id: [] for m in page}
|
||||
for link in links:
|
||||
linked[link.message_id].append(str(link.attachment_id))
|
||||
items = [{
|
||||
"id": str(m.id), "chat_id": str(m.chat_id), "sender_id": str(m.sender_id),
|
||||
"text": m.text, "created_at": now_iso(m.created_at),
|
||||
"attachment_ids": linked[m.id],
|
||||
} for m in page]
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import text
|
||||
from .db import Base, engine
|
||||
|
||||
|
||||
async def main():
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("SELECT pg_advisory_xact_lock(300300)"))
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
await conn.execute(text("ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code VARCHAR(64)"))
|
||||
|
||||
await conn.execute(text("INSERT INTO schema_versions(version) VALUES (3) ON CONFLICT DO NOTHING"))
|
||||
await conn.execute(text("INSERT INTO schema_versions(version) VALUES (4) ON CONFLICT DO NOTHING"))
|
||||
await conn.execute(text("INSERT INTO schema_versions(version) VALUES (5) ON CONFLICT DO NOTHING"))
|
||||
await conn.execute(text("ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempts INTEGER NOT NULL DEFAULT 0"))
|
||||
await conn.execute(text("ALTER TABLE outbox ADD COLUMN IF NOT EXISTS available_at TIMESTAMPTZ NOT NULL DEFAULT now()"))
|
||||
await conn.execute(text("INSERT INTO schema_versions(version) VALUES (6) ON CONFLICT DO NOTHING"))
|
||||
await engine.dispose()
|
||||
print("Migration 5 applied: outbox, variants and async processing added")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Prove S3 is not public and API-scoped credentials cannot list other buckets."""
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from botocore.exceptions import ClientError
|
||||
from .storage import s3, bucket
|
||||
import os
|
||||
import uuid
|
||||
|
||||
url = os.getenv("S3_ENDPOINT", "http://s3:8333").rstrip("/") + "/" + bucket + "/random-inaccessible-key"
|
||||
try:
|
||||
urllib.request.urlopen(url, timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
assert exc.code in (401, 403), f"expected S3 denied, got {exc.code}"
|
||||
else:
|
||||
raise AssertionError("UNSAFE: direct unsigned S3 request succeeded")
|
||||
try:
|
||||
s3.create_bucket(Bucket="forbidden-probe-" + uuid.uuid4().hex[:16])
|
||||
except ClientError as exc:
|
||||
assert exc.response["Error"]["Code"] in ("AccessDenied", "403"), "unexpected S3 result"
|
||||
else:
|
||||
raise AssertionError("UNSAFE: API credentials have global create-bucket privileges")
|
||||
print("PASS: S3 requires authentication; API credentials restricted to own bucket")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Safe orphan S3 cleanup (dry run default). Use inside api container.
|
||||
|
||||
A crash between S3 PUT and PostgreSQL COMMIT can leave an unreferenced object.
|
||||
Only objects older than the grace period are candidates. No deletion if DB read fails.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from sqlalchemy import select
|
||||
from .db import async_session, Attachment, engine
|
||||
from .storage import s3, bucket
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
|
||||
def list_s3():
|
||||
keys = []
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix="originals/"):
|
||||
keys.extend((i["Key"], i["LastModified"]) for i in page.get("Contents", []))
|
||||
return keys
|
||||
|
||||
|
||||
async def run(apply=False, grace_seconds=3600):
|
||||
if grace_seconds < 3600:
|
||||
raise ValueError("grace must be at least 3600 seconds")
|
||||
# Load DB references before enumerating/deleting S3 objects. Fail closed.
|
||||
async with async_session() as session:
|
||||
referenced = set((await session.scalars(select(Attachment.object_key))).all())
|
||||
objects = await run_in_threadpool(list_s3)
|
||||
threshold = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
candidates = [key for key, when in objects if key not in referenced and when < threshold]
|
||||
print(f"referenced={len(referenced)} objects={len(objects)} old_orphans={len(candidates)} apply={apply}")
|
||||
for key in candidates:
|
||||
# Avoid listing filenames/secrets: keys are random IDs, not user inputs.
|
||||
if apply:
|
||||
await run_in_threadpool(s3.delete_object, Bucket=bucket, Key=key)
|
||||
print(("deleted " if apply else "would_delete ") + key)
|
||||
return len(candidates)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="really delete aged orphan objects")
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
async def main():
|
||||
try:
|
||||
await run(args.apply, args.grace_seconds)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
"""S3-compatible private storage. Only API-scoped credentials are available here."""
|
||||
import os
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
s3 = boto3.client(
|
||||
"s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"),
|
||||
aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"),
|
||||
config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30,
|
||||
retries={"mode": "standard", "max_attempts": 2}),
|
||||
)
|
||||
|
||||
class StorageFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def check_storage():
|
||||
# Scoped user has ListBucket but no privileges outside this bucket.
|
||||
s3.list_objects_v2(Bucket=bucket, MaxKeys=1)
|
||||
|
||||
|
||||
def upload_original(file, key, size, content_type):
|
||||
file.seek(0)
|
||||
s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type)
|
||||
|
||||
|
||||
def delete_original(key):
|
||||
s3.delete_object(Bucket=bucket, Key=key)
|
||||
|
||||
|
||||
def download_original(key):
|
||||
try:
|
||||
return s3.get_object(Bucket=bucket, Key=key)["Body"]
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
|
||||
raise StorageFailure("Object missing") from None
|
||||
raise
|
||||
@@ -0,0 +1,87 @@
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.propagate import extract, inject
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
BASE = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
||||
SERVICE = os.getenv('OTEL_SERVICE_NAME', 'messenger-api')
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self):
|
||||
resource = Resource.create({
|
||||
'service.name': SERVICE,
|
||||
'service.version': '1.6.0',
|
||||
'service.instance.id': os.getenv('INSTANCE_ID', socket.gethostname()),
|
||||
'deployment.environment.name': 'lab',
|
||||
})
|
||||
self.trace_provider = TracerProvider(resource=resource)
|
||||
self.trace_provider.add_span_processor(BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=BASE+'/v1/traces', timeout=2),
|
||||
max_queue_size=256, max_export_batch_size=32,
|
||||
schedule_delay_millis=2000, export_timeout_millis=2000))
|
||||
trace.set_tracer_provider(self.trace_provider)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=BASE+'/v1/metrics', timeout=2),
|
||||
export_interval_millis=5000, export_timeout_millis=3000)
|
||||
self.metric_provider = MeterProvider(resource=resource, metric_readers=[reader])
|
||||
metrics.set_meter_provider(self.metric_provider)
|
||||
self.log_provider = LoggerProvider(resource=resource)
|
||||
self.log_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=BASE+'/v1/logs', timeout=2),
|
||||
max_queue_size=256, max_export_batch_size=32, schedule_delay_millis=2000))
|
||||
self.log = logging.getLogger('messenger.telemetry')
|
||||
self.log.setLevel(logging.INFO)
|
||||
self.log.propagate = False
|
||||
self.log.addHandler(LoggingHandler(level=logging.INFO, logger_provider=self.log_provider))
|
||||
self.tracer = trace.get_tracer('messenger', '1.6.0')
|
||||
meter = metrics.get_meter('messenger', '1.6.0')
|
||||
self.requests = meter.create_counter('messenger_http_requests', unit='1')
|
||||
self.http_duration = meter.create_histogram('messenger_http_duration', unit='s', explicit_bucket_boundaries_advisory=[0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.4, 0.8, 1.5, 3.0, 6.0, 12.0])
|
||||
self.jobs = meter.create_counter('messenger_jobs', unit='1')
|
||||
self.job_duration = meter.create_histogram('messenger_job_duration', unit='s', explicit_bucket_boundaries_advisory=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0])
|
||||
self.time_to_ready = meter.create_histogram('messenger_time_to_ready', unit='s', explicit_bucket_boundaries_advisory=[0.1, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300])
|
||||
self.queue_depth = 0
|
||||
self.queue_age = 0.0
|
||||
self.db_used = 0
|
||||
self.db_waiting = 0
|
||||
self.worker_busy = 0
|
||||
self.worker_capacity = 1 if SERVICE == 'messenger-worker' else 0
|
||||
self.queue_metric = meter.create_observable_gauge('messenger_queue_depth', callbacks=[lambda _: [metrics.Observation(self.queue_depth, {'queue':'attachment.process.v1'})],], unit='1')
|
||||
self.age_metric = meter.create_observable_gauge('messenger_queue_oldest_age', callbacks=[lambda _: [metrics.Observation(self.queue_age, {'queue':'attachment.process.v1'})],], unit='s')
|
||||
self.used_metric = meter.create_observable_gauge('messenger_db_pool_used', callbacks=[lambda _: [metrics.Observation(self.db_used)],], unit='1')
|
||||
self.waiting_metric = meter.create_observable_gauge('messenger_db_pool_waiting', callbacks=[lambda _: [metrics.Observation(self.db_waiting)],], unit='1')
|
||||
self.busy_metric = meter.create_observable_gauge('messenger_worker_busy', callbacks=[lambda _: [metrics.Observation(self.worker_busy)],], unit='1')
|
||||
self.capacity_metric = meter.create_observable_gauge('messenger_worker_capacity', callbacks=[lambda _: [metrics.Observation(self.worker_capacity)],], unit='1')
|
||||
|
||||
def event(self, name, **safe_fields):
|
||||
payload = 'event=' + name
|
||||
for key, value in safe_fields.items():
|
||||
if key in ('route', 'method', 'status', 'outcome', 'job_type', 'request_id'):
|
||||
payload += ' ' + key + '=' + str(value).replace(' ', '_')[:100]
|
||||
span_context = trace.get_current_span().get_span_context()
|
||||
if span_context.is_valid:
|
||||
payload += ' trace_id=%032x span_id=%016x' % (span_context.trace_id, span_context.span_id)
|
||||
self.log.info(payload)
|
||||
|
||||
def shutdown(self):
|
||||
self.trace_provider.shutdown()
|
||||
self.metric_provider.shutdown()
|
||||
self.log_provider.shutdown()
|
||||
|
||||
telemetry = Telemetry()
|
||||
|
||||
__all__ = ['telemetry', 'trace', 'extract', 'inject', 'SpanKind', 'Status', 'StatusCode']
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import aio_pika
|
||||
from PIL import Image, ImageOps
|
||||
from sqlalchemy import select
|
||||
|
||||
from .db import async_session, engine, Attachment, AttachmentVariant, Outbox
|
||||
from .storage import upload_original, download_original
|
||||
from .images import make_thumbnail, make_variants
|
||||
from .telemetry import telemetry, extract, SpanKind, Status, StatusCode
|
||||
|
||||
BROKER_URL = os.getenv('BROKER_URL', 'amqp://guest:guest@broker:5672/')
|
||||
QUEUE_NAME = 'attachment.process.v1'
|
||||
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
||||
|
||||
async def process_attachment(message: aio_pika.IncomingMessage):
|
||||
ctx = extract(message.headers or {})
|
||||
started = time.monotonic()
|
||||
telemetry.worker_busy = 1
|
||||
outcome = 'success'
|
||||
try:
|
||||
with telemetry.tracer.start_as_current_span(
|
||||
'attachment.process', context=ctx, kind=SpanKind.CONSUMER) as span:
|
||||
async with message.process(requeue=True):
|
||||
payload = json.loads(message.body.decode())
|
||||
attachment_id = uuid.UUID(payload['attachment_id'])
|
||||
telemetry.event('job_started', job_type='attachment')
|
||||
async with async_session() as session:
|
||||
att = await session.get(Attachment, attachment_id, with_for_update=True)
|
||||
if not att or att.status in ('ready', 'failed'):
|
||||
telemetry.event('job_duplicate_skipped', job_type='attachment')
|
||||
return
|
||||
att.status = 'processing'
|
||||
att.attempts = int(att.attempts or 0) + 1
|
||||
attempt = att.attempts
|
||||
await session.commit()
|
||||
try:
|
||||
with telemetry.tracer.start_as_current_span('s3.download_original', kind=SpanKind.CLIENT):
|
||||
stream = await asyncio.to_thread(download_original, att.object_key)
|
||||
data = await asyncio.to_thread(stream.read)
|
||||
await asyncio.to_thread(stream.close)
|
||||
with telemetry.tracer.start_as_current_span('image.create_variants', kind=SpanKind.INTERNAL):
|
||||
outputs = await asyncio.to_thread(make_variants, data)
|
||||
with telemetry.tracer.start_as_current_span('s3.store_variants', kind=SpanKind.CLIENT):
|
||||
for name, raw, mime, w, h, sha in outputs:
|
||||
key = f'variants/{attachment_id.hex}/{name}'
|
||||
await asyncio.to_thread(upload_original, io.BytesIO(raw), key, len(raw), mime)
|
||||
with telemetry.tracer.start_as_current_span('db.commit_ready', kind=SpanKind.CLIENT):
|
||||
async with async_session() as session:
|
||||
locked = await session.get(Attachment, attachment_id, with_for_update=True)
|
||||
if locked and locked.status != 'ready':
|
||||
for name, raw, mime, w, h, sha in outputs:
|
||||
previous = await session.scalar(select(AttachmentVariant).where(
|
||||
AttachmentVariant.attachment_id == attachment_id,
|
||||
AttachmentVariant.name == name))
|
||||
if previous is None:
|
||||
previous = AttachmentVariant(attachment_id=attachment_id, name=name)
|
||||
session.add(previous)
|
||||
previous.object_key = f'variants/{attachment_id.hex}/{name}'
|
||||
previous.content_type = mime
|
||||
previous.size_bytes = len(raw)
|
||||
previous.width = w
|
||||
previous.height = h
|
||||
previous.sha256 = sha
|
||||
locked.status = 'ready'
|
||||
locked.error_code = None
|
||||
await session.commit()
|
||||
elapsed = (datetime.now(timezone.utc) - locked.created_at).total_seconds()
|
||||
telemetry.time_to_ready.record(max(0.0,elapsed), {'job_type':'attachment'})
|
||||
telemetry.event('job_ready', job_type='attachment', outcome='success')
|
||||
except Exception as exc:
|
||||
outcome = 'failed'
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
async with async_session() as session:
|
||||
locked = await session.get(Attachment, attachment_id, with_for_update=True)
|
||||
if locked and locked.status != 'ready':
|
||||
if attempt >= MAX_ATTEMPTS or isinstance(exc, (ValueError, Image.DecompressionBombError)):
|
||||
locked.status = 'failed'
|
||||
locked.error_code = 'retry_exhausted' if attempt >= MAX_ATTEMPTS else 'invalid_image'
|
||||
telemetry.event('job_failed', job_type='attachment', outcome='permanent')
|
||||
else:
|
||||
locked.status = 'queued'
|
||||
locked.error_code = None
|
||||
delay = min(60, 2 ** min(attempt, 5))
|
||||
session.add(Outbox(
|
||||
event_type='attachment.process.v1',
|
||||
payload={'attachment_id':str(attachment_id), **dict(message.headers or {})},
|
||||
available_at=datetime.now(timezone.utc)+timedelta(seconds=delay)))
|
||||
telemetry.event('job_retry_scheduled', job_type='attachment', outcome='transient')
|
||||
await session.commit()
|
||||
except Exception:
|
||||
outcome = 'failed'
|
||||
telemetry.event('job_delivery_exception', job_type='attachment')
|
||||
raise
|
||||
finally:
|
||||
telemetry.jobs.add(1, {'job_type':'attachment', 'outcome':outcome})
|
||||
telemetry.job_duration.record(time.monotonic()-started, {'job_type':'attachment','outcome':outcome})
|
||||
telemetry.worker_busy = 0
|
||||
|
||||
async def main():
|
||||
connection = await aio_pika.connect_robust(BROKER_URL)
|
||||
channel = await connection.channel(publisher_confirms=True)
|
||||
queue = await channel.declare_queue(QUEUE_NAME, durable=True)
|
||||
await channel.set_qos(prefetch_count=1)
|
||||
await queue.consume(process_attachment)
|
||||
telemetry.event('worker_started', job_type='attachment')
|
||||
try:
|
||||
await asyncio.Future()
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
telemetry.shutdown()
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab3: concurrent idempotency and validation")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
c.call("POST", "/api/v1/users", 400,
|
||||
body={"username": "valid\n", "display_name": "Test", "password": "LongPassword123"})
|
||||
registration = {"username": "race_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Race", "password": "LongPassword123"}
|
||||
registration_barrier = threading.Barrier(2)
|
||||
|
||||
def register(_):
|
||||
local = Client(args.base_url, args.ca_file)
|
||||
registration_barrier.wait()
|
||||
return local.call("POST", "/api/v1/users", [201, 409],
|
||||
body=registration, schema="User")["status"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
check(sorted(pool.map(register, range(2))) == [201, 409], "registration race")
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(12)
|
||||
|
||||
def send(_):
|
||||
local = Client(args.base_url, args.ca_file)
|
||||
barrier.wait()
|
||||
return local.message(alice, chat, "Одна запись", key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses = list(pool.map(send, range(12)))
|
||||
first = responses[0]["json"]
|
||||
check(all(r["json"] == first for r in responses), "concurrent replay differs")
|
||||
instances = {r["headers"]["X-Instance-Id"] for r in responses}
|
||||
check(len(instances) >= 2, "same key must reach at least two replicas")
|
||||
page = c.call("GET", path, actor=bob, schema="MessagePage")["json"]
|
||||
check(page["items"] == [first], "concurrent retry created duplicates")
|
||||
c.call("POST", path, 409, actor=alice, body={"text": "Изменённый текст"},
|
||||
headers={"Idempotency-Key": key})
|
||||
c.call("POST", path, 404, actor=eve, body={"text": "Одна запись"},
|
||||
headers={"Idempotency-Key": key})
|
||||
# The same key belongs to a different scope for Bob and for another chat.
|
||||
other = c.message(bob, chat, "Другая область", key=key)["json"]
|
||||
check(other["id"] != first["id"], "different users share an idempotency scope")
|
||||
second_chat = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Второй", "member_ids": [alice["id"]]},
|
||||
schema="Chat")["json"]
|
||||
other = c.message(alice, second_chat, "Одна запись", key=key)["json"]
|
||||
check(other["id"] != first["id"], "different chats share an idempotency scope")
|
||||
for bad_key in ("", "bad key", "x" * 129):
|
||||
c.call("POST", path, 400, actor=alice, body={"text": "Текст"},
|
||||
headers={"Idempotency-Key": bad_key})
|
||||
for body in ({"text": " "}, {"text": 123}, {"text": "x", "sender_id": alice["id"]}):
|
||||
c.call("POST", path, 400, actor=alice, body=body,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
c.call("POST", path, 400, actor=alice, raw=b'{"text":',
|
||||
headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex})
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
c.call("POST", path, 401, actor=alice, body={"text": "Одна запись"},
|
||||
headers={"Idempotency-Key": key})
|
||||
print("PASS: 12 concurrent requests across %d replicas, one message" % len(instances))
|
||||
print("PASS: conflict, key scope, validation, access and revoked session")
|
||||
print("PASS: concurrent registration, exactly one 201 and one 409")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Additional grade-4 negative and access tests. Requires running Compose stack."""
|
||||
import hashlib
|
||||
import zlib
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, check, arguments, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def main():
|
||||
args = arguments("Lab5: file validation, permissions, idempotency and isolation")
|
||||
c = Client(args.base_url, args.ca_file, timeout=35)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
file = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
path = "/api/v1/chats/" + chat["id"]
|
||||
upload = path + "/attachments?filename=photo.png"
|
||||
|
||||
# 1. Загружаем (теперь ожидаем 202 вместо 201)
|
||||
result = c.call("POST",upload,202,actor=alice,raw=file,
|
||||
headers={"Content-Type":"image/png"},schema="Attachment")["json"]
|
||||
obj = "/api/v1/attachments/" + result["id"]
|
||||
check(result["sha256"] == hashlib.sha256(file).hexdigest(),"checksum mismatch")
|
||||
|
||||
# Ждем, пока воркер обработает картинку (переведет в статус ready)
|
||||
result = c.wait_ready(alice, result)
|
||||
|
||||
# Теперь контент доступен
|
||||
check(c.call("GET",obj+"/content",actor=bob)["bytes"] == file,"original differs")
|
||||
c.call("GET",obj,404,actor=eve)
|
||||
c.call("GET",obj+"/content",404,actor=eve)
|
||||
c.call("GET","/api/v1/attachments/"+str(uuid.uuid4()),404,actor=bob)
|
||||
|
||||
# Негативные тесты (они падают сразу на этапе валидации, до 202)
|
||||
c.call("POST",upload,415,actor=alice,raw=b"not an image",
|
||||
headers={"Content-Type":"image/png"})
|
||||
c.call("POST",upload,415,actor=alice,raw=file,
|
||||
headers={"Content-Type":"image/jpeg"})
|
||||
c.call("POST",upload,413,actor=alice,raw=b"\0"*10485761,
|
||||
headers={"Content-Type":"image/png"})
|
||||
|
||||
huge = bytearray(file)
|
||||
huge[16:20] = (10000).to_bytes(4, "big")
|
||||
huge[20:24] = (5000).to_bytes(4, "big")
|
||||
huge[29:33] = (zlib.crc32(huge[12:29]) & 0xffffffff).to_bytes(4, "big")
|
||||
c.call("POST",upload,413,actor=alice,raw=bytes(huge),
|
||||
headers={"Content-Type":"image/png"})
|
||||
|
||||
c.call("POST",path+"/attachments?filename=..%2Fhidden.png",400,actor=alice,
|
||||
raw=file,headers={"Content-Type":"image/png"})
|
||||
c.call("POST",path+"/attachments?filename=sample.txt",415,actor=alice,
|
||||
raw=file,headers={"Content-Type":"text/plain"})
|
||||
|
||||
second = c.call("POST","/api/v1/chats",201,actor=alice,
|
||||
body={"title":"Other Chat","member_ids":[alice["id"]]},schema="Chat")["json"]
|
||||
c.call("POST", "/api/v1/chats/"+second["id"]+"/messages",404,
|
||||
actor=alice,body={"text":"","attachment_ids":[result["id"]]},
|
||||
headers={"Idempotency-Key":uuid.uuid4().hex})
|
||||
|
||||
# В 5-й лабе можно прикреплять файлы в статусе queued/processing/ready
|
||||
attached = c.message(alice,chat,"",attachments=[result["id"]])["json"]
|
||||
check(attached["attachment_ids"] == [result["id"]],"message lost attachment")
|
||||
|
||||
key = uuid.uuid4().hex
|
||||
one = c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"]
|
||||
check(c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"] == one,
|
||||
"repeated idempotency response mismatch")
|
||||
c.call("POST",path+"/messages",409,actor=alice,
|
||||
body={"text":"Фото", "attachment_ids":[]},headers={"Idempotency-Key":key})
|
||||
|
||||
print("PASS: private downloads, MIME spoof, corrupt image, oversize, dangerous filename, foreign chat, idempotency")
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user