Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
428a3e7e77 | ||
|
|
33f2448e20 | ||
|
|
042d5f83f2 | ||
|
|
d954fa5f1b |
+22
@@ -0,0 +1,22 @@
|
|||||||
|
FROM python:3.11.7-slim-bookworm AS builder
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||||
|
|
||||||
|
FROM python:3.11.7-slim-bookworm AS runner
|
||||||
|
|
||||||
|
RUN useradd -u 10001 -m appuser
|
||||||
|
WORKDIR /home/appuser
|
||||||
|
|
||||||
|
COPY --from=builder /install /usr/local
|
||||||
|
COPY app.py worker.py ./
|
||||||
|
|
||||||
|
USER appuser
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=5s --timeout=3s --retries=3 \
|
||||||
|
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"]
|
||||||
@@ -1,37 +1,108 @@
|
|||||||
# Отчёт — лабораторная 6
|
# Отчёт — лабораторная 6
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: Бондаренко Мария, БПИ2302
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: 4
|
||||||
- Commit/PR: TODO после публикации
|
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## 1. Архитектура и стек наблюдаемости
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
В рамках лабораторной работы для распределённого сервиса мессенджера развёрнут единый контур наблюдаемости на базе OpenTelemetry и стека LGTM/Prometheus:
|
||||||
|
|
||||||
## Архитектура и решения
|
* Сбор и маршрутизация телеметрии — OpenTelemetry Collector (otel-collector:4317/4318):
|
||||||
|
* принимает сигналы по протоколу OTLP (gRPC и HTTP/protobuf) от messenger-api и messenger-worker;
|
||||||
|
* применяет процессоры memory_limiter (ограничение памяти 256 MiB) и batch (пакетная отправка по 256 записей или каждые 2 секунды);
|
||||||
|
* маршрутизирует данные в три независимых хранилища с использованием очередей отправки:
|
||||||
|
* Метрики -> prometheusremotewrite;
|
||||||
|
* Трейсы -> otlphttp/tempo;
|
||||||
|
* Логи -> otlphttp/loki.
|
||||||
|
* Хранилище метрик и правила алертинга — Prometheus:
|
||||||
|
* запущен с флагом --web.enable-remote-write-receiver и ретеншеном 7 дней;
|
||||||
|
* оценивает правила оповещений из /etc/prometheus/alerts.yml с интервалом 15s.
|
||||||
|
* Хранилище структурированных логов — Grafana Loki:
|
||||||
|
* использует схему хранения tsdb и локальное файловое хранилище;
|
||||||
|
* индексирует атрибуты ресурса service.name, service.namespace, deployment.environment, а также поддерживает allow_structured_metadata: true.
|
||||||
|
* Хранилище распределённых трассировок — Grafana Tempo:
|
||||||
|
* принимает OTLP HTTP трейсы на порту 4318, хранит блоки локально с периодом хранения 168 часов.
|
||||||
|
* Визуализация и корреляция — Grafana:
|
||||||
|
* настроена через provisioning (datasources.yml и dashboards.yml);
|
||||||
|
* реализована двусторонняя связка Logs <-Traces:
|
||||||
|
* в источнике Loki настроено derivedFields по регулярному выражению "trace_id":"([0-9a-f]{32})" для перехода из строки лога в соответствующий трейс в Tempo;
|
||||||
|
* в источнике Tempo настроено tracesToLogsV2 с фильтрацией по TraceID и тегу service.name.
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
---
|
||||||
|
|
||||||
## Доказательства критериев
|
## 2. Инструментирование сервисов
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
### 2.1. Сервис messenger-api (app.py)
|
||||||
| --- | --- | --- | --- |
|
* Инициализированы TracerProvider, MeterProvider (экспорт каждые 5000 ms) и LoggerProvider через OTLP HTTP экспортеры.
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
* Подключено автоматическое инструментирование FastAPI и кастомный HTTP middleware telemetry_middleware, фиксирующий низкокардинальные атрибуты.
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
* Реализовано структурированное JSON-логирование, автоматически извлекающее активный контекст OpenTelemetry и добавляющее в каждую запись поля trace_id (32 hex-символа) и span_id (16 hex-символов), а также контекст запроса (method, path, status_code).
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
* Реализован потокобезопасный пул соединений к PostgreSQL с корректным возвратом соединений в блоках finally.
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
* При постановке задачи обработки вложения в таблицу outbox вызывается inject(job_payload), что сохраняет traceparent в JSON-полезной нагрузке задачи.
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
### 2.2. Асинхронный обработчик messenger-worker (worker.py)
|
||||||
|
* Настроен экспорт метрик, логов и спанов с именем сервиса messenger-worker.
|
||||||
|
* При чтении задачи из Redis Stream (attachment_jobs) воркер извлекает контекст трассировки с помощью extract(job) и открывает дочерний спан worker.process_attachment с атрибутами job.pipeline_version, job.attempt, job.outcome.
|
||||||
|
* Реализован конвейер генерации трёх вариантов изображений (thumbnail 256px JPEG, optimized 1280px WebP, crop 128x128 JPEG), контроль таймаута (JOB_TIMEOUT_SECONDS=30), экспоненциальный backoff при временных сбоях и перенос исчерпавших попытки задач в Dead Letter Queue (attachment_jobs_dlq).
|
||||||
|
|
||||||
## Эксперименты и отказы
|
---
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
## 3. Реализованные метрики, дашборды и алерты
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
### 3.1. Набор метрик
|
||||||
| --- | --- | --- | --- |
|
1. Метрики API (RED и зависимости):
|
||||||
| TODO | TODO | TODO | TODO |
|
* http_server_requests_total (Counter) — общее число HTTP-запросов по методу, маршруту и коду ответа;
|
||||||
|
* http_server_errors_total (Counter) — количество ответов со статусом 4xx и 5xx;
|
||||||
|
* http_server_request_duration_seconds (Histogram) — распределение длительности обработки HTTP-запросов;
|
||||||
|
* http_server_dependency_errors_total (Counter) — счётчик ошибок внешних зависимостей (postgres, redis, s3, outbox, readiness);
|
||||||
|
* db_pool_connections (ObservableGauge) — состояние соединений пула PostgreSQL в разрезе state="used" и state="idle".
|
||||||
|
2. Метрики Worker (очередь, насыщение, длительность):
|
||||||
|
* worker_jobs_total (Counter) — количество завершённых задач по статусу (success, failed);
|
||||||
|
* worker_job_duration_seconds (Histogram) — время обработки задачи воркером;
|
||||||
|
* worker_jobs_in_progress (UpDownCounter) — число задач, обрабатываемых в текущий момент (saturation);
|
||||||
|
* worker_job_retries_total (Counter) — число запланированных повторных попыток по типу исключения (reason);
|
||||||
|
* worker_queue_depth (ObservableGauge) — текущая глубина очереди Redis Stream attachment_jobs;
|
||||||
|
* worker_oldest_job_age_seconds (ObservableGauge) — возраст самой старой необработанной задачи в очереди.
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
### 3.2. Дашборды Grafana
|
||||||
|
Подготовлены и автоматически загружены два дашборда в папку Messenger:
|
||||||
|
* MTUSI Messenger — Lab 6 (Grade 3) (mtusi-messenger-lab6-g3): базовый мониторинг RPS, частоты ошибок, p95 задержки API, скорости и p95 длительности задач воркера, а также поток структурированных логов из Loki.
|
||||||
|
* MTUSI Messenger — Lab 6 (Grade 4) (mtusi-messenger-lab6-g4): расширенный операционный дашборд, включающий:
|
||||||
|
* API RPS (req/s) и API Error Ratio (%);
|
||||||
|
* API Latency p50 / p95 / p99 (s);
|
||||||
|
* DB Connection Pool (used / idle);
|
||||||
|
* Queue Depth & Oldest Job Age;
|
||||||
|
* Worker Saturation & Retries;
|
||||||
|
* Worker Job Duration p50 / p95 / p99 (s);
|
||||||
|
* панель Logs with TraceID (Loki) для быстрой навигации в Tempo.
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
### 3.3. Правила алертинга Prometheus (alerts.yml)
|
||||||
|
Настроены две группы правил с интервалом оценки 15s:
|
||||||
|
* MessengerApiHighErrorRate (warning) — доля ошибок API превышает 5% в течение 30s при нагрузке > 0.2 req/s.
|
||||||
|
* MessengerApiDependencyErrors (critical) — зафиксированы ошибки обращения к PostgreSQL, Redis, S3 или в работе outbox в течение 30s.
|
||||||
|
* MessengerApiHighLatencyP95 (warning) — 95-й перцентиль времени ответа API превышает 1.0s в течение 1m.
|
||||||
|
* MessengerWorkerJobFailures (warning) — появление задач воркера со статусом failed в течение 30s.
|
||||||
|
* MessengerWorkerQueueLag (critical) — возраст самой старой задачи превышает 30s или глубина очереди превышает 20 сообщений в течение 30s.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Проверка работоспособности и сквозной корреляции
|
||||||
|
|
||||||
|
1. Проверка статуса компонентов и приёма телеметрии:
|
||||||
|
* Все контейнеры инфраструктуры (postgres, redis, s3, prometheus, loki, tempo, otel-collector, grafana) и приложения (api-1, api-2, worker, nginx) запущены и проходят проверки healthcheck.
|
||||||
|
* В Prometheus подтверждено поступление временных рядов от otel-collector через Remote Write и активный статус правил в разделе /alerts.
|
||||||
|
2. Сквозной пользовательский сценарий с вложением:
|
||||||
|
* Выполнена регистрация пользователя (POST /api/v1/users), создание сессии (POST /api/v1/auth/sessions), создание чата (POST /api/v1/chats) и загрузка изображения (POST /api/v1/chats/{chat_id}/attachments).
|
||||||
|
* Воркер забрал задачу из потока attachment_jobs, сгенерировал варианты thumbnail, optimized, crop, сохранил их в бакет messenger-attachments в S3 и перевёл статус вложения в ready.
|
||||||
|
3. Проверка связки Metrics -> Logs -> Traces:
|
||||||
|
* По всплеску на графиках Grafana открыта панель логов Loki, где для HTTP-запроса загрузки вложения зафиксирован JSON-лог с полем trace_id.
|
||||||
|
* Переход по ссылке TraceID в Tempo отобразил единый распределённый трейс, начинающийся со спана POST /api/v1/chats/{chat_id}/attachments в сервисе messenger-api и продолжающийся дочерним спаном worker.process_attachment в сервисе messenger-worker.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Диагностика типовых инцидентов по данным телеметрии
|
||||||
|
|
||||||
|
| Сценарий инцидента | Проявление на дашборде и в алертах | Локализация в Loki и Tempo |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| Остановка или деградация worker | Рост worker_queue_depth и линейный рост worker_oldest_job_age_seconds; срабатывание критического алерта MessengerWorkerQueueLag. | В логах messenger-api задачи успешно записываются в outbox и Redis Stream, однако спаны worker.process_attachment в Tempo отсутствуют до момента перезапуска воркера, после чего накопившаяся очередь дренируется. |
|
||||||
|
| Недоступность S3 (SeaweedFS) при загрузке | Всплеск API Error Ratio (%) (ответы 503), рост http_server_dependency_errors_total{component="s3"}, срабатывание MessengerApiDependencyErrors и MessengerApiHighErrorRate. | Поиск в Loki по status_code=503 выявляет логи с trace_id; в Tempo трейс обрывается на спане загрузки в S3 без постановки задачи в outbox. |
|
||||||
|
| Загрузка повреждённого файла изображения | Воркер фиксирует неудачную задачу (worker_jobs_total{status="failed"}), срабатывает алерт MessengerWorkerJobFailures без повторных ретраев (перманентная ошибка). | В логах messenger-worker фиксируется запись "Permanent image error" с error_code="invalid_image" и trace_id, позволяющая отследить исходный запрос загрузки в messenger-api. |
|
||||||
+311
@@ -0,0 +1,311 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:15.5-alpine3.19
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_USER: postgres
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- pgdata:/var/lib/postgresql/data
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.50'
|
||||||
|
memory: 512M
|
||||||
|
reservations:
|
||||||
|
cpus: '0.10'
|
||||||
|
memory: 128M
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"]
|
||||||
|
interval: 3s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.2.4-alpine
|
||||||
|
command: redis-server --appendonly yes --appendfsync everysec --loglevel warning
|
||||||
|
volumes:
|
||||||
|
- redisdata:/data
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.25'
|
||||||
|
memory: 256M
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 3s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
image: postgres:15.5-alpine3.19
|
||||||
|
environment:
|
||||||
|
PGPASSWORD: ${POSTGRES_PASSWORD}
|
||||||
|
volumes:
|
||||||
|
- ./migrations:/migrations
|
||||||
|
command: >
|
||||||
|
sh -c "until pg_isready -h postgres -U postgres; do sleep 1; done;
|
||||||
|
psql -h postgres -U postgres -d messenger -f /migrations/init.sql"
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
s3:
|
||||||
|
image: chrislusf/seaweedfs:3.67
|
||||||
|
command: ["server", "-s3", "-dir=/data", "-s3.port=8333"]
|
||||||
|
environment:
|
||||||
|
- AWS_ACCESS_KEY_ID=appuser
|
||||||
|
- AWS_SECRET_ACCESS_KEY=appsecretpassword
|
||||||
|
volumes:
|
||||||
|
- s3data:/data
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.50'
|
||||||
|
memory: 512M
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/ || test $$? -eq 8 || exit 1"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
s3-init:
|
||||||
|
image: python:3.11.7-slim-bookworm
|
||||||
|
volumes:
|
||||||
|
- ./scripts:/scripts
|
||||||
|
- ./requirements.txt:/requirements.txt
|
||||||
|
environment:
|
||||||
|
- S3_ENDPOINT=http://s3:8333
|
||||||
|
- S3_BUCKET=messenger-attachments
|
||||||
|
- S3_ROOT_ACCESS_KEY=appuser
|
||||||
|
- S3_ROOT_SECRET_KEY=appsecretpassword
|
||||||
|
command: >
|
||||||
|
sh -c "pip install --no-cache-dir -r /requirements.txt &&
|
||||||
|
python /scripts/bootstrap_s3.py"
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v2.51.0
|
||||||
|
command:
|
||||||
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||||
|
- "--storage.tsdb.path=/prometheus"
|
||||||
|
- "--storage.tsdb.retention.time=7d"
|
||||||
|
- "--web.enable-remote-write-receiver"
|
||||||
|
volumes:
|
||||||
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||||
|
- promdata:/prometheus
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.0.0
|
||||||
|
command: ["-config.file=/etc/loki/local-config.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
|
||||||
|
- lokidata:/loki
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3100/ready || exit 1"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
tempo:
|
||||||
|
image: grafana/tempo:2.4.1
|
||||||
|
command: ["-config.file=/etc/tempo.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/tempo/tempo-config.yaml:/etc/tempo.yaml:ro
|
||||||
|
- tempodata:/var/tempo
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3200/ready || exit 1"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
otel-collector:
|
||||||
|
image: otel/opentelemetry-collector-contrib:0.97.0
|
||||||
|
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||||
|
depends_on:
|
||||||
|
- prometheus
|
||||||
|
- loki
|
||||||
|
- tempo
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:10.4.2
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin}
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-admin}
|
||||||
|
- GF_AUTH_ANONYMOUS_ENABLED=false
|
||||||
|
volumes:
|
||||||
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
- grafanadata:/var/lib/grafana
|
||||||
|
depends_on:
|
||||||
|
prometheus:
|
||||||
|
condition: service_healthy
|
||||||
|
loki:
|
||||||
|
condition: service_healthy
|
||||||
|
tempo:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
api-1:
|
||||||
|
build: .
|
||||||
|
stop_grace_period: 15s
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger
|
||||||
|
- REDIS_URL=redis://redis:6379/0
|
||||||
|
- SESSION_TTL_SECONDS=3600
|
||||||
|
- INSTANCE_ID=api-replica-1
|
||||||
|
- SECRET_HMAC_KEY=${SECRET_HMAC_KEY}
|
||||||
|
- S3_ENDPOINT=http://s3:8333
|
||||||
|
- S3_BUCKET=messenger-attachments
|
||||||
|
- S3_ACCESS_KEY_ID=appuser
|
||||||
|
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||||
|
- BROKER_STREAM=attachment_jobs
|
||||||
|
- OTEL_SERVICE_NAME=messenger-api
|
||||||
|
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
|
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||||
|
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.50'
|
||||||
|
memory: 256M
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
api-2:
|
||||||
|
build: .
|
||||||
|
stop_grace_period: 15s
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger
|
||||||
|
- REDIS_URL=redis://redis:6379/0
|
||||||
|
- SESSION_TTL_SECONDS=3600
|
||||||
|
- INSTANCE_ID=api-replica-2
|
||||||
|
- SECRET_HMAC_KEY=${SECRET_HMAC_KEY}
|
||||||
|
- S3_ENDPOINT=http://s3:8333
|
||||||
|
- S3_BUCKET=messenger-attachments
|
||||||
|
- S3_ACCESS_KEY_ID=appuser
|
||||||
|
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||||
|
- BROKER_STREAM=attachment_jobs
|
||||||
|
- OTEL_SERVICE_NAME=messenger-api
|
||||||
|
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
|
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||||
|
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.50'
|
||||||
|
memory: 256M
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: ["python", "worker.py"]
|
||||||
|
stop_grace_period: 15s
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger
|
||||||
|
- REDIS_URL=redis://redis:6379/0
|
||||||
|
- S3_ENDPOINT=http://s3:8333
|
||||||
|
- S3_BUCKET=messenger-attachments
|
||||||
|
- S3_ACCESS_KEY_ID=appuser
|
||||||
|
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||||
|
- BROKER_STREAM=attachment_jobs
|
||||||
|
- BROKER_GROUP=attachment_workers
|
||||||
|
- JOB_MAX_ATTEMPTS=5
|
||||||
|
- MAX_IMAGE_PIXELS=40000000
|
||||||
|
- OTEL_SERVICE_NAME=messenger-worker
|
||||||
|
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
|
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||||
|
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.50'
|
||||||
|
memory: 256M
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.25.4-alpine
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
volumes:
|
||||||
|
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '0.25'
|
||||||
|
memory: 128M
|
||||||
|
depends_on:
|
||||||
|
- api-1
|
||||||
|
- api-2
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "nginx", "-t"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
pgdata:
|
||||||
|
redisdata:
|
||||||
|
s3data:
|
||||||
|
promdata:
|
||||||
|
lokidata:
|
||||||
|
tempodata:
|
||||||
|
grafanadata:
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
{
|
||||||
|
"title": "MTUSI Messenger — Lab 6 (Grade 3)",
|
||||||
|
"uid": "mtusi-messenger-lab6-g3",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 2,
|
||||||
|
"refresh": "5s",
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "API RPS",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(http_server_requests_total[1m]))",
|
||||||
|
"legendFormat": "req/s"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "API Errors (4xx/5xx/s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(http_server_errors_total[1m]))",
|
||||||
|
"legendFormat": "errors/s"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "API Duration p95 (s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "Worker Jobs (success / failed)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(worker_jobs_total[1m])) by (status)",
|
||||||
|
"legendFormat": "{{status}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Worker Job Duration p95 (s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Structured Logs (API & Worker)",
|
||||||
|
"type": "logs",
|
||||||
|
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "{service_name=~\".+\"}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
{
|
||||||
|
"title": "MTUSI Messenger — Lab 6 (Grade 4)",
|
||||||
|
"uid": "mtusi-messenger-lab6-g4",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 3,
|
||||||
|
"refresh": "5s",
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "API RPS (req/s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(http_server_requests_total[1m]))",
|
||||||
|
"legendFormat": "req/s"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "API Error Ratio (%)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percentunit" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(http_server_errors_total[1m])) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.0001)",
|
||||||
|
"legendFormat": "error_ratio"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "API Latency p50 / p95 / p99 (s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "DB Connection Pool (used / idle)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(db_pool_connections) by (state)",
|
||||||
|
"legendFormat": "{{state}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Queue Depth & Oldest Job Age",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(worker_queue_depth)",
|
||||||
|
"legendFormat": "queue_depth (jobs)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "max(worker_oldest_job_age_seconds)",
|
||||||
|
"legendFormat": "oldest_job_age (s)"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Worker Saturation & Retries",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(worker_jobs_in_progress)",
|
||||||
|
"legendFormat": "in_progress (jobs)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "sum(rate(worker_job_retries_total[1m])) by (reason)",
|
||||||
|
"legendFormat": "retries/s: {{reason}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"title": "Worker Job Duration p50 / p95 / p99 (s)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 },
|
||||||
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.50, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"expr": "histogram_quantile(0.99, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"title": "Logs with TraceID (Loki)",
|
||||||
|
"type": "logs",
|
||||||
|
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "{service_name=~\".+\"}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: MTUSI Messenger Lab6
|
||||||
|
orgId: 1
|
||||||
|
folder: Messenger
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
editable: true
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
uid: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: true
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
type: loki
|
||||||
|
uid: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: true
|
||||||
|
jsonData:
|
||||||
|
derivedFields:
|
||||||
|
- datasourceUid: tempo
|
||||||
|
matcherRegex: '"trace_id":"([0-9a-f]{32})"'
|
||||||
|
name: TraceID
|
||||||
|
url: '$${__value.raw}'
|
||||||
|
|
||||||
|
- name: Tempo
|
||||||
|
type: tempo
|
||||||
|
uid: tempo
|
||||||
|
access: proxy
|
||||||
|
url: http://tempo:3200
|
||||||
|
editable: true
|
||||||
|
jsonData:
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
filterByTraceID: true
|
||||||
|
tags:
|
||||||
|
- key: service.name
|
||||||
|
value: service
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2024-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
ruler:
|
||||||
|
alertmanager_url: http://localhost:9093
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
allow_structured_metadata: true
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
otlp_config:
|
||||||
|
default_resource_attributes_as_index_labels:
|
||||||
|
- service.name
|
||||||
|
- service.namespace
|
||||||
|
- deployment.environment
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
processors:
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 256
|
||||||
|
spike_limit_mib: 64
|
||||||
|
batch:
|
||||||
|
send_batch_size: 256
|
||||||
|
timeout: 2s
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: 0.0.0.0:13133
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
prometheusremotewrite:
|
||||||
|
endpoint: http://prometheus:9090/api/v1/write
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
queue_size: 1000
|
||||||
|
otlphttp/tempo:
|
||||||
|
endpoint: http://tempo:4318
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
queue_size: 1000
|
||||||
|
otlphttp/loki:
|
||||||
|
endpoint: http://loki:3100/otlp
|
||||||
|
headers:
|
||||||
|
X-Scope-OrgID: fake
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
queue_size: 1000
|
||||||
|
|
||||||
|
service:
|
||||||
|
extensions: [health_check]
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/tempo]
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [prometheusremotewrite]
|
||||||
|
logs:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/loki, debug]
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
groups:
|
||||||
|
- name: messenger_api_alerts
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
- alert: MessengerApiHighErrorRate
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
sum(rate(http_server_errors_total[1m]))
|
||||||
|
/
|
||||||
|
clamp_min(sum(rate(http_server_requests_total[1m])), 0.001)
|
||||||
|
) > 0.05
|
||||||
|
and sum(rate(http_server_requests_total[1m])) > 0.2
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: messenger-api
|
||||||
|
annotations:
|
||||||
|
summary: "High HTTP error ratio (>5%) in messenger-api"
|
||||||
|
description: "Error ratio exceeded 5% over 30s; inspect Loki logs and Tempo traces."
|
||||||
|
|
||||||
|
- alert: MessengerApiDependencyErrors
|
||||||
|
expr: sum(rate(http_server_dependency_errors_total[1m])) > 0
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: messenger-api
|
||||||
|
annotations:
|
||||||
|
summary: "Dependency errors in messenger-api"
|
||||||
|
description: "PostgreSQL, Redis, S3, or outbox errors occurred."
|
||||||
|
|
||||||
|
- alert: MessengerApiHighLatencyP95
|
||||||
|
expr: histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le)) > 1.0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: messenger-api
|
||||||
|
annotations:
|
||||||
|
summary: "High API p95 latency (>1.0s)"
|
||||||
|
description: "API p95 duration is above 1.0s."
|
||||||
|
|
||||||
|
- name: messenger_worker_alerts
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
- alert: MessengerWorkerJobFailures
|
||||||
|
expr: sum(rate(worker_jobs_total{status="failed"}[1m])) > 0
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: messenger-worker
|
||||||
|
annotations:
|
||||||
|
summary: "Attachment worker jobs failing"
|
||||||
|
description: "Failed worker jobs detected; check error_code and trace_id in Loki/Tempo."
|
||||||
|
|
||||||
|
- alert: MessengerWorkerQueueLag
|
||||||
|
expr: max(worker_oldest_job_age_seconds) > 30 or sum(worker_queue_depth) > 20
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: messenger-worker
|
||||||
|
annotations:
|
||||||
|
summary: "Growing attachment queue lag or backlog"
|
||||||
|
description: "Oldest job age > 30s or queue depth > 20; check worker saturation and dependencies."
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/alerts.yml
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: prometheus
|
||||||
|
static_configs:
|
||||||
|
- targets: ["localhost:9090"]
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 3200
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
ingester:
|
||||||
|
trace_idle_period: 10s
|
||||||
|
max_block_duration: 5m
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
compaction:
|
||||||
|
block_retention: 168h
|
||||||
|
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
local:
|
||||||
|
path: /var/tempo/blocks
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
username VARCHAR(32) UNIQUE NOT NULL,
|
||||||
|
display_name VARCHAR(100) NOT NULL,
|
||||||
|
password_hash VARCHAR(255) NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS chats (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
title VARCHAR(100) NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS chat_members (
|
||||||
|
chat_id UUID REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS messages (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
chat_id UUID REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
sender_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
text VARCHAR(4000) NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL,
|
||||||
|
attachment_ids UUID[] DEFAULT '{}'
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_chat_pagination ON messages(chat_id, created_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||||
|
user_id UUID NOT NULL,
|
||||||
|
method VARCHAR(16) NOT NULL,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
key VARCHAR(128) NOT NULL,
|
||||||
|
request_hash VARCHAR(64) NOT NULL,
|
||||||
|
response_status INT NOT NULL,
|
||||||
|
response_body JSONB NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
PRIMARY KEY (user_id, method, path, key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS attachments (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
chat_id UUID REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
owner_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
filename VARCHAR(255) NOT NULL,
|
||||||
|
content_type VARCHAR(50) NOT NULL,
|
||||||
|
size_bytes BIGINT NOT NULL,
|
||||||
|
sha256 VARCHAR(64) NOT NULL,
|
||||||
|
status VARCHAR(32) NOT NULL DEFAULT 'queued',
|
||||||
|
variants JSONB NOT NULL DEFAULT '[]',
|
||||||
|
error_code VARCHAR(64),
|
||||||
|
pipeline_version INT NOT NULL DEFAULT 1,
|
||||||
|
attempt INT NOT NULL DEFAULT 0,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS outbox (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
stream VARCHAR(64) NOT NULL,
|
||||||
|
payload JSONB NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
published_at TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
|
||||||
|
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}';
|
||||||
|
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
|
||||||
|
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished ON outbox(created_at) WHERE published_at IS NULL;
|
||||||
+28
@@ -0,0 +1,28 @@
|
|||||||
|
events { worker_connections 1024; }
|
||||||
|
|
||||||
|
http {
|
||||||
|
upstream messenger_api {
|
||||||
|
server api-1:8080 max_fails=1 fail_timeout=5s;
|
||||||
|
server api-2:8080 max_fails=1 fail_timeout=5s;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://messenger_api;
|
||||||
|
|
||||||
|
proxy_connect_timeout 2s;
|
||||||
|
proxy_send_timeout 5s;
|
||||||
|
proxy_read_timeout 5s;
|
||||||
|
|
||||||
|
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||||
|
proxy_next_upstream_tries 2;
|
||||||
|
proxy_next_upstream_timeout 5s;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
fastapi==0.110.0
|
||||||
|
uvicorn==0.28.0
|
||||||
|
pydantic==2.6.2
|
||||||
|
starlette==0.36.3
|
||||||
|
idna==3.6
|
||||||
|
anyio==4.3.0
|
||||||
|
sniffio==1.3.1
|
||||||
|
click==8.1.7
|
||||||
|
h11==0.14.0
|
||||||
|
annotated-types==0.6.0
|
||||||
|
psycopg2-binary==2.9.9
|
||||||
|
bcrypt==4.1.2
|
||||||
|
redis==5.0.1
|
||||||
|
boto3==1.34.0
|
||||||
|
Pillow==10.2.0
|
||||||
|
opentelemetry-api==1.24.0
|
||||||
|
opentelemetry-sdk==1.24.0
|
||||||
|
opentelemetry-exporter-otlp-proto-http==1.24.0
|
||||||
|
opentelemetry-instrumentation-fastapi==0.45b0
|
||||||
|
opentelemetry-instrumentation-requests==0.45b0
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import os
|
||||||
|
import boto3
|
||||||
|
from botocore.client import Config
|
||||||
|
|
||||||
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||||
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||||
|
S3_ROOT_ACCESS_KEY = os.getenv("S3_ROOT_ACCESS_KEY", "minioadmin")
|
||||||
|
S3_ROOT_SECRET_KEY = os.getenv("S3_ROOT_SECRET_KEY", "minioadminpassword")
|
||||||
|
|
||||||
|
s3_client = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=S3_ENDPOINT,
|
||||||
|
aws_access_key_id=S3_ROOT_ACCESS_KEY,
|
||||||
|
aws_secret_access_key=S3_ROOT_SECRET_KEY,
|
||||||
|
config=Config(signature_version="s3v4"),
|
||||||
|
region_name="us-east-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
def init_bucket():
|
||||||
|
print(f"Checking/Creating bucket: {S3_BUCKET}")
|
||||||
|
try:
|
||||||
|
s3_client.head_bucket(Bucket=S3_BUCKET)
|
||||||
|
print("Bucket already exists.")
|
||||||
|
except Exception:
|
||||||
|
s3_client.create_bucket(Bucket=S3_BUCKET)
|
||||||
|
print("Bucket successfully created.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
init_bucket()
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import os, sys, urllib.request, urllib.error
|
||||||
|
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||||
|
KEY = os.getenv("S3_KEY")
|
||||||
|
URL = f"http://s3:8333/{BUCKET}/{KEY}"
|
||||||
|
try:
|
||||||
|
urllib.request.urlopen(URL)
|
||||||
|
print("FAIL: object accessible without auth"); sys.exit(1)
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code in (401, 403):
|
||||||
|
print(f"PASS: anonymous GET -> {e.code}"); sys.exit(0)
|
||||||
|
print(f"FAIL: unexpected {e.code}"); sys.exit(1)
|
||||||
@@ -8,8 +8,8 @@ import re
|
|||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number']
|
||||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||||
assert spec['openapi']=='3.1.0'
|
assert spec['openapi']=='3.1.0'
|
||||||
assert spec['info']['version']=='1.%d.0'%lab
|
assert spec['info']['version']=='1.%d.0'%lab
|
||||||
operation_ids = set()
|
operation_ids = set()
|
||||||
@@ -41,11 +41,11 @@ def main():
|
|||||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||||
ast.parse(file.read_text(),filename=str(file))
|
ast.parse(file.read_text(encoding='utf-8'),filename=str(file))
|
||||||
for file in ROOT.rglob('*.json'):
|
for file in ROOT.rglob('*.json'):
|
||||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||||
continue
|
continue
|
||||||
json.loads(file.read_text())
|
json.loads(file.read_text(encoding='utf-8-sig'))
|
||||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||||
assert (ROOT/required).is_file(), 'missing '+required
|
assert (ROOT/required).is_file(), 'missing '+required
|
||||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import boto3
|
||||||
|
from botocore.client import Config
|
||||||
|
import psycopg2
|
||||||
|
import psycopg2.extras
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||||
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||||
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||||
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||||
|
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||||
|
|
||||||
|
s3_client = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=S3_ENDPOINT,
|
||||||
|
aws_access_key_id=S3_ACCESS_KEY,
|
||||||
|
aws_secret_access_key=S3_SECRET_KEY,
|
||||||
|
config=Config(signature_version="s3v4"),
|
||||||
|
region_name="us-east-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
def run_reconciliation(dry_run: bool = False):
|
||||||
|
print(f"Starting reconciliation (dry_run={dry_run})...")
|
||||||
|
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||||
|
|
||||||
|
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||||
|
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("""
|
||||||
|
SELECT id, chat_id, filename, created_at FROM attachments
|
||||||
|
WHERE status = 'pending' AND created_at < %s
|
||||||
|
""", (threshold,))
|
||||||
|
stale_pending = cur.fetchall()
|
||||||
|
|
||||||
|
for row in stale_pending:
|
||||||
|
att_id = str(row['id'])
|
||||||
|
chat_id = str(row['chat_id'])
|
||||||
|
filename = row['filename']
|
||||||
|
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
|
||||||
|
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
try:
|
||||||
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to delete S3 key {s3_key}: {e}")
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("SELECT id, chat_id, filename FROM attachments")
|
||||||
|
all_db_attachments = cur.fetchall()
|
||||||
|
|
||||||
|
valid_s3_keys = set()
|
||||||
|
for row in all_db_attachments:
|
||||||
|
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
|
||||||
|
|
||||||
|
paginator = s3_client.get_paginator('list_objects_v2')
|
||||||
|
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
|
||||||
|
for obj in page.get('Contents', []):
|
||||||
|
key = obj['Key']
|
||||||
|
last_modified = obj['LastModified']
|
||||||
|
|
||||||
|
if key not in valid_s3_keys and last_modified < threshold:
|
||||||
|
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
|
||||||
|
if not dry_run:
|
||||||
|
try:
|
||||||
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||||
|
print(f"Deleted orphan object {key} from S3.")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to delete orphan {key}: {e}")
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
print("Reconciliation completed successfully.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
dry_run_mode = "--dry-run" in sys.argv
|
||||||
|
run_reconciliation(dry_run=dry_run_mode)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
@echo off
|
||||||
|
docker compose restart
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
docker compose restart
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
wait_ready() {
|
||||||
|
svc=$1
|
||||||
|
timeout=60
|
||||||
|
echo "Waiting for $svc readiness (up to ${timeout}s)..."
|
||||||
|
while [ "$timeout" -gt 0 ]; do
|
||||||
|
if docker compose exec -T "$svc" python -c \
|
||||||
|
"import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" 2>/dev/null; then
|
||||||
|
echo "$svc is ready!"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
timeout=$((timeout - 1))
|
||||||
|
done
|
||||||
|
echo "$svc failed to become ready" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Starting Rolling Update"
|
||||||
|
|
||||||
|
echo "1. Rebuilding and replacing api-1..."
|
||||||
|
docker compose up -d --no-deps --build --force-recreate api-1
|
||||||
|
wait_ready api-1
|
||||||
|
echo "Reloading nginx to pick up new api-1 IP..."
|
||||||
|
docker compose exec -T nginx nginx -s reload
|
||||||
|
|
||||||
|
echo "2. Rebuilding and replacing api-2..."
|
||||||
|
docker compose up -d --no-deps --build --force-recreate api-2
|
||||||
|
wait_ready api-2
|
||||||
|
echo "Reloading nginx to pick up new api-2 IP..."
|
||||||
|
docker compose exec -T nginx nginx -s reload
|
||||||
|
|
||||||
|
echo "Rolling Update Completed Successfully"
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||||
|
|
||||||
|
if [ "$TARGET_INSTANCE_ID" = "api-replica-1" ]; then
|
||||||
|
docker compose stop -t 3 api-1
|
||||||
|
elif [ "$TARGET_INSTANCE_ID" = "api-replica-2" ]; then
|
||||||
|
docker compose stop -t 3 api-2
|
||||||
|
else
|
||||||
|
echo "Unknown target instance: $TARGET_INSTANCE_ID" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import base64, concurrent.futures, json, pathlib, time, urllib.request, uuid
|
||||||
|
|
||||||
|
BASE = "http://localhost:8080"
|
||||||
|
|
||||||
|
def _req(method, path, body=None, headers=None, token=None):
|
||||||
|
h = {"Accept": "application/json"}
|
||||||
|
if token: h["Authorization"] = f"Bearer {token}"
|
||||||
|
h.update(headers or {})
|
||||||
|
req = urllib.request.Request(BASE + path, data=body, headers=h, method=method)
|
||||||
|
with urllib.request.urlopen(req) as r:
|
||||||
|
return r.status, r.read()
|
||||||
|
|
||||||
|
def register_and_login():
|
||||||
|
u = f"u_{uuid.uuid4().hex[:10]}"
|
||||||
|
p = "T3st-" + uuid.uuid4().hex
|
||||||
|
_req("POST", "/api/v1/users",
|
||||||
|
json.dumps({"username": u, "display_name": "Load", "password": p}).encode(),
|
||||||
|
{"Content-Type": "application/json"})
|
||||||
|
basic = base64.b64encode(f"{u}:{p}".encode()).decode()
|
||||||
|
_, body = _req("POST", "/api/v1/auth/sessions",
|
||||||
|
headers={"Authorization": f"Basic {basic}"})
|
||||||
|
return json.loads(body)["token"]
|
||||||
|
|
||||||
|
def make_chat(token):
|
||||||
|
_, me = _req("GET", "/api/v1/users/me", token=token)
|
||||||
|
uid = json.loads(me)["id"]
|
||||||
|
_, chat = _req("POST", "/api/v1/chats",
|
||||||
|
json.dumps({"title": "load", "member_ids": [uid]}).encode(),
|
||||||
|
{"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
|
||||||
|
token=token)
|
||||||
|
return json.loads(chat)["id"]
|
||||||
|
|
||||||
|
def main():
|
||||||
|
token = register_and_login()
|
||||||
|
chat = make_chat(token)
|
||||||
|
png = pathlib.Path("tests/fixtures/sample.png").read_bytes()
|
||||||
|
|
||||||
|
def one(_):
|
||||||
|
try:
|
||||||
|
status, _ = _req("POST",
|
||||||
|
f"/api/v1/chats/{chat}/attachments?filename=f.png",
|
||||||
|
png, {"Content-Type": "image/png"}, token)
|
||||||
|
return status
|
||||||
|
except Exception as e:
|
||||||
|
return repr(e)
|
||||||
|
|
||||||
|
t0 = time.monotonic()
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex:
|
||||||
|
results = list(ex.map(one, range(10)))
|
||||||
|
print(json.dumps({"results": results,
|
||||||
|
"elapsed_s": round(time.monotonic() - t0, 3)}))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from opentelemetry import metrics, trace
|
||||||
|
from opentelemetry._logs import set_logger_provider
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||||
|
|
||||||
|
|
||||||
|
class JsonLogFormatter(logging.Formatter):
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
span = trace.get_current_span()
|
||||||
|
ctx = span.get_span_context() if span else None
|
||||||
|
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
||||||
|
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||||
|
"level": record.levelname,
|
||||||
|
"logger": record.name,
|
||||||
|
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-app"),
|
||||||
|
"msg": record.getMessage(),
|
||||||
|
}
|
||||||
|
if trace_id and span_id:
|
||||||
|
payload["trace_id"] = trace_id
|
||||||
|
payload["span_id"] = span_id
|
||||||
|
|
||||||
|
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
||||||
|
if hasattr(record, attr):
|
||||||
|
payload[attr] = getattr(record, attr)
|
||||||
|
|
||||||
|
if record.exc_info:
|
||||||
|
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
||||||
|
|
||||||
|
return json.dumps(payload, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_telemetry(default_service_name: str):
|
||||||
|
service_name = os.getenv("OTEL_SERVICE_NAME", default_service_name)
|
||||||
|
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||||
|
env_name = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
||||||
|
|
||||||
|
attrs = {"service.name": service_name}
|
||||||
|
for part in env_name.split(","):
|
||||||
|
if "=" in part:
|
||||||
|
k, v = part.split("=", 1)
|
||||||
|
attrs[k.strip()] = v.strip()
|
||||||
|
|
||||||
|
resource = Resource.create(attrs)
|
||||||
|
|
||||||
|
tracer_provider = TracerProvider(resource=resource)
|
||||||
|
tracer_provider.add_span_processor(
|
||||||
|
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
||||||
|
)
|
||||||
|
trace.set_tracer_provider(tracer_provider)
|
||||||
|
|
||||||
|
metric_reader = PeriodicExportingMetricReader(
|
||||||
|
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
||||||
|
export_interval_millis=5000,
|
||||||
|
)
|
||||||
|
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
||||||
|
metrics.set_meter_provider(meter_provider)
|
||||||
|
|
||||||
|
logger_provider = LoggerProvider(resource=resource)
|
||||||
|
logger_provider.add_log_record_processor(
|
||||||
|
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
||||||
|
)
|
||||||
|
set_logger_provider(logger_provider)
|
||||||
|
|
||||||
|
root_logger = logging.getLogger()
|
||||||
|
root_logger.setLevel(logging.INFO)
|
||||||
|
root_logger.handlers.clear()
|
||||||
|
|
||||||
|
stream_handler = logging.StreamHandler()
|
||||||
|
stream_handler.setFormatter(JsonLogFormatter())
|
||||||
|
root_logger.addHandler(stream_handler)
|
||||||
|
|
||||||
|
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||||
|
otel_handler.setFormatter(JsonLogFormatter())
|
||||||
|
root_logger.addHandler(otel_handler)
|
||||||
|
|
||||||
|
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||||
+1
-1
@@ -14,7 +14,7 @@ import uuid
|
|||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
|
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
|
||||||
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
|
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||||
|
|
||||||
class Failure(AssertionError):
|
class Failure(AssertionError):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import concurrent.futures
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
from client import Client
|
||||||
|
|
||||||
|
class TestMessengerLab4(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.client = Client()
|
||||||
|
self.alice = self.client.user("alice")
|
||||||
|
self.bob = self.client.user("bob")
|
||||||
|
self.eve = self.client.user("eve")
|
||||||
|
|
||||||
|
res = self.client.call(
|
||||||
|
'POST', '/api/v1/chats', 201,
|
||||||
|
actor=self.alice,
|
||||||
|
body={"title": "Test Chat", "member_ids": [self.alice["id"], self.bob["id"]]},
|
||||||
|
schema='Chat'
|
||||||
|
)
|
||||||
|
self.chat = res['json']
|
||||||
|
|
||||||
|
def test_unauthorized_access_to_chat(self):
|
||||||
|
self.client.call('GET', f"/api/v1/chats/{self.chat['id']}", 404, actor=self.eve)
|
||||||
|
|
||||||
|
def test_empty_message(self):
|
||||||
|
self.client.call('POST', f"/api/v1/chats/{self.chat['id']}/messages", 400, actor=self.alice, body={"text": ""})
|
||||||
|
|
||||||
|
def test_malformed_json(self):
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat['id']}/messages"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=b"{broken json content without quotes",
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"X-User-Id": self.alice["id"]
|
||||||
|
},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
self.fail("Expected HTTP 400 for malformed JSON")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
self.assertEqual(e.code, 400)
|
||||||
|
body = json.loads(e.read().decode())
|
||||||
|
self.assertEqual(body["error"]["code"], "invalid_request")
|
||||||
|
|
||||||
|
def test_duplicate_username(self):
|
||||||
|
self.client.call('POST', '/api/v1/users', 409, body=self.alice["registration"])
|
||||||
|
|
||||||
|
def test_pagination_multiple_pages(self):
|
||||||
|
import time
|
||||||
|
msg_texts = ["msg-page-1", "msg-page-2", "msg-page-3"]
|
||||||
|
for txt in msg_texts:
|
||||||
|
self.client.message(self.alice, self.chat, txt)
|
||||||
|
time.sleep(0.01)
|
||||||
|
|
||||||
|
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||||
|
collected_items = []
|
||||||
|
cursor = None
|
||||||
|
|
||||||
|
for _ in range(5):
|
||||||
|
url_suffix = "?limit=1"
|
||||||
|
if cursor:
|
||||||
|
url_suffix += f"&cursor={cursor}"
|
||||||
|
|
||||||
|
page_res = self.client.call('GET', path + url_suffix, actor=self.alice, schema='MessagePage')
|
||||||
|
page = page_res['json']
|
||||||
|
|
||||||
|
for item in page['items']:
|
||||||
|
self.assertNotIn(item['id'], [x['id'] for x in collected_items])
|
||||||
|
collected_items.append(item)
|
||||||
|
|
||||||
|
cursor = page.get('next_keyword') or page.get('next_cursor')
|
||||||
|
if not cursor:
|
||||||
|
break
|
||||||
|
|
||||||
|
self.assertEqual(len(collected_items), 3)
|
||||||
|
self.assertEqual(sorted([x['text'] for x in collected_items]), sorted(msg_texts))
|
||||||
|
|
||||||
|
def test_concurrent_messages_strict(self):
|
||||||
|
def worker_send(worker_idx):
|
||||||
|
local_client = Client()
|
||||||
|
ids = []
|
||||||
|
for i in range(10):
|
||||||
|
res = local_client.message(self.alice, self.chat, f"w-{worker_idx}-msg-{i}")
|
||||||
|
ids.append(res['json']['id'])
|
||||||
|
return ids
|
||||||
|
|
||||||
|
all_ids = []
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
|
futures = [executor.submit(worker_send, w) for w in range(10)]
|
||||||
|
for f in concurrent.futures.as_completed(futures):
|
||||||
|
all_ids.extend(f.result())
|
||||||
|
|
||||||
|
self.assertEqual(len(all_ids), 100)
|
||||||
|
self.assertEqual(len(set(all_ids)), 100)
|
||||||
|
|
||||||
|
page = self.client.call(
|
||||||
|
'GET', f"/api/v1/chats/{self.chat['id']}/messages?limit=100",
|
||||||
|
actor=self.alice,
|
||||||
|
schema='MessagePage'
|
||||||
|
)
|
||||||
|
server_ids = [m['id'] for m in page['json']['items']]
|
||||||
|
self.assertEqual(len(server_ids), 100)
|
||||||
|
self.assertEqual(len(set(server_ids)), 100)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
from client import Client
|
||||||
|
|
||||||
|
class TestMessengerLevel4Complete(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.client = Client()
|
||||||
|
self.alice = self.client.user("alice")
|
||||||
|
self.bob = self.client.user("bob")
|
||||||
|
self.eve = self.client.user("eve")
|
||||||
|
|
||||||
|
res_chat1 = self.client.call(
|
||||||
|
'POST', '/api/v1/chats', 201,
|
||||||
|
actor=self.alice,
|
||||||
|
body={"title": "Team Chat", "member_ids": [self.alice["id"], self.bob["id"]]},
|
||||||
|
schema='Chat'
|
||||||
|
)
|
||||||
|
self.chat1 = res_chat1['json']
|
||||||
|
|
||||||
|
res_chat2 = self.client.call(
|
||||||
|
'POST', '/api/v1/chats', 201,
|
||||||
|
actor=self.eve,
|
||||||
|
body={"title": "Private Chat", "member_ids": [self.eve["id"], self.bob["id"]]},
|
||||||
|
schema='Chat'
|
||||||
|
)
|
||||||
|
self.chat2 = res_chat2['json']
|
||||||
|
|
||||||
|
def test_unsupported_media_type(self):
|
||||||
|
"""Проверка 415: передача текстового файла под видом картинки."""
|
||||||
|
fake_bytes = b"Just a simple plain text file content."
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=notes.txt"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=fake_bytes,
|
||||||
|
headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
self.fail("Expected HTTP 415")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
self.assertEqual(e.code, 415)
|
||||||
|
|
||||||
|
def test_payload_too_large(self):
|
||||||
|
"""Проверка 413: файл размером больше 10 MiB."""
|
||||||
|
large_data = b'\x89PNG\r\n\x1a\n' + b'\x00' * (10 * 1024 * 1024 + 100)
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=huge.png"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=large_data,
|
||||||
|
headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
self.fail("Expected HTTP 413")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
self.assertEqual(e.code, 413)
|
||||||
|
|
||||||
|
def test_corrupted_image_signature(self):
|
||||||
|
"""Проверка 415: битая структура / неверные размеры PNG."""
|
||||||
|
corrupted_png = b'\x89PNG\r\n\x1a\n' + b'\xFF' * 50
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=corrupt.png"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=corrupted_png,
|
||||||
|
headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
self.fail("Expected HTTP 415")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
self.assertEqual(e.code, 415)
|
||||||
|
|
||||||
|
def test_invalid_filename_characters(self):
|
||||||
|
"""Проверка 400: недопустимые символы (path traversal) в имени файла."""
|
||||||
|
valid_png = (
|
||||||
|
b'\x89PNG\r\n\x1a\n'
|
||||||
|
b'\x00\x00\x00\rIHDR'
|
||||||
|
b'\x00\x00\x00\n\x00\x00\x00\n'
|
||||||
|
b'\x08\x02\x00\x00\x00'
|
||||||
|
b'\x02p\xdd\xb5'
|
||||||
|
b'\x00\x00\x00\x0fcIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q'
|
||||||
|
b'\x00\x00\x00\x00IEND\xaeB`\x82'
|
||||||
|
)
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=../../etc/passwd"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=valid_png,
|
||||||
|
headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
self.fail("Expected HTTP 400")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
self.assertEqual(e.code, 400)
|
||||||
|
|
||||||
|
def test_nonexistent_attachment_id(self):
|
||||||
|
"""Проверка 404: передача несуществующего ID вложения при отправке сообщения."""
|
||||||
|
fake_uuid = "12345678-1234-4123-8123-123456789abc"
|
||||||
|
res = self.client.call(
|
||||||
|
'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404,
|
||||||
|
actor=self.alice,
|
||||||
|
body={"text": "Check this out", "attachment_ids": [fake_uuid]},
|
||||||
|
headers={"Idempotency-Key": "test-key-lvl4-1"}
|
||||||
|
)
|
||||||
|
self.assertEqual(res['status'], 404)
|
||||||
|
|
||||||
|
def test_cross_chat_attachment_isolation(self):
|
||||||
|
"""Проверка 404: попытка прикрепить вложение из чужого чата."""
|
||||||
|
valid_png = (
|
||||||
|
b'\x89PNG\r\n\x1a\n'
|
||||||
|
b'\x00\x00\x00\rIHDR'
|
||||||
|
b'\x00\x00\x00\n\x00\x00\x00\n'
|
||||||
|
b'\x08\x02\x00\x00\x00'
|
||||||
|
b'\x02p\xdd\xb5'
|
||||||
|
b'\x00\x00\x00\x0fcID1Tx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q'
|
||||||
|
b'\x00\x00\x00\x00IEND\xaeB`\x82'
|
||||||
|
)
|
||||||
|
url = self.client.base + f"/api/v1/chats/{self.chat2['id']}/attachments?filename=secret.png"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=valid_png,
|
||||||
|
headers={"Authorization": f"Bearer {self.eve['token']}", "Content-Type": "image/png"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
data = json.loads(resp.read().decode())
|
||||||
|
eve_att_id = data["id"]
|
||||||
|
|
||||||
|
res = self.client.call(
|
||||||
|
'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404,
|
||||||
|
actor=self.alice,
|
||||||
|
body={"text": "Stealing file", "attachment_ids": [eve_att_id]},
|
||||||
|
headers={"Idempotency-Key": "test-key-lvl4-2"}
|
||||||
|
)
|
||||||
|
self.assertEqual(res['status'], 404)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
+1
-1
@@ -55,7 +55,7 @@ def main():
|
|||||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||||
# No shell interpolation and no Docker commands in this test.
|
# No shell interpolation and no Docker commands in this test.
|
||||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
successes,failures,after_instances = [],0,[]
|
successes,failures,after_instances = [],0,[]
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -0,0 +1,628 @@
|
|||||||
|
import os
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import signal
|
||||||
|
import random
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
import psycopg2.extras
|
||||||
|
import redis
|
||||||
|
import boto3
|
||||||
|
import botocore.exceptions
|
||||||
|
from botocore.client import Config
|
||||||
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||||
|
|
||||||
|
from opentelemetry import metrics, trace
|
||||||
|
from opentelemetry._logs import set_logger_provider
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||||
|
from opentelemetry.propagate import extract
|
||||||
|
|
||||||
|
|
||||||
|
class JsonLogFormatter(logging.Formatter):
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
span = trace.get_current_span()
|
||||||
|
ctx = span.get_span_context() if span else None
|
||||||
|
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
||||||
|
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||||
|
"level": record.levelname,
|
||||||
|
"logger": record.name,
|
||||||
|
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"),
|
||||||
|
"msg": record.getMessage(),
|
||||||
|
}
|
||||||
|
if trace_id and span_id:
|
||||||
|
payload["trace_id"] = trace_id
|
||||||
|
payload["span_id"] = span_id
|
||||||
|
|
||||||
|
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
||||||
|
if hasattr(record, attr):
|
||||||
|
payload[attr] = getattr(record, attr)
|
||||||
|
|
||||||
|
if record.exc_info:
|
||||||
|
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
||||||
|
|
||||||
|
return json.dumps(payload, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_worker_telemetry():
|
||||||
|
service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker")
|
||||||
|
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||||
|
env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
||||||
|
|
||||||
|
attrs = {"service.name": service_name}
|
||||||
|
for part in env_attrs.split(","):
|
||||||
|
if "=" in part:
|
||||||
|
k, v = part.split("=", 1)
|
||||||
|
attrs[k.strip()] = v.strip()
|
||||||
|
|
||||||
|
resource = Resource.create(attrs)
|
||||||
|
|
||||||
|
tracer_provider = TracerProvider(resource=resource)
|
||||||
|
tracer_provider.add_span_processor(
|
||||||
|
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
||||||
|
)
|
||||||
|
trace.set_tracer_provider(tracer_provider)
|
||||||
|
|
||||||
|
metric_reader = PeriodicExportingMetricReader(
|
||||||
|
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
||||||
|
export_interval_millis=5000,
|
||||||
|
)
|
||||||
|
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
||||||
|
metrics.set_meter_provider(meter_provider)
|
||||||
|
|
||||||
|
logger_provider = LoggerProvider(resource=resource)
|
||||||
|
logger_provider.add_log_record_processor(
|
||||||
|
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
||||||
|
)
|
||||||
|
set_logger_provider(logger_provider)
|
||||||
|
|
||||||
|
root_logger = logging.getLogger()
|
||||||
|
root_logger.setLevel(logging.INFO)
|
||||||
|
root_logger.handlers.clear()
|
||||||
|
|
||||||
|
stream_handler = logging.StreamHandler()
|
||||||
|
stream_handler.setFormatter(JsonLogFormatter())
|
||||||
|
root_logger.addHandler(stream_handler)
|
||||||
|
|
||||||
|
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||||
|
otel_handler.setFormatter(JsonLogFormatter())
|
||||||
|
root_logger.addHandler(otel_handler)
|
||||||
|
|
||||||
|
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||||
|
|
||||||
|
|
||||||
|
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||||
|
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||||
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||||
|
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||||
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||||
|
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||||
|
|
||||||
|
STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs")
|
||||||
|
DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq"
|
||||||
|
GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers")
|
||||||
|
CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1")
|
||||||
|
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||||
|
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||||
|
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
||||||
|
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
|
||||||
|
|
||||||
|
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
|
||||||
|
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||||
|
|
||||||
|
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
||||||
|
s3_client = boto3.client(
|
||||||
|
"s3", endpoint_url=S3_ENDPOINT,
|
||||||
|
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
|
||||||
|
config=Config(signature_version="s3v4"), region_name="us-east-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def observe_queue_metrics(options):
|
||||||
|
try:
|
||||||
|
q_len = redis_client.xlen(STREAM_NAME)
|
||||||
|
entries = redis_client.xrange(STREAM_NAME, count=1)
|
||||||
|
oldest_age = 0.0
|
||||||
|
if entries:
|
||||||
|
msg_id, fields = entries[0]
|
||||||
|
try:
|
||||||
|
payload = json.loads(fields.get("payload", "{}"))
|
||||||
|
created_str = payload.get("created_at")
|
||||||
|
if created_str:
|
||||||
|
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
||||||
|
oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
||||||
|
except Exception:
|
||||||
|
ms = int(msg_id.split("-")[0])
|
||||||
|
oldest_age = max(0.0, time.time() - (ms / 1000.0))
|
||||||
|
yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME})
|
||||||
|
except Exception:
|
||||||
|
yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME})
|
||||||
|
|
||||||
|
|
||||||
|
def observe_oldest_job_age(options):
|
||||||
|
try:
|
||||||
|
entries = redis_client.xrange(STREAM_NAME, count=1)
|
||||||
|
if not entries:
|
||||||
|
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
||||||
|
return
|
||||||
|
msg_id, fields = entries[0]
|
||||||
|
try:
|
||||||
|
payload = json.loads(fields.get("payload", "{}"))
|
||||||
|
created_str = payload.get("created_at")
|
||||||
|
if created_str:
|
||||||
|
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
||||||
|
age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
||||||
|
yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME})
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
ms = int(msg_id.split("-")[0])
|
||||||
|
yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME})
|
||||||
|
except Exception:
|
||||||
|
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
||||||
|
|
||||||
|
|
||||||
|
tracer, meter = setup_worker_telemetry()
|
||||||
|
logger = logging.getLogger("attachment-worker")
|
||||||
|
|
||||||
|
worker_jobs_counter = meter.create_counter(
|
||||||
|
name="worker_jobs_total",
|
||||||
|
description="Completed and failed attachment jobs",
|
||||||
|
unit="1",
|
||||||
|
)
|
||||||
|
worker_retries_counter = meter.create_counter(
|
||||||
|
name="worker_job_retries_total",
|
||||||
|
description="Transient retry attempts scheduled for attachment jobs",
|
||||||
|
unit="1",
|
||||||
|
)
|
||||||
|
worker_duration_histogram = meter.create_histogram(
|
||||||
|
name="worker_job_duration_seconds",
|
||||||
|
description="Attachment job processing duration in seconds",
|
||||||
|
unit="s",
|
||||||
|
)
|
||||||
|
worker_jobs_in_progress = meter.create_up_down_counter(
|
||||||
|
name="worker_jobs_in_progress",
|
||||||
|
description="Jobs currently being processed",
|
||||||
|
unit="1",
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
name="worker_queue_depth",
|
||||||
|
callbacks=[observe_queue_metrics],
|
||||||
|
description="Number of messages currently in attachment jobs stream",
|
||||||
|
unit="1",
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
name="worker_oldest_job_age_seconds",
|
||||||
|
callbacks=[observe_oldest_job_age],
|
||||||
|
description="Age of the oldest message in attachment jobs stream in seconds",
|
||||||
|
unit="s",
|
||||||
|
)
|
||||||
|
|
||||||
|
running = True
|
||||||
|
|
||||||
|
|
||||||
|
def handle_sigterm(signum, frame):
|
||||||
|
global running
|
||||||
|
logger.info("Received shutdown signal, finishing in-flight jobs...")
|
||||||
|
running = False
|
||||||
|
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, handle_sigterm)
|
||||||
|
signal.signal(signal.SIGINT, handle_sigterm)
|
||||||
|
|
||||||
|
|
||||||
|
def get_db_connection():
|
||||||
|
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_consumer_groups():
|
||||||
|
while running:
|
||||||
|
try:
|
||||||
|
redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
||||||
|
redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True)
|
||||||
|
logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME)
|
||||||
|
return
|
||||||
|
except redis.exceptions.ResponseError as e:
|
||||||
|
if "BUSYGROUP" in str(e):
|
||||||
|
return
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Waiting for Redis: %s", type(e).__name__)
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_base_image(raw_bytes):
|
||||||
|
img = Image.open(io.BytesIO(raw_bytes))
|
||||||
|
img = ImageOps.exif_transpose(img)
|
||||||
|
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||||
|
rgba = img.convert("RGBA")
|
||||||
|
bg = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||||
|
bg.paste(rgba, mask=rgba.split()[3])
|
||||||
|
img = bg
|
||||||
|
elif img.mode != "RGB":
|
||||||
|
img = img.convert("RGB")
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def generate_grade4_variants(raw_bytes):
|
||||||
|
base_img = prepare_base_image(raw_bytes)
|
||||||
|
orig_w, orig_h = base_img.size
|
||||||
|
meta = []
|
||||||
|
blobs = {}
|
||||||
|
|
||||||
|
thumb = base_img.copy()
|
||||||
|
if orig_w <= 256 and orig_h <= 256:
|
||||||
|
t_w, t_h = orig_w, orig_h
|
||||||
|
else:
|
||||||
|
s = min(256.0 / orig_w, 256.0 / orig_h)
|
||||||
|
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||||
|
if (t_w, t_h) != (orig_w, orig_h):
|
||||||
|
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
|
||||||
|
buf = io.BytesIO()
|
||||||
|
thumb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||||
|
tb = buf.getvalue()
|
||||||
|
blobs["thumbnail"] = tb
|
||||||
|
meta.append({
|
||||||
|
"name": "thumbnail", "content_type": "image/jpeg",
|
||||||
|
"size_bytes": len(tb), "width": t_w, "height": t_h,
|
||||||
|
"sha256": hashlib.sha256(tb).hexdigest()
|
||||||
|
})
|
||||||
|
|
||||||
|
opt = base_img.copy()
|
||||||
|
if orig_w <= 1280 and orig_h <= 1280:
|
||||||
|
o_w, o_h = orig_w, orig_h
|
||||||
|
else:
|
||||||
|
s = min(1280.0 / orig_w, 1280.0 / orig_h)
|
||||||
|
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||||
|
if (o_w, o_h) != (orig_w, orig_h):
|
||||||
|
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
|
||||||
|
buf = io.BytesIO()
|
||||||
|
opt.save(buf, format="WEBP", quality=85)
|
||||||
|
ob = buf.getvalue()
|
||||||
|
blobs["optimized"] = ob
|
||||||
|
meta.append({
|
||||||
|
"name": "optimized", "content_type": "image/webp",
|
||||||
|
"size_bytes": len(ob), "width": o_w, "height": o_h,
|
||||||
|
"sha256": hashlib.sha256(ob).hexdigest()
|
||||||
|
})
|
||||||
|
|
||||||
|
c = min(orig_w, orig_h)
|
||||||
|
left, top = (orig_w - c) // 2, (orig_h - c) // 2
|
||||||
|
crop = base_img.crop((left, top, left + c, top + c))
|
||||||
|
if c != 128:
|
||||||
|
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
|
||||||
|
buf = io.BytesIO()
|
||||||
|
crop.save(buf, format="JPEG", quality=85, optimize=True)
|
||||||
|
cb = buf.getvalue()
|
||||||
|
blobs["crop"] = cb
|
||||||
|
meta.append({
|
||||||
|
"name": "crop", "content_type": "image/jpeg",
|
||||||
|
"size_bytes": len(cb), "width": 128, "height": 128,
|
||||||
|
"sha256": hashlib.sha256(cb).hexdigest()
|
||||||
|
})
|
||||||
|
|
||||||
|
return meta, blobs
|
||||||
|
|
||||||
|
|
||||||
|
def mark_failed(attachment_id, error_code):
|
||||||
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||||
|
conn = get_db_connection()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
|
||||||
|
"WHERE id=%s AND status != 'ready'",
|
||||||
|
(error_code, now, attachment_id)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
TRANSIENT_ERRORS = (
|
||||||
|
TimeoutError,
|
||||||
|
botocore.exceptions.EndpointConnectionError,
|
||||||
|
botocore.exceptions.ReadTimeoutError,
|
||||||
|
psycopg2.OperationalError,
|
||||||
|
redis.exceptions.ConnectionError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_job_outcome(span, status: str, elapsed: float):
|
||||||
|
span.set_attribute("job.outcome", status)
|
||||||
|
worker_jobs_counter.add(1, {"status": status})
|
||||||
|
worker_duration_histogram.record(elapsed, {"status": status})
|
||||||
|
|
||||||
|
|
||||||
|
def process_job(message_id, fields):
|
||||||
|
job_start = time.monotonic()
|
||||||
|
try:
|
||||||
|
job = json.loads(fields.get("payload", "{}"))
|
||||||
|
except Exception:
|
||||||
|
logger.error("Malformed job %s, ack to drop", message_id)
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
ctx = extract(job)
|
||||||
|
|
||||||
|
with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span:
|
||||||
|
worker_jobs_in_progress.add(1)
|
||||||
|
try:
|
||||||
|
attachment_id = job.get("attachment_id")
|
||||||
|
job_id = job.get("job_id")
|
||||||
|
pipeline_version = int(job.get("pipeline_version", 1))
|
||||||
|
attempt = int(job.get("attempt", 1))
|
||||||
|
|
||||||
|
span.set_attribute("job.pipeline_version", pipeline_version)
|
||||||
|
span.set_attribute("job.attempt", attempt)
|
||||||
|
|
||||||
|
if not attachment_id:
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
return
|
||||||
|
|
||||||
|
conn = get_db_connection()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||||
|
att = cur.fetchone()
|
||||||
|
if not att:
|
||||||
|
conn.commit()
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
return
|
||||||
|
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||||
|
conn.commit()
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
||||||
|
return
|
||||||
|
if att["status"] == "failed":
|
||||||
|
conn.commit()
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
return
|
||||||
|
|
||||||
|
if att["status"] == "processing":
|
||||||
|
updated = att["updated_at"]
|
||||||
|
if isinstance(updated, str):
|
||||||
|
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||||
|
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||||
|
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||||
|
conn.commit()
|
||||||
|
logger.info(
|
||||||
|
"Attachment already processing, ack duplicate",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||||
|
)
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||||
|
(attempt, now, attachment_id)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
chat_id = str(att["chat_id"])
|
||||||
|
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||||
|
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||||
|
|
||||||
|
start_time = time.monotonic()
|
||||||
|
|
||||||
|
def check_timeout():
|
||||||
|
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||||
|
raise TimeoutError("Job exceeded time limit")
|
||||||
|
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||||
|
except Exception:
|
||||||
|
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||||
|
raw_bytes = obj["Body"].read()
|
||||||
|
check_timeout()
|
||||||
|
|
||||||
|
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||||
|
raise ValueError("Staging checksum mismatch")
|
||||||
|
|
||||||
|
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
|
||||||
|
check_timeout()
|
||||||
|
|
||||||
|
s3_client.put_object(
|
||||||
|
Bucket=S3_BUCKET, Key=final_original_key,
|
||||||
|
Body=raw_bytes, ContentType=att["content_type"]
|
||||||
|
)
|
||||||
|
for var_name, var_bytes in variants_bytes.items():
|
||||||
|
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||||
|
s3_client.put_object(
|
||||||
|
Bucket=S3_BUCKET,
|
||||||
|
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||||
|
Body=var_bytes, ContentType=ct
|
||||||
|
)
|
||||||
|
check_timeout()
|
||||||
|
|
||||||
|
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||||
|
conn = get_db_connection()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("""
|
||||||
|
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||||
|
error_code=NULL, updated_at=%s
|
||||||
|
WHERE id=%s AND pipeline_version <= %s
|
||||||
|
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
||||||
|
logger.info(
|
||||||
|
"Job completed: attachment ready",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
except (UnidentifiedImageError, Image.DecompressionBombError):
|
||||||
|
logger.error(
|
||||||
|
"Permanent image error",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
|
||||||
|
)
|
||||||
|
mark_failed(attachment_id, "invalid_image")
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
|
||||||
|
except ValueError as e:
|
||||||
|
logger.error(
|
||||||
|
"Checksum error",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
|
||||||
|
)
|
||||||
|
mark_failed(attachment_id, "invalid_checksum")
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
|
||||||
|
except psycopg2.errors.DataError:
|
||||||
|
logger.error(
|
||||||
|
"DB data error",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
|
||||||
|
)
|
||||||
|
mark_failed(attachment_id, "data_error")
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
|
||||||
|
except TRANSIENT_ERRORS as e:
|
||||||
|
logger.warning(
|
||||||
|
"Transient error during job processing",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||||
|
)
|
||||||
|
if attempt >= JOB_MAX_ATTEMPTS:
|
||||||
|
mark_failed(attachment_id, "retry_exhausted")
|
||||||
|
try:
|
||||||
|
dlq_job = dict(job)
|
||||||
|
dlq_job["attempt"] = 1
|
||||||
|
redis_client.xadd(
|
||||||
|
DLQ_STREAM_NAME,
|
||||||
|
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
else:
|
||||||
|
worker_retries_counter.add(1, {"reason": type(e).__name__})
|
||||||
|
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||||
|
time.sleep(delay)
|
||||||
|
next_job = dict(job)
|
||||||
|
next_job["attempt"] = attempt + 1
|
||||||
|
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.error(
|
||||||
|
"Unknown error during job processing",
|
||||||
|
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
|
||||||
|
)
|
||||||
|
mark_failed(attachment_id, "processing_failed")
|
||||||
|
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||||
|
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||||
|
finally:
|
||||||
|
worker_jobs_in_progress.add(-1)
|
||||||
|
|
||||||
|
|
||||||
|
def run_staging_cleanup():
|
||||||
|
try:
|
||||||
|
cutoff = datetime.now(timezone.utc).timestamp() - 3600
|
||||||
|
paginator = s3_client.get_paginator("list_objects_v2")
|
||||||
|
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"):
|
||||||
|
for obj in page.get("Contents", []):
|
||||||
|
if obj["LastModified"].timestamp() < cutoff:
|
||||||
|
try:
|
||||||
|
s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("Staging cleanup: %s", type(e).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def safe_process(msg_id, fields):
|
||||||
|
try:
|
||||||
|
process_job(msg_id, fields)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ensure_consumer_groups()
|
||||||
|
logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY)
|
||||||
|
|
||||||
|
executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY)
|
||||||
|
last_claim = 0
|
||||||
|
last_cleanup = 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
while running:
|
||||||
|
try:
|
||||||
|
now = time.time()
|
||||||
|
|
||||||
|
if now - last_claim > 15:
|
||||||
|
last_claim = now
|
||||||
|
try:
|
||||||
|
claimed = redis_client.xautoclaim(
|
||||||
|
STREAM_NAME, GROUP_NAME, CONSUMER_NAME,
|
||||||
|
min_idle_time=CLAIM_MIN_IDLE_MS,
|
||||||
|
start_id="0-0", count=WORKER_CONCURRENCY
|
||||||
|
)
|
||||||
|
if claimed and len(claimed) > 1 and claimed[1]:
|
||||||
|
for msg_id, fields in claimed[1]:
|
||||||
|
if not running:
|
||||||
|
break
|
||||||
|
executor.submit(safe_process, msg_id, fields)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if now - last_cleanup > 600:
|
||||||
|
last_cleanup = now
|
||||||
|
executor.submit(run_staging_cleanup)
|
||||||
|
|
||||||
|
resp = redis_client.xreadgroup(
|
||||||
|
GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"},
|
||||||
|
count=WORKER_CONCURRENCY, block=2000
|
||||||
|
)
|
||||||
|
if not resp:
|
||||||
|
continue
|
||||||
|
for _, messages in resp:
|
||||||
|
for msg_id, fields in messages:
|
||||||
|
if not running:
|
||||||
|
break
|
||||||
|
executor.submit(safe_process, msg_id, fields)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Worker loop: %s", type(e).__name__)
|
||||||
|
time.sleep(1)
|
||||||
|
finally:
|
||||||
|
logger.info("Shutting down, waiting for in-flight jobs...")
|
||||||
|
executor.shutdown(wait=True)
|
||||||
|
logger.info("Worker stopped cleanly")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user