forked from mtusi2026/mtusi2026lab6
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d34a03c80 | ||
|
|
ad8db1f32b |
@@ -18,3 +18,10 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
.git
|
||||
.env
|
||||
.venv
|
||||
__pycache__
|
||||
.pytest_cache
|
||||
*.pyc
|
||||
evidence
|
||||
+5
-5
@@ -23,8 +23,8 @@ WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=change-me
|
||||
RABBITMQ_QUEUE=image_jobs
|
||||
IMAGE_MAX_ATTEMPTS=5
|
||||
GRAFANA_ADMIN_PASSWORD=replace_me
|
||||
@@ -30,3 +30,8 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
.env
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
evidence/*.tmp
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY src/ ./src/
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
ENV HOST=0.0.0.0
|
||||
ENV PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||
@@ -1,37 +1,329 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Матвиенко Павел, БПИ2304
|
||||
- **Целевая оценка; выбранный трек 5, если нужен:** 3.
|
||||
- **Commit/PR:** ad8db1f32b898ae43df22b1d3c0a431e553beff8; https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/71
|
||||
- **Использованные источники, библиотеки и помощь ИИ:** задание и контракты лабораторной №6, документация OpenTelemetry, Prometheus, Loki, Tempo, Grafana, FastAPI, Python SDK; при разработке и отладке использовалась помощь ИИ.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Среда:** Windows с WSL Ubuntu и Docker Desktop с поддержкой Docker Compose.
|
||||
|
||||
**Необходимое ПО:** Git, Docker Engine / Docker Desktop, Docker Compose, Python 3, GNU Make.
|
||||
|
||||
**Основные версии компонентов:**
|
||||
|
||||
| Компонент | Версия |
|
||||
| --- | --- |
|
||||
| PostgreSQL | 16-alpine |
|
||||
| RustFS | 1.0.3 |
|
||||
| RabbitMQ | 4.1.8-management |
|
||||
| Nginx | 1.29-alpine |
|
||||
| OpenTelemetry Collector Contrib | 0.136.0 |
|
||||
| OpenTelemetry Python SDK | 1.38.0 |
|
||||
| OpenTelemetry FastAPI instrumentation | 0.59b0 |
|
||||
| Prometheus | 3.5.0 |
|
||||
| Loki | 3.5.5 |
|
||||
| Tempo | 2.8.2 |
|
||||
| Grafana | 12.1.1 |
|
||||
|
||||
**Подготовка окружения:**
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
В `.env` необходимо установить собственные локальные значения паролей PostgreSQL, RabbitMQ, администратора Grafana и ключей S3. Для остальных параметров использовать значения и комментарии из `.env.example`.
|
||||
|
||||
Файл `.env` не должен попадать в Git. Секреты в отчёте не публикуются.
|
||||
|
||||
**Сборка и запуск:**
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
При запуске через Compose выполняются предусмотренные конфигурацией миграции PostgreSQL и инициализация приватного S3 bucket.
|
||||
|
||||
Проверить состояние:
|
||||
|
||||
```bash
|
||||
docker compose ps -a
|
||||
```
|
||||
|
||||
Проверить готовность бизнес-API:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8080/health/ready
|
||||
```
|
||||
|
||||
Ожидаемый результат:
|
||||
|
||||
```json
|
||||
{"status":"ok"}
|
||||
```
|
||||
|
||||
Проверить публичный контракт:
|
||||
|
||||
```bash
|
||||
make test
|
||||
```
|
||||
|
||||
Проверить корреляцию логов и трасс:
|
||||
|
||||
```bash
|
||||
docker compose exec -T api1 python - < scripts/test-trace-correlation.py
|
||||
```
|
||||
|
||||
**Доступ к Grafana:** `http://localhost:3000`. Логин администратора — `admin`; пароль задаётся через `GRAFANA_ADMIN_PASSWORD`.
|
||||
|
||||
Dashboard автоматически импортируется из файла `observability/grafana/dashboards/messenger-lab6.json`.
|
||||
|
||||
**Безопасная остановка с сохранением данных:**
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
|
||||
**Намеренное удаление всех учебных данных и volumes:**
|
||||
|
||||
```bash
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
Последняя команда используется только при осознанном сбросе стенда, поскольку удаляет сохранённые данные приложений и подсистемы наблюдаемости.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
**Схема процессов и хранилищ:**
|
||||
|
||||
```text
|
||||
Client
|
||||
|
|
||||
Nginx
|
||||
|
|
||||
+------ API1 / API2 ------ PostgreSQL
|
||||
| |
|
||||
| +---------- RustFS (S3)
|
||||
|
|
||||
Dispatcher ---- RabbitMQ ---- Worker
|
||||
| |
|
||||
+------------+-------------+
|
||||
|
|
||||
OTLP / HTTP
|
||||
|
|
||||
OpenTelemetry Collector
|
||||
|
|
||||
+-------+-------+
|
||||
| | |
|
||||
Prometheus Loki Tempo
|
||||
| | |
|
||||
+-------+-------+
|
||||
|
|
||||
Grafana
|
||||
```
|
||||
|
||||
**Что перенесено из lab5:**
|
||||
|
||||
- Две реплики FastAPI за Nginx.
|
||||
- Авторизация, пользователи, чаты, сообщения и вложения.
|
||||
- PostgreSQL для транзакционных данных, сессий и заданий.
|
||||
- RustFS как S3-совместимое хранилище изображений.
|
||||
- RabbitMQ с устойчивой очередью `image_jobs`.
|
||||
- Отдельные процессы dispatcher и worker.
|
||||
- Асинхронная обработка PNG/JPEG и создание thumbnail.
|
||||
- Механизмы восстановления заданий, обработки повторных доставок и миграции БД.
|
||||
|
||||
**Что добавлено в lab6:**
|
||||
|
||||
- OpenTelemetry SDK для API и worker.
|
||||
- Автоматические HTTP server spans FastAPI.
|
||||
- Consumer spans обработки изображений.
|
||||
- Счётчики HTTP-запросов и обработки заданий.
|
||||
- Гистограммы длительности HTTP-запросов и работы worker.
|
||||
- Структурированные логи с `trace_id`, `span_id` и `service.name`.
|
||||
- Отдельный OpenTelemetry Collector.
|
||||
- Prometheus, Loki, Tempo и Grafana.
|
||||
- Provisioning источников данных и импортируемого dashboard.
|
||||
- Скрипты проверки корреляции и контролируемых инцидентов.
|
||||
|
||||
**Назначение хранилищ и источник истины:**
|
||||
|
||||
PostgreSQL остаётся источником истины для пользователей, сообщений, метаданных и состояний заданий. RustFS хранит бинарные объекты изображений. RabbitMQ доставляет задания worker.
|
||||
|
||||
Prometheus хранит временные ряды метрик; Loki — структурированные логи; Tempo — трассы. Grafana обеспечивает отображение и поиск данных, но не заменяет эти хранилища. Collector принимает, обрабатывает и экспортирует телеметрию, не являясь долговременной базой данных.
|
||||
|
||||
**Транзакционные границы:**
|
||||
|
||||
Принятие изображения фиксируется сохранением исходных байтов в RustFS и записи задания в PostgreSQL. Передача задания в RabbitMQ допускает повторную доставку. Worker проверяет состояние задания и сохраняет результат до подтверждения обработки сообщения брокеру.
|
||||
|
||||
Между PostgreSQL и S3 отсутствует единая распределённая транзакция. Используются повторные попытки и идемпотентная обработка.
|
||||
|
||||
**Ресурсы и конфигурация OpenTelemetry:**
|
||||
|
||||
Для API применяется `service.name=messenger-api`, для worker — `service.name=messenger-worker`. Используются также `service.version`, `service.instance.id` и `deployment.environment.name=lab`.
|
||||
|
||||
API и worker отправляют OTLP/HTTP в Collector на `otel-collector:4318`. Collector экспортирует метрики в Prometheus-compatible endpoint, логи через OTLP/HTTP в Loki, трассы через OTLP/gRPC в Tempo.
|
||||
|
||||
Пароли, Authorization, session token, тексты сообщений и байты изображений не включаются в специально создаваемые события телеметрии. Пользовательские UUID не используются как labels метрик. Для HTTP-маршрутов используются шаблоны, а не фактические URL с идентификаторами.
|
||||
|
||||
**Фактические имена метрик:**
|
||||
|
||||
| Логический сигнал | Имя в Prometheus | Единица |
|
||||
| --- | --- | --- |
|
||||
| HTTP requests | `messenger_messenger_http_requests_total` | запросы |
|
||||
| HTTP duration | `messenger_http_server_duration_milliseconds_bucket` | миллисекунды |
|
||||
| Jobs processed / failed | `messenger_messenger_worker_jobs_processed_total` | попытки |
|
||||
| Job processing duration | `messenger_messenger_worker_job_duration_seconds_bucket` | секунды |
|
||||
|
||||
Для API error ratio числитель — HTTP 5xx, знаменатель — все учитываемые бизнес-запросы, без health checks. Значения p95 рассчитываются из гистограмм с агрегацией buckets. Миллисекунды HTTP duration преобразуются в секунды.
|
||||
|
||||
**Ограничения архитектуры:**
|
||||
|
||||
Сквозной trace context между HTTP upload и обработкой задания worker не реализован. Логи API и worker коррелируют со своими реальными трассами, но не образуют единую parent-child трассу через RabbitMQ.
|
||||
|
||||
Возможности дополнительных метрик очереди, насыщения ресурсов и alert rules относятся к более высокому уровню и в текущей реализации не заявлены.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `make test` | PASS, бизнес-API работает с включённой телеметрией | `evidence/smoke.txt` |
|
||||
| Smoke после инцидентов | `make test` | PASS после восстановления PostgreSQL и worker | `evidence/smoke-after-incidents.txt` |
|
||||
| Collector и хранилища трёх сигналов | `docker compose ps -a` | Collector healthy; Prometheus, Loki, Tempo и Grafana запущены | `evidence/containers.txt` |
|
||||
| HTTP request counter | Запрос Prometheus API | Найдена `messenger_messenger_http_requests_total` | `evidence/prometheus-metrics.json` |
|
||||
| HTTP duration histogram | Запрос Prometheus API | Найдены buckets, sum и count HTTP duration | `evidence/prometheus-metrics.json` |
|
||||
| Worker jobs counter | Выполнение `make test` и запрос Prometheus API | Найден счётчик заданий с outcome | `evidence/prometheus-metrics.json` |
|
||||
| Worker duration histogram | Выполнение `make test` и запрос Prometheus API | Найдена гистограмма длительности worker | `evidence/prometheus-metrics.json` |
|
||||
| Структурированные логи API | Запрос Loki / Grafana Explore | Найдены HTTP-события с техническими атрибутами | `evidence/grafana-dashboard.png` |
|
||||
| Структурированные логи worker | Журналы worker и запрос Loki | Найдено событие `Image job handled` | `evidence/worker-telemetry.txt` |
|
||||
| Реальная трасса API | Loki → Tempo по trace ID | PASS, 4 span, включая server span | `evidence/trace-correlation.txt` |
|
||||
| Реальная трасса worker | Loki → Tempo по trace ID | PASS, consumer span `image_job.process` | `evidence/trace-correlation.txt` |
|
||||
| Импортируемый dashboard | Provisioning Grafana | Dashboard загружен, 8 панелей | `observability/grafana/dashboards/messenger-lab6.json` |
|
||||
| Контролируемый отказ worker | Остановка worker, upload, восстановление | PASS; задание сохранено, затем обработано | `evidence/incident-worker.txt` |
|
||||
| Контролируемый отказ PostgreSQL | `python3 scripts/test-db-incident.py` | 5/5 неудачных запросов, восстановление HTTP 201 | `evidence/incident-db.txt` |
|
||||
| Критерии уровня 4 | Не заявлены | Дополнительные метрики и alerts не реализованы | Не применяется |
|
||||
| Трек уровня 5 | Не заявлен | Не выполнялся | Не применяется |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
**Подтверждённая корреляция API:**
|
||||
|
||||
- Service: `messenger-api`
|
||||
- Event: `HTTP request completed`
|
||||
- Trace ID: `c3ea28d96d7c536e222d8e4c7cba2727`
|
||||
- Span ID: `3be2550ec51156e2`
|
||||
- Результат: найден реальный HTTP server span `POST /api/v1/auth/sessions`.
|
||||
|
||||
**Подтверждённая корреляция worker:**
|
||||
|
||||
- Service: `messenger-worker`
|
||||
- Event: `Image job handled`
|
||||
- Trace ID: `af16dbcb7b7ca25b06d39902dc48afd8`
|
||||
- Span ID: `644d3cda73387e3f`
|
||||
- Результат: найден consumer span `image_job.process`.
|
||||
|
||||
Обе проверки завершились `PASS: API and worker trace correlation`.
|
||||
|
||||
Dashboard и конфигурации provisioning сохранены в репозитории, поэтому после чистого запуска они загружаются без ручной настройки через браузер.
|
||||
|
||||
Проверки инфраструктуры подтверждаются состоянием контейнеров и результатами отдельных запросов к Prometheus, Loki и Tempo; один только публичный smoke не считается доказательством работающей наблюдаемости.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотезы:**
|
||||
|
||||
1. При остановке worker API сохранит возможность принимать изображения, а принятые задания будут обработаны после восстановления worker.
|
||||
2. При остановке PostgreSQL запросы, требующие доступа к БД, начнут завершаться ошибками; после восстановления БД авторизация снова заработает.
|
||||
3. Метрики, логи и трассы позволят различить отсутствие фоновой обработки и недоступность базы данных.
|
||||
|
||||
**Условия проведения:**
|
||||
|
||||
- Среда: локальный Docker Desktop с WSL Ubuntu.
|
||||
- Контейнеры: две API-реплики, PostgreSQL, RustFS, RabbitMQ, dispatcher, worker, Collector, Prometheus, Loki, Tempo, Grafana.
|
||||
- Нагрузка: синтетические операции публичного smoke и специальные последовательные запросы скриптов инцидентов.
|
||||
- Concurrency: в сценарии отказа БД проверочные запросы выполнялись последовательно.
|
||||
- Прогрев: перед отказом выполнялись успешные запросы к API.
|
||||
- Инцидент worker: принудительная остановка worker и дополнительная пауза 60 секунд.
|
||||
- Инцидент PostgreSQL: остановка контейнера БД, пять проверочных запросов и восстановление.
|
||||
- Аппаратная конфигурация, выделенные CPU/RAM и лимиты Docker: отдельно не фиксировались.
|
||||
- Продолжительная нагрузка с фиксированным arrival rate и несколькими уровнями concurrency: не выполнялась.
|
||||
- Повторяемость: приведены результаты одного зафиксированного прогона каждого контролируемого инцидента.
|
||||
|
||||
**Зафиксированные измерения:**
|
||||
|
||||
Для инцидента PostgreSQL:
|
||||
|
||||
| Запрос | Результат | Длительность |
|
||||
| --- | --- | --- |
|
||||
| До отказа | HTTP 201 | Не зафиксирована в выводе |
|
||||
| Probe 1 | TimeoutError | 5,005 с |
|
||||
| Probe 2 | HTTP 503 | 0,046 с |
|
||||
| Probe 3 | HTTP 503 | 4,071 с |
|
||||
| Probe 4 | HTTP 503 | 0,038 с |
|
||||
| Probe 5 | HTTP 503 | 0,076 с |
|
||||
| После восстановления | HTTP 201 | 0,058 с |
|
||||
|
||||
Во время отказа БД все пять запросов завершились неуспешно: четыре HTTP 503 и один клиентский тайм-аут.
|
||||
|
||||
В Grafana зафиксирован рост доли HTTP 5xx примерно до 67% в двухминутном окне, а также изменение p95 длительности запросов.
|
||||
|
||||
Отдельные нагрузочные измерения устойчивого RPS, p50/p95/p99 по серии прогонов, CPU/RAM и распределения задержек не собирались. Точки на dashboard характеризуют конкретный наблюдаемый период, но не являются результатом полноценного нагрузочного теста.
|
||||
|
||||
**Таблица инцидентов:**
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка worker | API принимает upload; задания остаются в `queued` | HTTP 202, сообщение создано, после запуска worker статус `ready` | Пауза 60 секунд; потеря принятого изображения не обнаружена |
|
||||
| Перезапуск RabbitMQ и API при остановленном worker | Принятое задание не теряется | Статус `queued` сохранился после перезапуска | После возобновления worker задание обработано |
|
||||
| Повторное включение worker | Появляется обработка и thumbnail | Создан JPEG 256×160, оригинал проверен SHA-256 | Точное время перехода в `ready` отдельно не измерялось |
|
||||
| Остановка PostgreSQL | Авторизация, зависящая от БД, перестаёт работать | 4 HTTP 503 и 1 TimeoutError | После восстановления БД — HTTP 201 за 0,058 с |
|
||||
| Повторный smoke после отказов | Вся бизнес-функциональность восстановлена | `make test` — PASS | Потерь подтверждённых данных в проверенных сценариях не обнаружено |
|
||||
|
||||
**Анализ инцидента worker:**
|
||||
|
||||
На dashboard в период остановки сохранялись успешные HTTP-запросы, но новые успешные обработки worker не появлялись. После восстановления worker был зафиксирован новый успешный job и данные о длительности обработки.
|
||||
|
||||
Причина: отсутствие активного фонового обработчика, а не отказ API. Это подтверждено контролируемой остановкой контейнера и результатом `PASS: durable worker recovery`.
|
||||
|
||||
Доказательства: `evidence/incident-worker.txt`, `evidence/incident-worker-dashboard.png`, `evidence/incident-worker-recovered.png`.
|
||||
|
||||
**Анализ инцидента PostgreSQL:**
|
||||
|
||||
Во время остановки БД запросы авторизации завершались HTTP 503 или тайм-аутом. График 5xx в Grafana показал рост серверных ошибок. После восстановления PostgreSQL авторизация вернулась к HTTP 201.
|
||||
|
||||
Причина: недоступность PostgreSQL. Она подтверждается характером запросов, контролируемой остановкой контейнера, журналами и восстановлением после запуска БД.
|
||||
|
||||
Отдельные DB client spans отсутствуют, поэтому точная локализация по одной только трассе API ограничена.
|
||||
|
||||
Доказательства: `evidence/incident-db.txt`, `evidence/incident-db-dashboard.png`, `evidence/incident-db-api-logs.txt`, `evidence/incident-db-postgres-logs.txt`.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**Какие отказы выдерживаются:**
|
||||
|
||||
- Остановка worker не приводит к потере принятого задания при сохранности PostgreSQL и RustFS.
|
||||
- Перезапуск RabbitMQ и API не нарушает сохранность зафиксированного задания.
|
||||
- После восстановления PostgreSQL приложение возвращается к нормальной работе.
|
||||
- Публичный smoke проходит при включённой телеметрии.
|
||||
|
||||
**Какие отказы и ограничения остаются:**
|
||||
|
||||
- PostgreSQL, RustFS и RabbitMQ развёрнуты одиночными экземплярами и остаются точками отказа.
|
||||
- При удалении volumes учебные данные и история наблюдаемости теряются.
|
||||
- Трассы API и worker не связаны единым W3C-контекстом через брокер.
|
||||
- Нет отдельной SQL-инструментации с DB client spans.
|
||||
- Метрики глубины и возраста очереди, насыщения ресурсов и end-to-end времени до `ready` не реализованы.
|
||||
- Alert rules и автоматическая проверка их переходов в firing/resolved не реализованы.
|
||||
- Не проводилось отдельное испытание недоступности backend-телеметрии и измерение потерь.
|
||||
- Метрики worker могут сбрасываться при перезапуске процесса; для анализа необходимы временные ряды и события восстановления.
|
||||
- Не выполнялись длительные нагрузочные испытания и измерения производительности с фиксированными hardware limits.
|
||||
|
||||
**Следующие шаги:**
|
||||
|
||||
1. Добавить передачу W3C trace context через RabbitMQ.
|
||||
2. Инструментировать запросы к PostgreSQL и S3 отдельными spans.
|
||||
3. Добавить метрики очереди и насыщения ресурсов.
|
||||
4. Добавить alert rules с порогами, окнами и runbook.
|
||||
5. Провести повторяемые нагрузочные испытания с фиксацией CPU/RAM, RPS и p50/p95/p99.
|
||||
6. Протестировать частичную недоступность Loki, Tempo и Collector и измерить потери телеметрии.
|
||||
|
||||
**Вывод:** реализована работающая подсистема наблюдаемости на основе OpenTelemetry, Prometheus, Loki, Tempo и Grafana. Подтверждён экспорт трёх сигналов, обнаружение реальных трасс по связанным логам, работоспособность dashboard и восстановление системы после двух контролируемых отказов. Заявленный уровень — оценка 3.
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
|
||||
services:
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
volumes:
|
||||
- ./observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.5.5
|
||||
command:
|
||||
- -config.file=/etc/loki/local-config.yaml
|
||||
volumes:
|
||||
- ./observability/loki.yaml:/etc/loki/local-config.yaml:ro
|
||||
- loki_data:/loki
|
||||
user: "0"
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.8.2
|
||||
command:
|
||||
- -config.file=/etc/tempo.yaml
|
||||
volumes:
|
||||
- ./observability/tempo.yaml:/etc/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
user: "0"
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.136.0
|
||||
command:
|
||||
- --config=/etc/otelcol-contrib/config.yaml
|
||||
volumes:
|
||||
- ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:14317:4317"
|
||||
- "127.0.0.1:14318:4318"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- /otelcol-contrib
|
||||
- components
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.8-management
|
||||
hostname: rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?Set RABBITMQ_USER in .env}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
minio:
|
||||
image: rustfs/rustfs:1.0.3
|
||||
command: ["/data"]
|
||||
environment:
|
||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env}
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
build: .
|
||||
command: ["python", "-m", "src.s3_setup"]
|
||||
environment: &s3_env
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_started
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ["python", "-m", "src.migrate"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api1:
|
||||
build: .
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*"
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
<<: *s3_env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
api2:
|
||||
build: .
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*"
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
<<: *s3_env
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
|
||||
dispatcher:
|
||||
build: .
|
||||
command: ["python", "-m", "src.dispatcher"]
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||
depends_on:
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "src.worker"]
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||
depends_on:
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
|
||||
nginx:
|
||||
image: nginx:1.29-alpine
|
||||
ports:
|
||||
- "127.0.0.1:8080:80"
|
||||
volumes:
|
||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
minio_data:
|
||||
rabbitmq_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,16 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 22 minutes ago Up 22 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 22 minutes ago Up 22 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 27 minutes ago Up 27 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 27 minutes ago Up 27 minutes 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 9 minutes ago Up 15 seconds 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 44 minutes ago Up 44 minutes 3100/tcp
|
||||
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 16 minutes ago Exited (0) 16 minutes ago
|
||||
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 27 minutes ago Up 27 minutes 9000-9001/tcp
|
||||
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 27 minutes ago Up 27 minutes 127.0.0.1:8080->80/tcp
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 41 minutes ago Up 41 minutes (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 44 minutes ago Up 44 minutes 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 27 minutes ago Up 27 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 16 minutes ago Exited (0) 16 minutes ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 44 minutes ago Up 44 minutes
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 16 minutes ago Up 16 minutes 8080/tcp
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 260 KiB |
@@ -0,0 +1,200 @@
|
||||
api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats HTTP/1.1" 201 Created
|
||||
api2-1 | 2026-11-21 16:13:27,824 INFO event=http_request_completed trace_id=1889717945c3eddd806a8e37da7368d7 span_id=778a902e8dfd6c54 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats/4b9eb2b4-44e4-4bb3-863b-4943b9c37c70/messages HTTP/1.1" 201 Created
|
||||
api2-1 | INFO: 127.0.0.1:54924 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: Shutting down
|
||||
api2-1 | INFO: Waiting for application shutdown.
|
||||
api2-1 | INFO: Application shutdown complete.
|
||||
api2-1 | INFO: Finished server process [1]
|
||||
api2-1 | INFO: Started server process [1]
|
||||
api2-1 | INFO: Waiting for application startup.
|
||||
api2-1 | INFO: Application startup complete.
|
||||
api2-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit)
|
||||
api2-1 | INFO: 127.0.0.1:45734 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44306 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44322 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:37222 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:37234 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59306 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59346 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:46812 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:46838 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57764 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:60662 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | 2026-11-21 16:14:40,820 INFO event=http_request_completed trace_id=853e8a1ac0c8acd7182a241ca8c17990 span_id=05519f4c759960a2 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||
api2-1 | 2026-11-21 16:14:41,864 INFO event=http_request_completed trace_id=3bb041b87253ffd0bc2b24360061cb7e span_id=3815f86eb8e0f940 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:60712 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:41942 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:41974 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:50386 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:50402 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:39396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:39416 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:33268 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:33276 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57776 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57788 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:47430 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:47450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:42750 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:42762 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49882 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:40420 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59716 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59740 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49390 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:52874 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:52888 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:34432 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:50824 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:50842 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44010 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44022 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:42162 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:42192 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:52500 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:52514 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:43736 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:43756 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:34588 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:34616 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57026 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:57052 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:60044 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:60050 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59624 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:59646 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:41186 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | 2026-11-21 16:18:29,812 INFO event=http_request_completed trace_id=1756efab020974bc3f9555ce40099e7d span_id=30bfd1f2baa385c0 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/users HTTP/1.1" 201 Created
|
||||
api2-1 | 2026-11-21 16:18:35,246 INFO event=http_request_completed trace_id=8fdf394cb3bda0228a70e17cddcaa622 span_id=3da8e0fea00100e0 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||
api2-1 | 2026-11-21 16:18:43,293 INFO event=http_request_completed trace_id=60c1a9ff611169aa6530d289c1cf7075 span_id=c5c5694d6f107123 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||
api2-1 | 2026-11-21 16:18:47,408 INFO event=http_request_completed trace_id=e79c6299dbfb2c31e3d1bbd58c7a253e span_id=0f6ee54ea39b9385 HTTP request completed
|
||||
api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||
api2-1 | INFO: 127.0.0.1:33068 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:33090 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38568 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38598 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44470 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:44482 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:39106 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:39116 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49932 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49962 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:55630 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:55634 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:54914 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: Shutting down
|
||||
api1-1 | INFO: Waiting for application shutdown.
|
||||
api1-1 | INFO: Application shutdown complete.
|
||||
api1-1 | INFO: Finished server process [1]
|
||||
api1-1 | INFO: Started server process [1]
|
||||
api1-1 | INFO: Waiting for application startup.
|
||||
api1-1 | INFO: Application startup complete.
|
||||
api1-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit)
|
||||
api1-1 | INFO: 172.20.0.14:39646 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | 2026-11-21 16:13:39,393 INFO event=http_request_completed trace_id=22e54b0e6d69077f18bfdb7c4e02661d span_id=45ddb3c5048edc9f HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:39646 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:45730 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44304 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44318 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:37220 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:37226 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59320 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59332 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:46808 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:46822 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57752 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57796 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:60666 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | 2026-11-21 16:14:41,842 INFO event=http_request_completed trace_id=c58c902e3393684ca4f780fd7f70e520 span_id=a07ee23af91c0bf7 HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||
api1-1 | 2026-11-21 16:14:41,907 INFO event=http_request_completed trace_id=4bcc0c6f19ddf75a5fd7c5e364d9e32b span_id=99fff1d1ee4ace8a HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content?variant=thumbnail HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:60724 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:41948 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:41962 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:50380 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:50396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38278 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38280 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38540 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:38560 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:32862 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:32890 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api2-1 | INFO: 127.0.0.1:49622 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:39382 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:39402 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:33254 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:33272 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57766 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:47440 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:47448 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:42758 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:42770 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49880 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:40414 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59714 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59724 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49378 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49410 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:52872 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:52904 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:34444 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:50832 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:50840 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44004 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44036 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:42176 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:42178 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:52498 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:52518 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:43746 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:43758 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:34594 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:34600 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57012 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:57038 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:60030 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:60054 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59630 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:59654 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:41172 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | 2026-11-21 16:18:29,864 INFO event=http_request_completed trace_id=951f6fcc6a32d838b2dc5fa1e2d7343a span_id=776bbf9a8e2b986a HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:48674 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created
|
||||
api1-1 | 2026-11-21 16:18:37,223 INFO event=http_request_completed trace_id=1f8dc7db901d925f6b15fb2348b1a048 span_id=d2e85dfd82bc72b7 HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:33554 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||
api1-1 | 2026-11-21 16:18:45,331 INFO event=http_request_completed trace_id=2afc8659a56936ef96018bc515cd6e61 span_id=88f8963e2289dcfc HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:50422 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||
api1-1 | 2026-11-21 16:18:55,183 INFO event=http_request_completed trace_id=3d6852b9fb0e7047f085b8c239515e9c span_id=2802fa3fff95e680 HTTP request completed
|
||||
api1-1 | INFO: 172.20.0.14:39552 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created
|
||||
api1-1 | INFO: 127.0.0.1:33084 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:33098 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38580 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38592 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44456 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:44476 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:39102 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:39112 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49946 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49978 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:55624 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:55638 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38268 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38290 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38552 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:38574 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:32866 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:32876 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api1-1 | INFO: 127.0.0.1:49606 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
@@ -0,0 +1,16 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 6 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 6 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up About a minute (healthy) 5432/tcp
|
||||
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp
|
||||
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) 5 minutes ago
|
||||
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp
|
||||
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 7 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) 5 minutes ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up 5 minutes 8080/tcp
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 247 KiB |
@@ -0,0 +1,80 @@
|
||||
db-1 | 2026-11-19 10:59:29.991 UTC [1] LOG: listening on IPv6 address "::", port 5432
|
||||
db-1 | 2026-11-19 10:59:29.995 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
|
||||
db-1 | 2026-11-19 10:59:30.000 UTC [57] LOG: database system was shut down at 2026-11-19 10:59:29 UTC
|
||||
db-1 | 2026-11-19 10:59:30.004 UTC [1] LOG: database system is ready to accept connections
|
||||
db-1 | 2026-11-19 11:01:46.203 UTC [451] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:01:46.203 UTC [451] DETAIL: Key (username)=(u_alice_f0123aa15331) already exists.
|
||||
db-1 | 2026-11-19 11:01:46.203 UTC [451] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:04:31.001 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-19 11:04:48.288 UTC [55] LOG: checkpoint complete: wrote 175 buffers (1.1%); 0 WAL file(s) added, 0 removed, 0 recycled; write=17.233 s, sync=0.045 s, total=17.288 s; sync files=104, longest=0.002 s, average=0.001 s; distance=803 kB, estimate=803 kB; lsn=0/19E7700, redo lsn=0/19E76C8
|
||||
db-1 | 2026-11-19 11:04:50.938 UTC [988] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:04:50.938 UTC [988] DETAIL: Key (username)=(u_alice_dff3cf8f9360) already exists.
|
||||
db-1 | 2026-11-19 11:04:50.938 UTC [988] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:09:31.281 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-19 11:09:37.044 UTC [55] LOG: checkpoint complete: wrote 58 buffers (0.4%); 0 WAL file(s) added, 0 removed, 0 recycled; write=5.713 s, sync=0.021 s, total=5.763 s; sync files=43, longest=0.002 s, average=0.001 s; distance=14 kB, estimate=724 kB; lsn=0/19EB220, redo lsn=0/19EB1E8
|
||||
db-1 | 2026-11-19 11:11:03.824 UTC [2110] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:11:03.824 UTC [2110] DETAIL: Key (username)=(u_alice_a072f61fcaf9) already exists.
|
||||
db-1 | 2026-11-19 11:11:03.824 UTC [2110] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:11:24.481 UTC [2231] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:11:24.481 UTC [2231] DETAIL: Key (username)=(u_alice_ed51a8803fc1) already exists.
|
||||
db-1 | 2026-11-19 11:11:24.481 UTC [2231] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:14:31.133 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-19 11:14:34.469 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.019 s, total=3.336 s; sync files=34, longest=0.003 s, average=0.001 s; distance=30 kB, estimate=655 kB; lsn=0/19F2DB0, redo lsn=0/19F2D78
|
||||
db-1 | 2026-11-19 11:27:17.811 UTC [4991] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:27:17.811 UTC [4991] DETAIL: Key (username)=(u_alice_5c60fc1beaf6) already exists.
|
||||
db-1 | 2026-11-19 11:27:17.811 UTC [4991] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:29:31.636 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-19 11:29:34.995 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.022 s, total=3.359 s; sync files=34, longest=0.003 s, average=0.001 s; distance=32 kB, estimate=592 kB; lsn=0/19FB020, redo lsn=0/19FAFE8
|
||||
db-1 | 2026-11-19 11:31:47.459 UTC [5815] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||
db-1 | 2026-11-19 11:31:47.459 UTC [5815] DETAIL: Key (username)=(u_alice_afb8c8b7058a) already exists.
|
||||
db-1 | 2026-11-19 11:31:47.459 UTC [5815] STATEMENT:
|
||||
db-1 | INSERT INTO users
|
||||
db-1 | (id, username, display_name, password_hash)
|
||||
db-1 | VALUES ($1, $2, $3, $4)
|
||||
db-1 | RETURNING id, username, display_name, created_at
|
||||
db-1 |
|
||||
db-1 | 2026-11-19 11:34:32.002 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-19 11:34:35.335 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.016 s, total=3.334 s; sync files=34, longest=0.002 s, average=0.001 s; distance=38 kB, estimate=537 kB; lsn=0/1A04AD0, redo lsn=0/1A04A98
|
||||
db-1 | 2026-11-21 16:15:11.638 UTC [55] LOG: checkpoint starting: time
|
||||
db-1 | 2026-11-21 16:15:14.988 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.319 s, sync=0.016 s, total=3.351 s; sync files=34, longest=0.002 s, average=0.001 s; distance=40 kB, estimate=487 kB; lsn=0/1A0EB68, redo lsn=0/1A0EB30
|
||||
db-1 | 2026-11-21 16:18:29.948 UTC [1] LOG: received fast shutdown request
|
||||
db-1 | 2026-11-21 16:18:29.950 UTC [1] LOG: aborting any active transactions
|
||||
db-1 | 2026-11-21 16:18:29.956 UTC [1] LOG: background worker "logical replication launcher" (PID 60) exited with exit code 1
|
||||
db-1 | 2026-11-21 16:18:29.957 UTC [55] LOG: shutting down
|
||||
db-1 | 2026-11-21 16:18:29.958 UTC [55] LOG: checkpoint starting: shutdown immediate
|
||||
db-1 | 2026-11-21 16:18:29.972 UTC [55] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.002 s, sync=0.005 s, total=0.016 s; sync files=8, longest=0.002 s, average=0.001 s; distance=13 kB, estimate=440 kB; lsn=0/1A12070, redo lsn=0/1A12070
|
||||
db-1 | 2026-11-21 16:18:29.979 UTC [1] LOG: database system is shut down
|
||||
db-1 |
|
||||
db-1 | PostgreSQL Database directory appears to contain a database; Skipping initialization
|
||||
db-1 |
|
||||
db-1 | 2026-11-21 16:18:49.717 UTC [1] LOG: starting PostgreSQL 16.14 on x86_64-pc-linux-musl, compiled by gcc (Alpine 15.2.0) 15.2.0, 64-bit
|
||||
db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432
|
||||
db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv6 address "::", port 5432
|
||||
db-1 | 2026-11-21 16:18:49.722 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
|
||||
db-1 | 2026-11-21 16:18:49.728 UTC [29] LOG: database system was shut down at 2026-11-21 16:18:29 UTC
|
||||
db-1 | 2026-11-21 16:18:49.735 UTC [1] LOG: database system is ready to accept connections
|
||||
@@ -0,0 +1,18 @@
|
||||
Baseline: authentication HTTP 201
|
||||
INCIDENT START: stopping PostgreSQL
|
||||
Container mtusi2026lab6-db-1 Stopping
|
||||
Container mtusi2026lab6-db-1 Stopped
|
||||
Probe 1: status=TimeoutError, duration=5.005s
|
||||
Probe 2: status=503, duration=0.046s
|
||||
Probe 3: status=503, duration=4.071s
|
||||
Probe 4: status=503, duration=0.038s
|
||||
Probe 5: status=503, duration=0.076s
|
||||
INCIDENT END: restoring PostgreSQL
|
||||
Container mtusi2026lab6-db-1 Created
|
||||
Container mtusi2026lab6-db-1 Starting
|
||||
Container mtusi2026lab6-db-1 Started
|
||||
Container mtusi2026lab6-db-1 Waiting
|
||||
Container mtusi2026lab6-db-1 Healthy
|
||||
Recovery: HTTP 201, 0.058s
|
||||
Failed requests during incident: 5/5
|
||||
PASS: database failure observed and recovered
|
||||
@@ -0,0 +1,50 @@
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): Normal shutdown
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1d1c0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1d1c0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:27:18,595 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:27:18,595 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||
dispatcher-1 | 2026-11-19 11:27:18,595 INFO Published job 6d77b086-944f-447d-8156-81995b5c562b
|
||||
dispatcher-1 | 2026-11-19 11:31:47,858 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||
dispatcher-1 | 2026-11-19 11:31:47,859 INFO Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,859 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,861 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,861 INFO Created channel=1
|
||||
dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing connection (200): Normal shutdown
|
||||
dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing connection (200): 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-19 11:31:47,868 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||
dispatcher-1 | 2026-11-19 11:31:47,869 INFO Published job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7
|
||||
dispatcher-1 | 2026-11-21 16:13:27,871 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||
dispatcher-1 | 2026-11-21 16:13:27,875 INFO Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,878 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
dispatcher-1 | 2026-11-21 16:13:27,893 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,894 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,895 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,899 INFO Created channel=1
|
||||
dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing connection (200): Normal shutdown
|
||||
dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,920 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,920 INFO Closing connection (200): 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||
dispatcher-1 | 2026-11-21 16:13:27,922 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||
dispatcher-1 | 2026-11-21 16:13:27,923 INFO Published job e687ea16-ef44-416e-a96f-15dfac07230e
|
||||
@@ -0,0 +1,16 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up 2 days (healthy) 5432/tcp
|
||||
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp
|
||||
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp
|
||||
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp
|
||||
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) About a minute ago
|
||||
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp
|
||||
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp
|
||||
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp
|
||||
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) About a minute ago
|
||||
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days
|
||||
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up About a minute 8080/tcp
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 226 KiB |
@@ -0,0 +1,26 @@
|
||||
worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||
worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 50892), raddr=('172.20.0.7', 5672)>
|
||||
worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1
|
||||
worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs
|
||||
worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled
|
||||
worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled
|
||||
worker-1 | 2026-11-19 11:27:18,636 INFO Job 6d77b086-944f-447d-8156-81995b5c562b ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:27:18,643 INFO event=image_job_handled trace_id=6d39def6a9347b9766c7d9820480cbea span_id=91110edf3e0e5f53 Image job handled
|
||||
worker-1 | 2026-11-19 11:31:47,910 INFO Job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7 ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:31:47,918 INFO event=image_job_handled trace_id=5a9611a8564da15253517786b3b87c35 span_id=692dee1a1ba56089 Image job handled
|
||||
worker-1 | 2026-11-21 16:14:41,241 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||
worker-1 | 2026-11-21 16:14:41,242 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 37148), raddr=('172.20.0.7', 5672)>
|
||||
worker-1 | 2026-11-21 16:14:41,242 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-21 16:14:41,249 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-21 16:14:41,250 INFO Created channel=1
|
||||
worker-1 | 2026-11-21 16:14:41,253 INFO Worker listening on image_jobs
|
||||
worker-1 | 2026-11-21 16:14:41,323 INFO Job e687ea16-ef44-416e-a96f-15dfac07230e ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-21 16:14:41,331 INFO event=image_job_handled trace_id=b768e258011bde0b8084a6665f1067d1 span_id=bee607ec28cf87ab Image job handled
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 241 KiB |
@@ -0,0 +1,60 @@
|
||||
Stopping worker...
|
||||
Container mtusi2026lab6-worker-1 Stopping
|
||||
Container mtusi2026lab6-worker-1 Stopped
|
||||
Upload: 202 queued
|
||||
Message created while worker is stopped
|
||||
Restarting RabbitMQ and API...
|
||||
Container mtusi2026lab6-rabbitmq-1 Restarting
|
||||
Container mtusi2026lab6-rabbitmq-1 Started
|
||||
Container mtusi2026lab6-rabbitmq-1 Running
|
||||
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||
Container mtusi2026lab6-api2-1 Restarting
|
||||
Container mtusi2026lab6-api1-1 Restarting
|
||||
Container mtusi2026lab6-api1-1 Started
|
||||
Container mtusi2026lab6-api2-1 Started
|
||||
Queued attachment survived restart
|
||||
INCIDENT: worker stopped for 60 seconds
|
||||
Starting worker...
|
||||
Container mtusi2026lab6-db-1 Running
|
||||
Container mtusi2026lab6-rabbitmq-1 Running
|
||||
Container mtusi2026lab6-minio-1 Running
|
||||
Container mtusi2026lab6-migrate-1 Created
|
||||
Container mtusi2026lab6-s3-init-1 Created
|
||||
Container mtusi2026lab6-worker-1 Created
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-db-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-db-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||
Container mtusi2026lab6-worker-1 Starting
|
||||
Container mtusi2026lab6-worker-1 Started
|
||||
Attachment: ready
|
||||
Original: SHA-256 verified
|
||||
Thumbnail: JPEG 256x160 verified
|
||||
PASS: durable worker recovery
|
||||
Container mtusi2026lab6-db-1 Running
|
||||
Container mtusi2026lab6-migrate-1 Created
|
||||
Container mtusi2026lab6-minio-1 Running
|
||||
Container mtusi2026lab6-rabbitmq-1 Running
|
||||
Container mtusi2026lab6-s3-init-1 Created
|
||||
Container mtusi2026lab6-worker-1 Running
|
||||
Container mtusi2026lab6-db-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Starting
|
||||
Container mtusi2026lab6-s3-init-1 Started
|
||||
Container mtusi2026lab6-db-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Starting
|
||||
Container mtusi2026lab6-migrate-1 Started
|
||||
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||
Container mtusi2026lab6-migrate-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Waiting
|
||||
Container mtusi2026lab6-s3-init-1 Exited
|
||||
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||
Container mtusi2026lab6-migrate-1 Exited
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": [
|
||||
"messenger_http_server_active_requests",
|
||||
"messenger_http_server_duration_milliseconds_bucket",
|
||||
"messenger_http_server_duration_milliseconds_count",
|
||||
"messenger_http_server_duration_milliseconds_sum",
|
||||
"messenger_http_server_request_size_bytes_bucket",
|
||||
"messenger_http_server_request_size_bytes_count",
|
||||
"messenger_http_server_request_size_bytes_sum",
|
||||
"messenger_http_server_response_size_bytes_bucket",
|
||||
"messenger_http_server_response_size_bytes_count",
|
||||
"messenger_http_server_response_size_bytes_sum",
|
||||
"messenger_messenger_http_requests_total",
|
||||
"messenger_messenger_worker_job_duration_seconds_bucket",
|
||||
"messenger_messenger_worker_job_duration_seconds_count",
|
||||
"messenger_messenger_worker_job_duration_seconds_sum",
|
||||
"messenger_messenger_worker_jobs_processed_total",
|
||||
"messenger_target_info",
|
||||
"scrape_duration_seconds",
|
||||
"scrape_samples_post_metric_relabeling",
|
||||
"scrape_samples_scraped",
|
||||
"scrape_series_added",
|
||||
"up"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,20 @@
|
||||
Service: messenger-api
|
||||
Event: HTTP request completed
|
||||
Trace ID: c3ea28d96d7c536e222d8e4c7cba2727
|
||||
Span ID: 3be2550ec51156e2
|
||||
Tempo spans: 4
|
||||
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||
Span: POST /api/v1/auth/sessions kind: SPAN_KIND_SERVER
|
||||
PASS: Loki log linked to Tempo trace
|
||||
|
||||
Service: messenger-worker
|
||||
Event: Image job handled
|
||||
Trace ID: af16dbcb7b7ca25b06d39902dc48afd8
|
||||
Span ID: 644d3cda73387e3f
|
||||
Tempo spans: 1
|
||||
Span: image_job.process kind: SPAN_KIND_CONSUMER
|
||||
PASS: Loki log linked to Tempo trace
|
||||
|
||||
PASS: API and worker trace correlation
|
||||
@@ -0,0 +1,12 @@
|
||||
worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||
worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 50892), raddr=('172.20.0.7', 5672)>
|
||||
worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1
|
||||
worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs
|
||||
worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled
|
||||
worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160
|
||||
worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled
|
||||
@@ -0,0 +1,74 @@
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
|
||||
CONSTRAINT username_format
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT display_name_length
|
||||
CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
CONSTRAINT display_name_not_blank
|
||||
CHECK (length(btrim(display_name)) > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
|
||||
CONSTRAINT title_not_blank
|
||||
CHECK (length(btrim(title)) > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL
|
||||
REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL
|
||||
REFERENCES users(id),
|
||||
position INTEGER NOT NULL,
|
||||
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL
|
||||
REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL
|
||||
REFERENCES users(id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
|
||||
CONSTRAINT message_length
|
||||
CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
CONSTRAINT message_not_blank
|
||||
CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash CHAR(64) PRIMARY KEY,
|
||||
user_id UUID NOT NULL
|
||||
REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_members_user
|
||||
ON chat_members(user_id, chat_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_order
|
||||
ON chats(created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_history
|
||||
ON messages(chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user
|
||||
ON sessions(user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expiry
|
||||
ON sessions(expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
idempotency_key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_created
|
||||
ON message_idempotency(created_at);
|
||||
@@ -0,0 +1,34 @@
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
uploader_id UUID NOT NULL REFERENCES users(id),
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'ready',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
CONSTRAINT attachment_size_valid
|
||||
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
|
||||
CONSTRAINT attachment_dimensions_valid
|
||||
CHECK (width > 0 AND height > 0),
|
||||
|
||||
CONSTRAINT attachment_status_valid
|
||||
CHECK (status IN ('ready', 'failed'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat
|
||||
ON attachments(chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id),
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
@@ -0,0 +1,13 @@
|
||||
|
||||
-- Lab4: messages with attachments may have empty text.
|
||||
-- The API checks that a message contains text or attachments.
|
||||
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT IF EXISTS message_length;
|
||||
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT IF EXISTS message_not_blank;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT message_length
|
||||
CHECK (char_length(text) <= 4000);
|
||||
@@ -0,0 +1,95 @@
|
||||
-- Allow asynchronous attachment states.
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT IF EXISTS attachment_status_valid;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachment_status_valid
|
||||
CHECK (
|
||||
status IN (
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed'
|
||||
)
|
||||
);
|
||||
|
||||
-- Staging object stored in the private S3 bucket.
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS staging_key TEXT;
|
||||
|
||||
-- Machine-readable final error.
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS error_code TEXT;
|
||||
|
||||
-- Last metadata update timestamp.
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS updated_at
|
||||
TIMESTAMPTZ NOT NULL DEFAULT NOW();
|
||||
|
||||
-- Worker jobs and durable publication intent.
|
||||
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL UNIQUE
|
||||
REFERENCES attachments(id),
|
||||
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||
|
||||
published_at TIMESTAMPTZ,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
CONSTRAINT attachment_job_status_valid
|
||||
CHECK (
|
||||
status IN (
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed'
|
||||
)
|
||||
),
|
||||
|
||||
CONSTRAINT attachment_job_attempts_valid
|
||||
CHECK (
|
||||
attempts >= 0
|
||||
AND max_attempts BETWEEN 1 AND 20
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending
|
||||
ON attachment_jobs(status, next_attempt_at);
|
||||
|
||||
-- Image variants created by the worker.
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id UUID NOT NULL
|
||||
REFERENCES attachments(id),
|
||||
|
||||
variant_name TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
PRIMARY KEY (attachment_id, variant_name),
|
||||
|
||||
CONSTRAINT variant_name_valid
|
||||
CHECK (variant_name IN ('thumbnail')),
|
||||
|
||||
CONSTRAINT variant_size_valid
|
||||
CHECK (size_bytes > 0),
|
||||
|
||||
CONSTRAINT variant_dimensions_valid
|
||||
CHECK (width > 0 AND height > 0)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachment_variants_key
|
||||
ON attachment_variants(object_key);
|
||||
@@ -0,0 +1,37 @@
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 2s;
|
||||
|
||||
upstream messenger_api {
|
||||
zone messenger_api 64k;
|
||||
|
||||
server api1:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
server api2:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
client_max_body_size 11m;
|
||||
listen 80;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger Lab6 - Observability",
|
||||
"tags": ["lab6", "messenger", "opentelemetry"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API - Requests per second",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0, "y": 0, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_messenger_http_requests_total[2m]))",
|
||||
"legendFormat": "All business requests"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API - HTTP 5xx error ratio",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12, "y": 0, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(100 * sum(rate(messenger_messenger_http_requests_total{http_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(messenger_messenger_http_requests_total[2m])), 0.000001)) or vector(0)",
|
||||
"legendFormat": "5xx / all business requests"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API - HTTP p95 duration",
|
||||
"description": "OTel FastAPI duration histogram is in milliseconds; result converted to seconds.",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0, "y": 8, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_server_duration_milliseconds_bucket[5m]))) / 1000",
|
||||
"legendFormat": "p95 HTTP duration"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Worker - Jobs by outcome",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12, "y": 8, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (increase(messenger_messenger_worker_jobs_processed_total[5m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Worker - Processing p95 duration",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0, "y": 16, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_messenger_worker_job_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95 job duration"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker - Completed job attempts",
|
||||
"type": "stat",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12, "y": 16, "w": 12, "h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_messenger_worker_jobs_processed_total)",
|
||||
"legendFormat": "Processing attempts"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"values": false,
|
||||
"calcs": ["lastNotNull"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "API - Structured logs",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0, "y": 24, "w": 24, "h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=\"messenger-api\"}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"enableLogDetails": true,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Worker - Structured logs",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0, "y": 33, "w": 24, "h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=\"messenger-worker\"}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"enableLogDetails": true,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Messenger Lab6
|
||||
orgId: 1
|
||||
folder: Messenger Lab6
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,38 @@
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace_id=([a-f0-9]{32})'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,36 @@
|
||||
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/index
|
||||
cache_location: /loki/index_cache
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,67 @@
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 256
|
||||
spike_limit_mib: 64
|
||||
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 512
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
namespace: messenger
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,9 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets:
|
||||
- otel-collector:8889
|
||||
@@ -0,0 +1,27 @@
|
||||
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
trace_idle_period: 10s
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 48h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
@@ -0,0 +1,11 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
psycopg[binary]==3.2.6
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.40.70
|
||||
Pillow==11.2.1
|
||||
pika==1.3.2
|
||||
opentelemetry-api==1.38.0
|
||||
opentelemetry-sdk==1.38.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.38.0
|
||||
opentelemetry-instrumentation-fastapi==0.59b0
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
echo "Stopping API and image processing..."
|
||||
docker compose stop api1 api2 worker dispatcher
|
||||
|
||||
echo "Restarting persistent storage and broker..."
|
||||
docker compose restart db minio rabbitmq
|
||||
|
||||
echo "Waiting for PostgreSQL and RabbitMQ..."
|
||||
docker compose up -d --wait db rabbitmq
|
||||
|
||||
echo "Checking S3 bucket..."
|
||||
docker compose run --rm --no-deps s3-init
|
||||
|
||||
echo "Starting dispatcher and worker..."
|
||||
docker compose up -d dispatcher worker
|
||||
|
||||
echo "Starting API replicas..."
|
||||
docker compose up -d --wait api1 api2
|
||||
|
||||
echo "Restart complete. Persistent volumes preserved."
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then
|
||||
echo "TARGET_INSTANCE_ID is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for service in api1 api2; do
|
||||
instance_id=$(
|
||||
docker compose exec -T "$service" python -c '
|
||||
import urllib.request
|
||||
|
||||
response = urllib.request.urlopen(
|
||||
"http://127.0.0.1:8080/health/live",
|
||||
timeout=2
|
||||
)
|
||||
|
||||
print(response.headers.get("X-Instance-Id", ""))
|
||||
' 2>/dev/null
|
||||
) || continue
|
||||
|
||||
if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then
|
||||
echo "Stopping $service"
|
||||
docker compose stop -t 2 "$service"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Target API instance not found" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,175 @@
|
||||
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
BASE_URL = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def docker(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def request(method, path, body=None, headers=None):
|
||||
request_headers = dict(headers or {})
|
||||
|
||||
if isinstance(body, dict):
|
||||
body = json.dumps(body).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
|
||||
req = urllib.request.Request(
|
||||
BASE_URL + path,
|
||||
data=body,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
response.read()
|
||||
return response.status, time.monotonic() - started
|
||||
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.read()
|
||||
return exc.code, time.monotonic() - started
|
||||
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
return type(exc).__name__, time.monotonic() - started
|
||||
|
||||
|
||||
def login_header(username, password):
|
||||
encoded = base64.b64encode(
|
||||
f"{username}:{password}".encode("utf-8")
|
||||
).decode("ascii")
|
||||
|
||||
return {
|
||||
"Authorization": f"Basic {encoded}"
|
||||
}
|
||||
|
||||
|
||||
def wait_for_recovery(headers):
|
||||
deadline = time.monotonic() + 90
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if status == 201:
|
||||
print(
|
||||
f"Recovery: HTTP 201, {duration:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
raise RuntimeError(
|
||||
"Authentication did not recover after DB restart"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
|
||||
username = "lab6_db_" + suffix
|
||||
password = "IncidentTest_" + suffix
|
||||
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": username,
|
||||
"display_name": "DB incident test",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
raise RuntimeError(
|
||||
f"Cannot create test user: HTTP {status}"
|
||||
)
|
||||
|
||||
headers = login_header(username, password)
|
||||
|
||||
status, _ = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
raise RuntimeError(
|
||||
f"Baseline login failed: HTTP {status}"
|
||||
)
|
||||
|
||||
print("Baseline: authentication HTTP 201", flush=True)
|
||||
|
||||
observed_failures = 0
|
||||
|
||||
print("INCIDENT START: stopping PostgreSQL", flush=True)
|
||||
|
||||
try:
|
||||
docker("stop", "db")
|
||||
|
||||
for attempt in range(1, 6):
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
print(
|
||||
f"Probe {attempt}: status={status}, "
|
||||
f"duration={duration:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
observed_failures += 1
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
finally:
|
||||
print(
|
||||
"INCIDENT END: restoring PostgreSQL",
|
||||
flush=True,
|
||||
)
|
||||
docker("up", "-d", "--wait", "db")
|
||||
|
||||
wait_for_recovery(headers)
|
||||
|
||||
print(
|
||||
f"Failed requests during incident: "
|
||||
f"{observed_failures}/5",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if observed_failures == 0:
|
||||
raise RuntimeError(
|
||||
"The database incident did not affect requests"
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: database failure observed and recovered",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
sql() {
|
||||
docker compose exec -T db \
|
||||
psql -U messenger -d messenger -At -c "$1"
|
||||
}
|
||||
|
||||
echo "Finding completed image job..."
|
||||
|
||||
JOB_ID=$(sql "
|
||||
SELECT j.id
|
||||
FROM attachment_jobs j
|
||||
JOIN attachment_variants v
|
||||
ON v.attachment_id = j.attachment_id
|
||||
WHERE j.status = 'ready'
|
||||
AND v.variant_name = 'thumbnail'
|
||||
ORDER BY j.created_at DESC
|
||||
LIMIT 1;
|
||||
")
|
||||
|
||||
if [[ -z "$JOB_ID" ]]; then
|
||||
echo "FAIL: no completed image job found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Job: $JOB_ID"
|
||||
|
||||
STATE_QUERY="
|
||||
SELECT
|
||||
j.status || '|' ||
|
||||
j.attempts || '|' ||
|
||||
v.sha256 || '|' ||
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = j.attachment_id
|
||||
)
|
||||
FROM attachment_jobs j
|
||||
JOIN attachment_variants v
|
||||
ON v.attachment_id = j.attachment_id
|
||||
WHERE j.id = '$JOB_ID'
|
||||
AND v.variant_name = 'thumbnail';
|
||||
"
|
||||
|
||||
BEFORE=$(sql "$STATE_QUERY")
|
||||
|
||||
echo "Before: $BEFORE"
|
||||
echo "Publishing the same job twice..."
|
||||
|
||||
docker compose run --rm --no-deps api1 \
|
||||
python -c '
|
||||
import sys
|
||||
from src.queue import publish_job
|
||||
|
||||
job_id = sys.argv[1]
|
||||
publish_job(job_id)
|
||||
publish_job(job_id)
|
||||
print("Two duplicate messages published")
|
||||
' "$JOB_ID"
|
||||
|
||||
sleep 5
|
||||
|
||||
AFTER=$(sql "$STATE_QUERY")
|
||||
|
||||
echo "After: $AFTER"
|
||||
|
||||
if [[ "$BEFORE" != "$AFTER" ]]; then
|
||||
echo "FAIL: completed job changed after redelivery"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: duplicate delivery did not change completed job"
|
||||
echo "PASS: thumbnail checksum and variant count preserved"
|
||||
@@ -0,0 +1,154 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
LOKI = "http://loki:3100"
|
||||
TEMPO = "http://tempo:3200"
|
||||
|
||||
TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$")
|
||||
SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$")
|
||||
|
||||
|
||||
def get_json(url):
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def find_log(service, event):
|
||||
params = urllib.parse.urlencode({
|
||||
"query": f'{{service_name="{service}"}}',
|
||||
"start": str(time.time_ns() - 6 * 3600 * 10**9),
|
||||
"end": str(time.time_ns()),
|
||||
"limit": 500,
|
||||
"direction": "BACKWARD",
|
||||
})
|
||||
|
||||
payload = get_json(
|
||||
f"{LOKI}/loki/api/v1/query_range?{params}"
|
||||
)
|
||||
|
||||
streams = payload.get("data", {}).get("result", [])
|
||||
|
||||
for stream in streams:
|
||||
labels = stream.get("stream", {})
|
||||
|
||||
for entry in stream.get("values", []):
|
||||
if len(entry) < 2:
|
||||
continue
|
||||
|
||||
message = entry[1]
|
||||
metadata = entry[2] if len(entry) >= 3 else {}
|
||||
|
||||
if event not in message and (
|
||||
metadata.get("event") != event
|
||||
):
|
||||
continue
|
||||
|
||||
trace_id = (
|
||||
metadata.get("trace_id")
|
||||
or labels.get("trace_id")
|
||||
)
|
||||
|
||||
span_id = (
|
||||
metadata.get("span_id")
|
||||
or labels.get("span_id")
|
||||
)
|
||||
|
||||
if not trace_id or not TRACE_PATTERN.fullmatch(
|
||||
trace_id
|
||||
):
|
||||
continue
|
||||
|
||||
if not span_id or not SPAN_PATTERN.fullmatch(
|
||||
span_id
|
||||
):
|
||||
continue
|
||||
|
||||
return {
|
||||
"service": service,
|
||||
"event": event,
|
||||
"message": message,
|
||||
"trace_id": trace_id,
|
||||
"span_id": span_id,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def verify_trace(log, expected_kind):
|
||||
trace_id = log["trace_id"]
|
||||
|
||||
payload = get_json(
|
||||
f"{TEMPO}/api/traces/{trace_id}"
|
||||
)
|
||||
|
||||
batches = payload.get("batches", [])
|
||||
spans = []
|
||||
|
||||
for batch in batches:
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
spans.extend(scope.get("spans", []))
|
||||
|
||||
if not spans:
|
||||
raise AssertionError(
|
||||
f"No spans found for {trace_id}"
|
||||
)
|
||||
|
||||
kinds = [str(span.get("kind", "")) for span in spans]
|
||||
|
||||
if not any(expected_kind in kind for kind in kinds):
|
||||
raise AssertionError(
|
||||
f"Expected span kind {expected_kind}; got {kinds}"
|
||||
)
|
||||
|
||||
print(f"Service: {log['service']}")
|
||||
print(f"Event: {log['event']}")
|
||||
print(f"Trace ID: {trace_id}")
|
||||
print(f"Span ID: {log['span_id']}")
|
||||
print(f"Tempo spans: {len(spans)}")
|
||||
|
||||
for span in spans:
|
||||
print(
|
||||
" Span:",
|
||||
span.get("name"),
|
||||
"kind:",
|
||||
span.get("kind"),
|
||||
)
|
||||
|
||||
print("PASS: Loki log linked to Tempo trace")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
cases = [
|
||||
(
|
||||
"messenger-api",
|
||||
"HTTP request completed",
|
||||
"SERVER",
|
||||
),
|
||||
(
|
||||
"messenger-worker",
|
||||
"Image job handled",
|
||||
"CONSUMER",
|
||||
),
|
||||
]
|
||||
|
||||
for service, event, kind in cases:
|
||||
log = find_log(service, event)
|
||||
|
||||
if log is None:
|
||||
raise AssertionError(
|
||||
f"No correlated log found for {service}"
|
||||
)
|
||||
|
||||
verify_trace(log, kind)
|
||||
|
||||
print("PASS: API and worker trace correlation")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,256 @@
|
||||
import os
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
import struct
|
||||
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def docker(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def api(method, path, token=None, body=None, headers=None):
|
||||
request_headers = dict(headers or {})
|
||||
|
||||
if token:
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
if isinstance(body, dict):
|
||||
body = json.dumps(body).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(
|
||||
BASE + path,
|
||||
data=body,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
data = response.read()
|
||||
return (
|
||||
response.status,
|
||||
json.loads(data) if data and
|
||||
"application/json" in response.headers.get(
|
||||
"Content-Type", ""
|
||||
) else data,
|
||||
)
|
||||
|
||||
|
||||
def png_chunk(name, data):
|
||||
value = name + data
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ value
|
||||
+ struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def make_png():
|
||||
width, height = 320, 200
|
||||
scanline = b"\x00" + bytes([220, 80, 40]) * width
|
||||
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(
|
||||
b"IHDR",
|
||||
struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0),
|
||||
)
|
||||
+ png_chunk(
|
||||
b"IDAT", zlib.compress(scanline * height)
|
||||
)
|
||||
+ png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def wait_ready():
|
||||
for _ in range(90):
|
||||
try:
|
||||
status, data = api("GET", "/health/ready")
|
||||
if status == 200 and data.get("status") == "ok":
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
|
||||
raise RuntimeError("API did not become ready")
|
||||
|
||||
|
||||
def main():
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
username = "lab5_" + suffix
|
||||
password = "Lab5TestPassword_" + suffix
|
||||
|
||||
print("Stopping worker...", flush=True)
|
||||
docker("stop", "worker")
|
||||
wait_ready()
|
||||
|
||||
status, user = api(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": username,
|
||||
"display_name": "Lab5 recovery test",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert status == 201
|
||||
|
||||
credentials = base64.b64encode(
|
||||
f"{username}:{password}".encode()
|
||||
).decode()
|
||||
|
||||
status, session = api(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": f"Basic {credentials}"},
|
||||
body=b"",
|
||||
)
|
||||
assert status == 201
|
||||
token = session["token"]
|
||||
|
||||
status, chat = api(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
token,
|
||||
{
|
||||
"title": "Worker recovery",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert status == 201
|
||||
|
||||
original = make_png()
|
||||
path = (
|
||||
f"/api/v1/chats/{chat['id']}/attachments"
|
||||
"?filename=recovery.png"
|
||||
)
|
||||
|
||||
status, attachment = api(
|
||||
"POST",
|
||||
path,
|
||||
token,
|
||||
original,
|
||||
{"Content-Type": "image/png"},
|
||||
)
|
||||
|
||||
assert status == 202, f"Expected 202, got {status}"
|
||||
assert attachment["status"] == "queued"
|
||||
attachment_id = attachment["id"]
|
||||
print("Upload: 202 queued", flush=True)
|
||||
|
||||
status, message = api(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
token,
|
||||
{"text": "", "attachment_ids": [attachment_id]},
|
||||
{"Idempotency-Key": "recovery-" + suffix},
|
||||
)
|
||||
assert status == 201
|
||||
assert attachment_id in message["attachment_ids"]
|
||||
print("Message created while worker is stopped", flush=True)
|
||||
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
assert status == 200 and metadata["status"] == "queued"
|
||||
|
||||
print("Restarting RabbitMQ and API...", flush=True)
|
||||
docker("restart", "rabbitmq")
|
||||
docker("up", "-d", "--wait", "rabbitmq")
|
||||
docker("restart", "api1", "api2")
|
||||
wait_ready()
|
||||
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
assert metadata["status"] == "queued"
|
||||
print("Queued attachment survived restart", flush=True)
|
||||
|
||||
pause_seconds = int(
|
||||
os.getenv("LAB6_WORKER_PAUSE_SECONDS", "60")
|
||||
)
|
||||
|
||||
print(
|
||||
f"INCIDENT: worker stopped for {pause_seconds} seconds",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(pause_seconds)
|
||||
|
||||
print("Starting worker...", flush=True)
|
||||
docker("up", "-d", "worker")
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
|
||||
if metadata["status"] == "ready":
|
||||
break
|
||||
|
||||
if metadata["status"] == "failed":
|
||||
raise RuntimeError(
|
||||
f"Processing failed: {metadata.get('error_code')}"
|
||||
)
|
||||
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("Attachment did not become ready")
|
||||
|
||||
status, downloaded = api(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment_id}/content",
|
||||
token,
|
||||
)
|
||||
assert status == 200
|
||||
assert downloaded == original
|
||||
assert hashlib.sha256(downloaded).hexdigest() == (
|
||||
hashlib.sha256(original).hexdigest()
|
||||
)
|
||||
|
||||
status, thumbnail = api(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment_id}/content"
|
||||
"?variant=thumbnail",
|
||||
token,
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert thumbnail.startswith(b"\xff\xd8")
|
||||
assert thumbnail.endswith(b"\xff\xd9")
|
||||
|
||||
variants = metadata["variants"]
|
||||
assert any(
|
||||
v["name"] == "thumbnail"
|
||||
and v["width"] == 256
|
||||
and v["height"] == 160
|
||||
for v in variants
|
||||
)
|
||||
|
||||
print("Attachment: ready", flush=True)
|
||||
print("Original: SHA-256 verified", flush=True)
|
||||
print("Thumbnail: JPEG 256x160 verified", flush=True)
|
||||
print("PASS: durable worker recovery", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
# Restore the worker even when the test fails.
|
||||
docker("up", "-d", "worker")
|
||||
@@ -0,0 +1,127 @@
|
||||
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from fastapi import Request
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.instrumentation.fastapi import (
|
||||
FastAPIInstrumentor,
|
||||
)
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from src.telemetry import setup_telemetry
|
||||
|
||||
|
||||
def configure_api_observability(app):
|
||||
tracer, meter, logger = setup_telemetry(
|
||||
"messenger-api"
|
||||
)
|
||||
|
||||
request_counter = meter.create_counter(
|
||||
name="messenger_http_requests",
|
||||
unit="{request}",
|
||||
description="Completed business HTTP requests",
|
||||
)
|
||||
|
||||
class ObservabilityMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
# Health checks should not distort business RPS.
|
||||
if request.url.path.startswith("/health/"):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
request_id = uuid.uuid4().hex
|
||||
started = time.monotonic()
|
||||
status_code = 500
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
status_code = response.status_code
|
||||
|
||||
# Preserve the request ID produced by the existing
|
||||
# API error handler. It must match the JSON body.
|
||||
existing_id = response.headers.get("X-Request-Id")
|
||||
|
||||
if existing_id:
|
||||
request_id = existing_id
|
||||
else:
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
|
||||
return response
|
||||
|
||||
|
||||
finally:
|
||||
duration = time.monotonic() - started
|
||||
|
||||
# FastAPI provides a route template after routing.
|
||||
route = request.scope.get("route")
|
||||
|
||||
if route is not None:
|
||||
route_template = getattr(
|
||||
route, "path", None
|
||||
)
|
||||
else:
|
||||
route_template = None
|
||||
|
||||
# Do not use raw paths or dynamic IDs as labels.
|
||||
if not route_template:
|
||||
route_template = "unmatched"
|
||||
|
||||
method = request.method
|
||||
|
||||
if method not in (
|
||||
"GET", "POST", "PUT",
|
||||
"PATCH", "DELETE", "HEAD", "OPTIONS"
|
||||
):
|
||||
method = "OTHER"
|
||||
|
||||
status_class = f"{status_code // 100}xx"
|
||||
|
||||
attributes = {
|
||||
"http.request.method": method,
|
||||
"http.route": route_template,
|
||||
"http.status_class": status_class,
|
||||
}
|
||||
|
||||
request_counter.add(
|
||||
1,
|
||||
attributes=attributes,
|
||||
)
|
||||
|
||||
# The span is created by FastAPIInstrumentor.
|
||||
span = trace.get_current_span()
|
||||
|
||||
if span.is_recording():
|
||||
span.set_attribute(
|
||||
"messenger.request_id",
|
||||
request_id,
|
||||
)
|
||||
span.set_attribute(
|
||||
"messenger.status_class",
|
||||
status_class,
|
||||
)
|
||||
|
||||
# No Authorization header, body, filename,
|
||||
# chat ID, user ID or raw URL in telemetry.
|
||||
logger.info(
|
||||
"HTTP request completed",
|
||||
extra={
|
||||
"event": "http_request_completed",
|
||||
"request_id": request_id,
|
||||
"http_method": method,
|
||||
"http_route": route_template,
|
||||
"status_code": status_code,
|
||||
"duration_ms": round(
|
||||
duration * 1000, 3
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
# Add our middleware first so it executes within
|
||||
# the span created by the OTel ASGI middleware.
|
||||
app.add_middleware(ObservabilityMiddleware)
|
||||
|
||||
FastAPIInstrumentor.instrument_app(
|
||||
app,
|
||||
excluded_urls=r".*/health/.*",
|
||||
)
|
||||
@@ -0,0 +1,108 @@
|
||||
|
||||
import io
|
||||
import re
|
||||
import uuid
|
||||
import hashlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def valid_filename(filename):
|
||||
if not isinstance(filename, str):
|
||||
return False
|
||||
|
||||
if not 1 <= len(filename) <= 255:
|
||||
return False
|
||||
|
||||
if filename.strip() == "":
|
||||
return False
|
||||
|
||||
if "/" in filename or "\\" in filename:
|
||||
return False
|
||||
|
||||
if any(ord(ch) < 32 or ord(ch) == 127 for ch in filename):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def inspect_image(data, declared_mime):
|
||||
if len(data) > MAX_BYTES:
|
||||
raise ValueError("too_large")
|
||||
|
||||
if not data:
|
||||
raise ValueError("invalid_image")
|
||||
|
||||
if declared_mime not in ("image/png", "image/jpeg"):
|
||||
raise ValueError("invalid_image")
|
||||
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image_format = image.format
|
||||
|
||||
width, height = image.size
|
||||
|
||||
if width * height > MAX_PIXELS:
|
||||
raise ValueError("too_many_pixels")
|
||||
|
||||
if image_format not in ("PNG", "JPEG"):
|
||||
raise ValueError("invalid_image")
|
||||
|
||||
actual_mime = (
|
||||
"image/png"
|
||||
if image_format == "PNG"
|
||||
else "image/jpeg"
|
||||
)
|
||||
|
||||
if declared_mime != actual_mime:
|
||||
raise ValueError("invalid_image")
|
||||
|
||||
image.verify()
|
||||
|
||||
except Image.DecompressionBombError:
|
||||
raise ValueError("too_many_pixels")
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
SyntaxError,
|
||||
EOFError,
|
||||
):
|
||||
raise ValueError("invalid_image")
|
||||
|
||||
return {
|
||||
"content_type": actual_mime,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"size_bytes": len(data),
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
}
|
||||
|
||||
def attachment_json(row, variants=None):
|
||||
from datetime import timezone
|
||||
|
||||
def timestamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z")
|
||||
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["uploader_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": variants if variants is not None else [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
"updated_at": timestamp(row["updated_at"]),
|
||||
}
|
||||
|
||||
def object_key():
|
||||
return f"originals/{uuid.uuid4()}"
|
||||
@@ -0,0 +1,84 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from src.queue import publish_job
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
logger = logging.getLogger("image-dispatcher")
|
||||
|
||||
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||
|
||||
|
||||
def dispatch_one():
|
||||
with psycopg.connect(
|
||||
DATABASE_URL,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
|
||||
# Only publish due, unpublished jobs.
|
||||
# The row lock also prevents two dispatchers
|
||||
# from selecting the same job simultaneously.
|
||||
job = conn.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'queued'
|
||||
AND published_at IS NULL
|
||||
AND next_attempt_at <= NOW()
|
||||
ORDER BY created_at, id
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
).fetchone()
|
||||
|
||||
if job is None:
|
||||
return False
|
||||
|
||||
job_id = str(job["id"])
|
||||
|
||||
# RabbitMQ publisher confirms are enabled.
|
||||
# An exception rolls back this DB transaction.
|
||||
publish_job(job_id)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET published_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
logger.info("Published job %s", job_id)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
logger.info("Dispatcher started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
if not dispatch_one():
|
||||
time.sleep(1)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Dispatch failed; retrying"
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1277
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
|
||||
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
|
||||
|
||||
|
||||
def run_migrations():
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
with conn.cursor() as cur:
|
||||
# Prevent simultaneous migration runs.
|
||||
cur.execute("SELECT pg_advisory_xact_lock(2026002)")
|
||||
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL
|
||||
DEFAULT clock_timestamp()
|
||||
)
|
||||
""")
|
||||
|
||||
for file in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
cur.execute(
|
||||
"SELECT 1 FROM schema_migrations WHERE version = %s",
|
||||
(file.name,)
|
||||
)
|
||||
|
||||
if cur.fetchone():
|
||||
print(f"Already applied: {file.name}")
|
||||
continue
|
||||
|
||||
sql = file.read_text(encoding="utf-8")
|
||||
|
||||
for statement in sql.split(";"):
|
||||
if statement.strip():
|
||||
cur.execute(statement)
|
||||
|
||||
cur.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES (%s)",
|
||||
(file.name,)
|
||||
)
|
||||
|
||||
print(f"Applied migration: {file.name}")
|
||||
|
||||
print("Migrations completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migrations()
|
||||
@@ -0,0 +1,89 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
|
||||
|
||||
RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq")
|
||||
RABBITMQ_USER = os.environ["RABBITMQ_USER"]
|
||||
RABBITMQ_PASSWORD = os.environ["RABBITMQ_PASSWORD"]
|
||||
RABBITMQ_QUEUE = os.getenv(
|
||||
"RABBITMQ_QUEUE",
|
||||
"image_jobs",
|
||||
)
|
||||
|
||||
|
||||
def connect():
|
||||
credentials = pika.PlainCredentials(
|
||||
RABBITMQ_USER,
|
||||
RABBITMQ_PASSWORD,
|
||||
)
|
||||
|
||||
parameters = pika.ConnectionParameters(
|
||||
host=RABBITMQ_HOST,
|
||||
port=5672,
|
||||
credentials=credentials,
|
||||
heartbeat=30,
|
||||
blocked_connection_timeout=15,
|
||||
connection_attempts=3,
|
||||
retry_delay=2,
|
||||
)
|
||||
|
||||
return pika.BlockingConnection(parameters)
|
||||
|
||||
|
||||
def declare_queue(channel):
|
||||
channel.queue_declare(
|
||||
queue=RABBITMQ_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-queue-type": "quorum",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def publish_job(job_id):
|
||||
connection = connect()
|
||||
|
||||
try:
|
||||
channel = connection.channel()
|
||||
declare_queue(channel)
|
||||
|
||||
# Wait for confirmation from RabbitMQ.
|
||||
channel.confirm_delivery()
|
||||
|
||||
payload = json.dumps({
|
||||
"job_id": str(uuid.UUID(str(job_id))),
|
||||
}).encode("utf-8")
|
||||
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=RABBITMQ_QUEUE,
|
||||
body=payload,
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
message_id=str(job_id),
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
finally:
|
||||
if connection.is_open:
|
||||
connection.close()
|
||||
|
||||
|
||||
def parse_job(body):
|
||||
payload = json.loads(body)
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Invalid job payload")
|
||||
|
||||
job_id = payload.get("job_id")
|
||||
|
||||
if not isinstance(job_id, str):
|
||||
raise ValueError("Missing job_id")
|
||||
|
||||
return str(uuid.UUID(job_id))
|
||||
@@ -0,0 +1,60 @@
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||
region_name="us-east-1",
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
for attempt in range(30):
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
print(f"Bucket already exists: {bucket}", flush=True)
|
||||
break
|
||||
|
||||
except ClientError as exc:
|
||||
code = exc.response["Error"]["Code"]
|
||||
|
||||
if code in ("404", "NoSuchBucket"):
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
print(f"Bucket created: {bucket}", flush=True)
|
||||
break
|
||||
except Exception as create_error:
|
||||
print(
|
||||
f"Waiting for S3: {create_error}",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
print(f"Waiting for S3: {exc}", flush=True)
|
||||
|
||||
except Exception as exc:
|
||||
print(f"Waiting for S3: {exc}", flush=True)
|
||||
|
||||
if attempt == 29:
|
||||
raise RuntimeError("Cannot initialize S3 bucket")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
# No public bucket policy is configured.
|
||||
print(f"Private S3 bucket is ready: {bucket}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,168 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
|
||||
OTLPLogExporter,
|
||||
)
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
||||
OTLPMetricExporter,
|
||||
)
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import (
|
||||
BatchLogRecordProcessor,
|
||||
)
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import (
|
||||
PeriodicExportingMetricReader,
|
||||
)
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import (
|
||||
BatchSpanProcessor,
|
||||
)
|
||||
|
||||
|
||||
SERVICE_VERSION = "lab6-1.0"
|
||||
ENVIRONMENT = "lab"
|
||||
|
||||
OTLP_ENDPOINT = os.getenv(
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
"http://otel-collector:4318",
|
||||
).rstrip("/")
|
||||
|
||||
|
||||
class TraceContextFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
context = trace.get_current_span().get_span_context()
|
||||
|
||||
if context.is_valid:
|
||||
record.trace_id = format(
|
||||
context.trace_id, "032x"
|
||||
)
|
||||
record.span_id = format(
|
||||
context.span_id, "016x"
|
||||
)
|
||||
else:
|
||||
record.trace_id = "-"
|
||||
record.span_id = "-"
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def setup_telemetry(service_name):
|
||||
"""
|
||||
Initialize OTLP traces, metrics and logs.
|
||||
|
||||
Do not record request bodies, Authorization
|
||||
headers, message text or image bytes.
|
||||
"""
|
||||
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": service_name,
|
||||
"service.version": SERVICE_VERSION,
|
||||
"service.instance.id": os.getenv(
|
||||
"HOSTNAME",
|
||||
socket.gethostname(),
|
||||
),
|
||||
"deployment.environment.name": ENVIRONMENT,
|
||||
}
|
||||
)
|
||||
|
||||
# Distributed traces -> Collector -> Tempo
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
|
||||
trace_exporter = OTLPSpanExporter(
|
||||
endpoint=f"{OTLP_ENDPOINT}/v1/traces",
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
trace_exporter,
|
||||
max_queue_size=1024,
|
||||
)
|
||||
)
|
||||
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
# Metrics -> Collector -> Prometheus
|
||||
metric_exporter = OTLPMetricExporter(
|
||||
endpoint=f"{OTLP_ENDPOINT}/v1/metrics",
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
metric_exporter,
|
||||
export_interval_millis=5000,
|
||||
export_timeout_millis=3000,
|
||||
)
|
||||
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[metric_reader],
|
||||
)
|
||||
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
# Structured logs -> Collector -> Loki
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
|
||||
log_exporter = OTLPLogExporter(
|
||||
endpoint=f"{OTLP_ENDPOINT}/v1/logs",
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
log_exporter,
|
||||
max_queue_size=1024,
|
||||
)
|
||||
)
|
||||
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
# Dedicated application logger.
|
||||
# We intentionally do not forward arbitrary
|
||||
# library logs or sensitive request data.
|
||||
logger = logging.getLogger("messenger.telemetry")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
|
||||
for handler in list(logger.handlers):
|
||||
logger.removeHandler(handler)
|
||||
|
||||
context_filter = TraceContextFilter()
|
||||
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.addFilter(context_filter)
|
||||
console_handler.setFormatter(
|
||||
logging.Formatter(
|
||||
"%(asctime)s %(levelname)s "
|
||||
"event=%(event)s "
|
||||
"trace_id=%(trace_id)s "
|
||||
"span_id=%(span_id)s "
|
||||
"%(message)s"
|
||||
)
|
||||
)
|
||||
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
otlp_handler = LoggingHandler(
|
||||
level=logging.INFO,
|
||||
logger_provider=logger_provider,
|
||||
)
|
||||
otlp_handler.addFilter(context_filter)
|
||||
logger.addHandler(otlp_handler)
|
||||
|
||||
tracer = trace.get_tracer(service_name, SERVICE_VERSION)
|
||||
meter = metrics.get_meter(service_name, SERVICE_VERSION)
|
||||
|
||||
return tracer, meter, logger
|
||||
+576
@@ -0,0 +1,576 @@
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import boto3
|
||||
import pika
|
||||
import psycopg
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from src.telemetry import setup_telemetry
|
||||
from botocore.config import Config
|
||||
from PIL import Image, ImageOps
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from src.queue import (
|
||||
RABBITMQ_QUEUE,
|
||||
connect,
|
||||
declare_queue,
|
||||
parse_job,
|
||||
)
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
logger = logging.getLogger("image-worker")
|
||||
|
||||
worker_tracer, worker_meter, worker_logger = setup_telemetry(
|
||||
"messenger-worker"
|
||||
)
|
||||
|
||||
jobs_processed = worker_meter.create_counter(
|
||||
name="messenger_worker_jobs_processed",
|
||||
unit="{job}",
|
||||
description="Image processing job attempts by outcome",
|
||||
)
|
||||
|
||||
job_duration = worker_meter.create_histogram(
|
||||
name="messenger_worker_job_duration",
|
||||
unit="s",
|
||||
description="Image processing duration in seconds",
|
||||
)
|
||||
|
||||
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||
S3_BUCKET = os.environ["S3_BUCKET"]
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||
region_name="us-east-1",
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=3,
|
||||
read_timeout=15,
|
||||
retries={"max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def make_thumbnail(original):
|
||||
try:
|
||||
with Image.open(io.BytesIO(original)) as image:
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise InvalidImage("Unsupported image")
|
||||
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise InvalidImage("Too many pixels")
|
||||
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.thumbnail(
|
||||
(256, 256),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
|
||||
# JPEG does not support transparency.
|
||||
rgba = image.convert("RGBA")
|
||||
|
||||
background = Image.new(
|
||||
"RGB",
|
||||
rgba.size,
|
||||
(255, 255, 255),
|
||||
)
|
||||
|
||||
background.paste(
|
||||
rgba,
|
||||
mask=rgba.getchannel("A"),
|
||||
)
|
||||
|
||||
output = io.BytesIO()
|
||||
|
||||
background.save(
|
||||
output,
|
||||
format="JPEG",
|
||||
quality=85,
|
||||
optimize=True,
|
||||
)
|
||||
|
||||
return (
|
||||
output.getvalue(),
|
||||
background.width,
|
||||
background.height,
|
||||
)
|
||||
|
||||
except (OSError, SyntaxError, EOFError) as exc:
|
||||
raise InvalidImage("Invalid image") from exc
|
||||
|
||||
|
||||
def get_staging_object(key):
|
||||
response = s3.get_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=key,
|
||||
)
|
||||
|
||||
stream = response["Body"]
|
||||
|
||||
try:
|
||||
data = stream.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
if not data or len(data) > MAX_BYTES:
|
||||
raise InvalidImage("Invalid staging size")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def process_job(job_id):
|
||||
"""
|
||||
Use a PostgreSQL row lock to prevent concurrent
|
||||
processing of the same logical job.
|
||||
|
||||
The transaction commits before RabbitMQ ACK.
|
||||
"""
|
||||
|
||||
with psycopg.connect(
|
||||
DATABASE_URL,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
|
||||
job = conn.execute(
|
||||
"""
|
||||
SELECT j.*, a.staging_key,
|
||||
a.object_key, a.sha256,
|
||||
a.size_bytes, a.status AS attachment_status
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a
|
||||
ON a.id = j.attachment_id
|
||||
WHERE j.id = %s
|
||||
FOR UPDATE OF j, a
|
||||
""",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
|
||||
if job is None:
|
||||
logger.info("Unknown job %s, ignoring", job_id)
|
||||
return
|
||||
|
||||
if job["status"] == "ready":
|
||||
logger.info("Job %s already completed", job_id)
|
||||
return
|
||||
|
||||
if job["status"] == "failed":
|
||||
logger.info("Job %s already failed", job_id)
|
||||
return
|
||||
|
||||
if job["next_attempt_at"] is not None:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
if job["next_attempt_at"] > datetime.now(timezone.utc):
|
||||
logger.info("Job %s is not due yet", job_id)
|
||||
return
|
||||
|
||||
attempts = job["attempts"] + 1
|
||||
|
||||
if attempts > job["max_attempts"]:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed',
|
||||
completed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = 'processing_failed',
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing',
|
||||
attempts = %s,
|
||||
started_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(attempts, job_id),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'processing',
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
|
||||
try:
|
||||
original = get_staging_object(
|
||||
job["staging_key"]
|
||||
)
|
||||
|
||||
if (
|
||||
len(original) != job["size_bytes"]
|
||||
or hashlib.sha256(original).hexdigest()
|
||||
!= job["sha256"]
|
||||
):
|
||||
raise InvalidImage("Staging checksum mismatch")
|
||||
|
||||
thumbnail, width, height = make_thumbnail(
|
||||
original
|
||||
)
|
||||
|
||||
attachment_id = str(job["attachment_id"])
|
||||
|
||||
# Stable keys make duplicate delivery safe.
|
||||
original_key = job["object_key"]
|
||||
thumb_key = (
|
||||
f"thumbnails/{attachment_id}.jpg"
|
||||
)
|
||||
|
||||
s3.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=original_key,
|
||||
Body=original,
|
||||
ContentType="image/png"
|
||||
if original[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
else "image/jpeg",
|
||||
)
|
||||
|
||||
s3.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=thumb_key,
|
||||
Body=thumbnail,
|
||||
ContentType="image/jpeg",
|
||||
)
|
||||
|
||||
thumb_sha256 = hashlib.sha256(
|
||||
thumbnail
|
||||
).hexdigest()
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_variants
|
||||
(
|
||||
attachment_id,
|
||||
variant_name,
|
||||
object_key,
|
||||
content_type,
|
||||
size_bytes,
|
||||
sha256,
|
||||
width,
|
||||
height
|
||||
)
|
||||
VALUES (
|
||||
%s, 'thumbnail', %s, 'image/jpeg',
|
||||
%s, %s, %s, %s
|
||||
)
|
||||
ON CONFLICT (
|
||||
attachment_id, variant_name
|
||||
)
|
||||
DO UPDATE SET
|
||||
object_key = EXCLUDED.object_key,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height
|
||||
""",
|
||||
(
|
||||
job["attachment_id"],
|
||||
thumb_key,
|
||||
len(thumbnail),
|
||||
thumb_sha256,
|
||||
width,
|
||||
height,
|
||||
),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
error_code = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready',
|
||||
completed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Job %s ready, thumbnail %sx%s",
|
||||
job_id,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Processing failed for job %s: %s",
|
||||
job_id,
|
||||
type(exc).__name__,
|
||||
)
|
||||
|
||||
if (
|
||||
attempts >= job["max_attempts"]
|
||||
or isinstance(exc, InvalidImage)
|
||||
):
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed',
|
||||
completed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = 'processing_failed',
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
next_attempt_at =
|
||||
NOW() + INTERVAL '5 seconds',
|
||||
published_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
|
||||
|
||||
def get_job_status(job_id):
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT status
|
||||
FROM attachment_jobs
|
||||
WHERE id = %s
|
||||
""",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def on_message(channel, method, properties, body):
|
||||
try:
|
||||
job_id = parse_job(body)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
logger.warning("Invalid queue message")
|
||||
channel.basic_ack(
|
||||
delivery_tag=method.delivery_tag
|
||||
)
|
||||
return
|
||||
|
||||
attributes = {
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": RABBITMQ_QUEUE,
|
||||
"messaging.operation.name": "process",
|
||||
"messenger.job_type": "image_thumbnail",
|
||||
}
|
||||
|
||||
# A real consumer span around the actual job processing.
|
||||
with worker_tracer.start_as_current_span(
|
||||
"image_job.process",
|
||||
kind=SpanKind.CONSUMER,
|
||||
attributes=attributes,
|
||||
) as span:
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
try:
|
||||
previous_status = get_job_status(job_id)
|
||||
|
||||
process_job(job_id)
|
||||
|
||||
current_status = get_job_status(job_id)
|
||||
duration = time.monotonic() - started
|
||||
|
||||
if previous_status in ("ready", "failed"):
|
||||
outcome = "duplicate"
|
||||
|
||||
elif current_status == "ready":
|
||||
outcome = "success"
|
||||
|
||||
elif current_status == "failed":
|
||||
outcome = "failed"
|
||||
|
||||
elif current_status == "queued":
|
||||
outcome = "retry"
|
||||
|
||||
else:
|
||||
outcome = "ignored"
|
||||
|
||||
span.set_attribute(
|
||||
"messenger.job.outcome",
|
||||
outcome,
|
||||
)
|
||||
|
||||
if outcome == "failed":
|
||||
span.set_status(
|
||||
Status(StatusCode.ERROR)
|
||||
)
|
||||
|
||||
# Count actual processing attempts, not
|
||||
# duplicate deliveries of completed jobs.
|
||||
if outcome in ("success", "failed", "retry"):
|
||||
metric_attributes = {
|
||||
"job_type": "image_thumbnail",
|
||||
"outcome": outcome,
|
||||
}
|
||||
|
||||
jobs_processed.add(
|
||||
1,
|
||||
attributes=metric_attributes,
|
||||
)
|
||||
|
||||
job_duration.record(
|
||||
duration,
|
||||
attributes=metric_attributes,
|
||||
)
|
||||
|
||||
worker_logger.info(
|
||||
"Image job handled",
|
||||
extra={
|
||||
"event": "image_job_handled",
|
||||
"job_type": "image_thumbnail",
|
||||
"outcome": outcome,
|
||||
"duration_seconds": round(duration, 4),
|
||||
},
|
||||
)
|
||||
|
||||
# ACK after process_job has committed the
|
||||
# PostgreSQL transaction.
|
||||
channel.basic_ack(
|
||||
delivery_tag=method.delivery_tag
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
span.record_exception(exc)
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
|
||||
worker_logger.error(
|
||||
"Image job processing interrupted",
|
||||
extra={
|
||||
"event": "image_job_interrupted",
|
||||
"job_type": "image_thumbnail",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
|
||||
logger.exception(
|
||||
"Unexpected failure for job %s",
|
||||
job_id,
|
||||
)
|
||||
|
||||
channel.basic_nack(
|
||||
delivery_tag=method.delivery_tag,
|
||||
requeue=True,
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
while True:
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
|
||||
declare_queue(channel)
|
||||
|
||||
channel.basic_qos(
|
||||
prefetch_count=1
|
||||
)
|
||||
|
||||
channel.basic_consume(
|
||||
queue=RABBITMQ_QUEUE,
|
||||
on_message_callback=on_message,
|
||||
auto_ack=False,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Worker listening on %s",
|
||||
RABBITMQ_QUEUE,
|
||||
)
|
||||
|
||||
channel.start_consuming()
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Worker disconnected, reconnecting"
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
finally:
|
||||
if (
|
||||
"connection" in locals()
|
||||
and connection.is_open
|
||||
):
|
||||
connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user