forked from mtusi2026/mtusi2026lab6
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d34a03c80 | ||
|
|
ad8db1f32b |
@@ -18,3 +18,10 @@ __pycache__
|
|||||||
node_modules
|
node_modules
|
||||||
evidence
|
evidence
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
.git
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
__pycache__
|
||||||
|
.pytest_cache
|
||||||
|
*.pyc
|
||||||
|
evidence
|
||||||
+5
-5
@@ -23,8 +23,8 @@ WORKER_CONCURRENCY=2
|
|||||||
JOB_MAX_ATTEMPTS=5
|
JOB_MAX_ATTEMPTS=5
|
||||||
JOB_TIMEOUT_SECONDS=60
|
JOB_TIMEOUT_SECONDS=60
|
||||||
STAGING_RETENTION_SECONDS=86400
|
STAGING_RETENTION_SECONDS=86400
|
||||||
OTEL_SERVICE_NAME=messenger-api
|
RABBITMQ_USER=messenger
|
||||||
# Worker must set a distinct service.name.
|
RABBITMQ_PASSWORD=change-me
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
RABBITMQ_QUEUE=image_jobs
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
IMAGE_MAX_ATTEMPTS=5
|
||||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
GRAFANA_ADMIN_PASSWORD=replace_me
|
||||||
@@ -30,3 +30,8 @@ coverage/
|
|||||||
.vscode/
|
.vscode/
|
||||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||||
|
.env
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
evidence/*.tmp
|
||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY src/ ./src/
|
||||||
|
COPY migrations/ ./migrations/
|
||||||
|
|
||||||
|
ENV HOST=0.0.0.0
|
||||||
|
ENV PORT=8080
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||||
@@ -1,37 +1,329 @@
|
|||||||
# Отчёт — лабораторная 6
|
# Отчёт — лабораторная 6
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- **Автор, группа:** Матвиенко Павел, БПИ2304
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- **Целевая оценка; выбранный трек 5, если нужен:** 3.
|
||||||
- Commit/PR: TODO после публикации
|
- **Commit/PR:** ad8db1f32b898ae43df22b1d3c0a431e553beff8; https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/71
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- **Использованные источники, библиотеки и помощь ИИ:** задание и контракты лабораторной №6, документация OpenTelemetry, Prometheus, Loki, Tempo, Grafana, FastAPI, Python SDK; при разработке и отладке использовалась помощь ИИ.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
**Среда:** Windows с WSL Ubuntu и Docker Desktop с поддержкой Docker Compose.
|
||||||
|
|
||||||
|
**Необходимое ПО:** Git, Docker Engine / Docker Desktop, Docker Compose, Python 3, GNU Make.
|
||||||
|
|
||||||
|
**Основные версии компонентов:**
|
||||||
|
|
||||||
|
| Компонент | Версия |
|
||||||
|
| --- | --- |
|
||||||
|
| PostgreSQL | 16-alpine |
|
||||||
|
| RustFS | 1.0.3 |
|
||||||
|
| RabbitMQ | 4.1.8-management |
|
||||||
|
| Nginx | 1.29-alpine |
|
||||||
|
| OpenTelemetry Collector Contrib | 0.136.0 |
|
||||||
|
| OpenTelemetry Python SDK | 1.38.0 |
|
||||||
|
| OpenTelemetry FastAPI instrumentation | 0.59b0 |
|
||||||
|
| Prometheus | 3.5.0 |
|
||||||
|
| Loki | 3.5.5 |
|
||||||
|
| Tempo | 2.8.2 |
|
||||||
|
| Grafana | 12.1.1 |
|
||||||
|
|
||||||
|
**Подготовка окружения:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env
|
||||||
|
```
|
||||||
|
|
||||||
|
В `.env` необходимо установить собственные локальные значения паролей PostgreSQL, RabbitMQ, администратора Grafana и ключей S3. Для остальных параметров использовать значения и комментарии из `.env.example`.
|
||||||
|
|
||||||
|
Файл `.env` не должен попадать в Git. Секреты в отчёте не публикуются.
|
||||||
|
|
||||||
|
**Сборка и запуск:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
При запуске через Compose выполняются предусмотренные конфигурацией миграции PostgreSQL и инициализация приватного S3 bucket.
|
||||||
|
|
||||||
|
Проверить состояние:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose ps -a
|
||||||
|
```
|
||||||
|
|
||||||
|
Проверить готовность бизнес-API:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://127.0.0.1:8080/health/ready
|
||||||
|
```
|
||||||
|
|
||||||
|
Ожидаемый результат:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"status":"ok"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Проверить публичный контракт:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test
|
||||||
|
```
|
||||||
|
|
||||||
|
Проверить корреляцию логов и трасс:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose exec -T api1 python - < scripts/test-trace-correlation.py
|
||||||
|
```
|
||||||
|
|
||||||
|
**Доступ к Grafana:** `http://localhost:3000`. Логин администратора — `admin`; пароль задаётся через `GRAFANA_ADMIN_PASSWORD`.
|
||||||
|
|
||||||
|
Dashboard автоматически импортируется из файла `observability/grafana/dashboards/messenger-lab6.json`.
|
||||||
|
|
||||||
|
**Безопасная остановка с сохранением данных:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
**Намеренное удаление всех учебных данных и volumes:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose down -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Последняя команда используется только при осознанном сбросе стенда, поскольку удаляет сохранённые данные приложений и подсистемы наблюдаемости.
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
**Схема процессов и хранилищ:**
|
||||||
|
|
||||||
|
```text
|
||||||
|
Client
|
||||||
|
|
|
||||||
|
Nginx
|
||||||
|
|
|
||||||
|
+------ API1 / API2 ------ PostgreSQL
|
||||||
|
| |
|
||||||
|
| +---------- RustFS (S3)
|
||||||
|
|
|
||||||
|
Dispatcher ---- RabbitMQ ---- Worker
|
||||||
|
| |
|
||||||
|
+------------+-------------+
|
||||||
|
|
|
||||||
|
OTLP / HTTP
|
||||||
|
|
|
||||||
|
OpenTelemetry Collector
|
||||||
|
|
|
||||||
|
+-------+-------+
|
||||||
|
| | |
|
||||||
|
Prometheus Loki Tempo
|
||||||
|
| | |
|
||||||
|
+-------+-------+
|
||||||
|
|
|
||||||
|
Grafana
|
||||||
|
```
|
||||||
|
|
||||||
|
**Что перенесено из lab5:**
|
||||||
|
|
||||||
|
- Две реплики FastAPI за Nginx.
|
||||||
|
- Авторизация, пользователи, чаты, сообщения и вложения.
|
||||||
|
- PostgreSQL для транзакционных данных, сессий и заданий.
|
||||||
|
- RustFS как S3-совместимое хранилище изображений.
|
||||||
|
- RabbitMQ с устойчивой очередью `image_jobs`.
|
||||||
|
- Отдельные процессы dispatcher и worker.
|
||||||
|
- Асинхронная обработка PNG/JPEG и создание thumbnail.
|
||||||
|
- Механизмы восстановления заданий, обработки повторных доставок и миграции БД.
|
||||||
|
|
||||||
|
**Что добавлено в lab6:**
|
||||||
|
|
||||||
|
- OpenTelemetry SDK для API и worker.
|
||||||
|
- Автоматические HTTP server spans FastAPI.
|
||||||
|
- Consumer spans обработки изображений.
|
||||||
|
- Счётчики HTTP-запросов и обработки заданий.
|
||||||
|
- Гистограммы длительности HTTP-запросов и работы worker.
|
||||||
|
- Структурированные логи с `trace_id`, `span_id` и `service.name`.
|
||||||
|
- Отдельный OpenTelemetry Collector.
|
||||||
|
- Prometheus, Loki, Tempo и Grafana.
|
||||||
|
- Provisioning источников данных и импортируемого dashboard.
|
||||||
|
- Скрипты проверки корреляции и контролируемых инцидентов.
|
||||||
|
|
||||||
|
**Назначение хранилищ и источник истины:**
|
||||||
|
|
||||||
|
PostgreSQL остаётся источником истины для пользователей, сообщений, метаданных и состояний заданий. RustFS хранит бинарные объекты изображений. RabbitMQ доставляет задания worker.
|
||||||
|
|
||||||
|
Prometheus хранит временные ряды метрик; Loki — структурированные логи; Tempo — трассы. Grafana обеспечивает отображение и поиск данных, но не заменяет эти хранилища. Collector принимает, обрабатывает и экспортирует телеметрию, не являясь долговременной базой данных.
|
||||||
|
|
||||||
|
**Транзакционные границы:**
|
||||||
|
|
||||||
|
Принятие изображения фиксируется сохранением исходных байтов в RustFS и записи задания в PostgreSQL. Передача задания в RabbitMQ допускает повторную доставку. Worker проверяет состояние задания и сохраняет результат до подтверждения обработки сообщения брокеру.
|
||||||
|
|
||||||
|
Между PostgreSQL и S3 отсутствует единая распределённая транзакция. Используются повторные попытки и идемпотентная обработка.
|
||||||
|
|
||||||
|
**Ресурсы и конфигурация OpenTelemetry:**
|
||||||
|
|
||||||
|
Для API применяется `service.name=messenger-api`, для worker — `service.name=messenger-worker`. Используются также `service.version`, `service.instance.id` и `deployment.environment.name=lab`.
|
||||||
|
|
||||||
|
API и worker отправляют OTLP/HTTP в Collector на `otel-collector:4318`. Collector экспортирует метрики в Prometheus-compatible endpoint, логи через OTLP/HTTP в Loki, трассы через OTLP/gRPC в Tempo.
|
||||||
|
|
||||||
|
Пароли, Authorization, session token, тексты сообщений и байты изображений не включаются в специально создаваемые события телеметрии. Пользовательские UUID не используются как labels метрик. Для HTTP-маршрутов используются шаблоны, а не фактические URL с идентификаторами.
|
||||||
|
|
||||||
|
**Фактические имена метрик:**
|
||||||
|
|
||||||
|
| Логический сигнал | Имя в Prometheus | Единица |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| HTTP requests | `messenger_messenger_http_requests_total` | запросы |
|
||||||
|
| HTTP duration | `messenger_http_server_duration_milliseconds_bucket` | миллисекунды |
|
||||||
|
| Jobs processed / failed | `messenger_messenger_worker_jobs_processed_total` | попытки |
|
||||||
|
| Job processing duration | `messenger_messenger_worker_job_duration_seconds_bucket` | секунды |
|
||||||
|
|
||||||
|
Для API error ratio числитель — HTTP 5xx, знаменатель — все учитываемые бизнес-запросы, без health checks. Значения p95 рассчитываются из гистограмм с агрегацией buckets. Миллисекунды HTTP duration преобразуются в секунды.
|
||||||
|
|
||||||
|
**Ограничения архитектуры:**
|
||||||
|
|
||||||
|
Сквозной trace context между HTTP upload и обработкой задания worker не реализован. Логи API и worker коррелируют со своими реальными трассами, но не образуют единую parent-child трассу через RabbitMQ.
|
||||||
|
|
||||||
|
Возможности дополнительных метрик очереди, насыщения ресурсов и alert rules относятся к более высокому уровню и в текущей реализации не заявлены.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Публичный smoke | `make test` | PASS, бизнес-API работает с включённой телеметрией | `evidence/smoke.txt` |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Smoke после инцидентов | `make test` | PASS после восстановления PostgreSQL и worker | `evidence/smoke-after-incidents.txt` |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| Collector и хранилища трёх сигналов | `docker compose ps -a` | Collector healthy; Prometheus, Loki, Tempo и Grafana запущены | `evidence/containers.txt` |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| HTTP request counter | Запрос Prometheus API | Найдена `messenger_messenger_http_requests_total` | `evidence/prometheus-metrics.json` |
|
||||||
|
| HTTP duration histogram | Запрос Prometheus API | Найдены buckets, sum и count HTTP duration | `evidence/prometheus-metrics.json` |
|
||||||
|
| Worker jobs counter | Выполнение `make test` и запрос Prometheus API | Найден счётчик заданий с outcome | `evidence/prometheus-metrics.json` |
|
||||||
|
| Worker duration histogram | Выполнение `make test` и запрос Prometheus API | Найдена гистограмма длительности worker | `evidence/prometheus-metrics.json` |
|
||||||
|
| Структурированные логи API | Запрос Loki / Grafana Explore | Найдены HTTP-события с техническими атрибутами | `evidence/grafana-dashboard.png` |
|
||||||
|
| Структурированные логи worker | Журналы worker и запрос Loki | Найдено событие `Image job handled` | `evidence/worker-telemetry.txt` |
|
||||||
|
| Реальная трасса API | Loki → Tempo по trace ID | PASS, 4 span, включая server span | `evidence/trace-correlation.txt` |
|
||||||
|
| Реальная трасса worker | Loki → Tempo по trace ID | PASS, consumer span `image_job.process` | `evidence/trace-correlation.txt` |
|
||||||
|
| Импортируемый dashboard | Provisioning Grafana | Dashboard загружен, 8 панелей | `observability/grafana/dashboards/messenger-lab6.json` |
|
||||||
|
| Контролируемый отказ worker | Остановка worker, upload, восстановление | PASS; задание сохранено, затем обработано | `evidence/incident-worker.txt` |
|
||||||
|
| Контролируемый отказ PostgreSQL | `python3 scripts/test-db-incident.py` | 5/5 неудачных запросов, восстановление HTTP 201 | `evidence/incident-db.txt` |
|
||||||
|
| Критерии уровня 4 | Не заявлены | Дополнительные метрики и alerts не реализованы | Не применяется |
|
||||||
|
| Трек уровня 5 | Не заявлен | Не выполнялся | Не применяется |
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
**Подтверждённая корреляция API:**
|
||||||
|
|
||||||
|
- Service: `messenger-api`
|
||||||
|
- Event: `HTTP request completed`
|
||||||
|
- Trace ID: `c3ea28d96d7c536e222d8e4c7cba2727`
|
||||||
|
- Span ID: `3be2550ec51156e2`
|
||||||
|
- Результат: найден реальный HTTP server span `POST /api/v1/auth/sessions`.
|
||||||
|
|
||||||
|
**Подтверждённая корреляция worker:**
|
||||||
|
|
||||||
|
- Service: `messenger-worker`
|
||||||
|
- Event: `Image job handled`
|
||||||
|
- Trace ID: `af16dbcb7b7ca25b06d39902dc48afd8`
|
||||||
|
- Span ID: `644d3cda73387e3f`
|
||||||
|
- Результат: найден consumer span `image_job.process`.
|
||||||
|
|
||||||
|
Обе проверки завершились `PASS: API and worker trace correlation`.
|
||||||
|
|
||||||
|
Dashboard и конфигурации provisioning сохранены в репозитории, поэтому после чистого запуска они загружаются без ручной настройки через браузер.
|
||||||
|
|
||||||
|
Проверки инфраструктуры подтверждаются состоянием контейнеров и результатами отдельных запросов к Prometheus, Loki и Tempo; один только публичный smoke не считается доказательством работающей наблюдаемости.
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Эксперименты и отказы
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
**Гипотезы:**
|
||||||
|
|
||||||
|
1. При остановке worker API сохранит возможность принимать изображения, а принятые задания будут обработаны после восстановления worker.
|
||||||
|
2. При остановке PostgreSQL запросы, требующие доступа к БД, начнут завершаться ошибками; после восстановления БД авторизация снова заработает.
|
||||||
|
3. Метрики, логи и трассы позволят различить отсутствие фоновой обработки и недоступность базы данных.
|
||||||
|
|
||||||
|
**Условия проведения:**
|
||||||
|
|
||||||
|
- Среда: локальный Docker Desktop с WSL Ubuntu.
|
||||||
|
- Контейнеры: две API-реплики, PostgreSQL, RustFS, RabbitMQ, dispatcher, worker, Collector, Prometheus, Loki, Tempo, Grafana.
|
||||||
|
- Нагрузка: синтетические операции публичного smoke и специальные последовательные запросы скриптов инцидентов.
|
||||||
|
- Concurrency: в сценарии отказа БД проверочные запросы выполнялись последовательно.
|
||||||
|
- Прогрев: перед отказом выполнялись успешные запросы к API.
|
||||||
|
- Инцидент worker: принудительная остановка worker и дополнительная пауза 60 секунд.
|
||||||
|
- Инцидент PostgreSQL: остановка контейнера БД, пять проверочных запросов и восстановление.
|
||||||
|
- Аппаратная конфигурация, выделенные CPU/RAM и лимиты Docker: отдельно не фиксировались.
|
||||||
|
- Продолжительная нагрузка с фиксированным arrival rate и несколькими уровнями concurrency: не выполнялась.
|
||||||
|
- Повторяемость: приведены результаты одного зафиксированного прогона каждого контролируемого инцидента.
|
||||||
|
|
||||||
|
**Зафиксированные измерения:**
|
||||||
|
|
||||||
|
Для инцидента PostgreSQL:
|
||||||
|
|
||||||
|
| Запрос | Результат | Длительность |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| До отказа | HTTP 201 | Не зафиксирована в выводе |
|
||||||
|
| Probe 1 | TimeoutError | 5,005 с |
|
||||||
|
| Probe 2 | HTTP 503 | 0,046 с |
|
||||||
|
| Probe 3 | HTTP 503 | 4,071 с |
|
||||||
|
| Probe 4 | HTTP 503 | 0,038 с |
|
||||||
|
| Probe 5 | HTTP 503 | 0,076 с |
|
||||||
|
| После восстановления | HTTP 201 | 0,058 с |
|
||||||
|
|
||||||
|
Во время отказа БД все пять запросов завершились неуспешно: четыре HTTP 503 и один клиентский тайм-аут.
|
||||||
|
|
||||||
|
В Grafana зафиксирован рост доли HTTP 5xx примерно до 67% в двухминутном окне, а также изменение p95 длительности запросов.
|
||||||
|
|
||||||
|
Отдельные нагрузочные измерения устойчивого RPS, p50/p95/p99 по серии прогонов, CPU/RAM и распределения задержек не собирались. Точки на dashboard характеризуют конкретный наблюдаемый период, но не являются результатом полноценного нагрузочного теста.
|
||||||
|
|
||||||
|
**Таблица инцидентов:**
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| TODO | TODO | TODO | TODO |
|
| Остановка worker | API принимает upload; задания остаются в `queued` | HTTP 202, сообщение создано, после запуска worker статус `ready` | Пауза 60 секунд; потеря принятого изображения не обнаружена |
|
||||||
|
| Перезапуск RabbitMQ и API при остановленном worker | Принятое задание не теряется | Статус `queued` сохранился после перезапуска | После возобновления worker задание обработано |
|
||||||
|
| Повторное включение worker | Появляется обработка и thumbnail | Создан JPEG 256×160, оригинал проверен SHA-256 | Точное время перехода в `ready` отдельно не измерялось |
|
||||||
|
| Остановка PostgreSQL | Авторизация, зависящая от БД, перестаёт работать | 4 HTTP 503 и 1 TimeoutError | После восстановления БД — HTTP 201 за 0,058 с |
|
||||||
|
| Повторный smoke после отказов | Вся бизнес-функциональность восстановлена | `make test` — PASS | Потерь подтверждённых данных в проверенных сценариях не обнаружено |
|
||||||
|
|
||||||
|
**Анализ инцидента worker:**
|
||||||
|
|
||||||
|
На dashboard в период остановки сохранялись успешные HTTP-запросы, но новые успешные обработки worker не появлялись. После восстановления worker был зафиксирован новый успешный job и данные о длительности обработки.
|
||||||
|
|
||||||
|
Причина: отсутствие активного фонового обработчика, а не отказ API. Это подтверждено контролируемой остановкой контейнера и результатом `PASS: durable worker recovery`.
|
||||||
|
|
||||||
|
Доказательства: `evidence/incident-worker.txt`, `evidence/incident-worker-dashboard.png`, `evidence/incident-worker-recovered.png`.
|
||||||
|
|
||||||
|
**Анализ инцидента PostgreSQL:**
|
||||||
|
|
||||||
|
Во время остановки БД запросы авторизации завершались HTTP 503 или тайм-аутом. График 5xx в Grafana показал рост серверных ошибок. После восстановления PostgreSQL авторизация вернулась к HTTP 201.
|
||||||
|
|
||||||
|
Причина: недоступность PostgreSQL. Она подтверждается характером запросов, контролируемой остановкой контейнера, журналами и восстановлением после запуска БД.
|
||||||
|
|
||||||
|
Отдельные DB client spans отсутствуют, поэтому точная локализация по одной только трассе API ограничена.
|
||||||
|
|
||||||
|
Доказательства: `evidence/incident-db.txt`, `evidence/incident-db-dashboard.png`, `evidence/incident-db-api-logs.txt`, `evidence/incident-db-postgres-logs.txt`.
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
**Какие отказы выдерживаются:**
|
||||||
|
|
||||||
|
- Остановка worker не приводит к потере принятого задания при сохранности PostgreSQL и RustFS.
|
||||||
|
- Перезапуск RabbitMQ и API не нарушает сохранность зафиксированного задания.
|
||||||
|
- После восстановления PostgreSQL приложение возвращается к нормальной работе.
|
||||||
|
- Публичный smoke проходит при включённой телеметрии.
|
||||||
|
|
||||||
|
**Какие отказы и ограничения остаются:**
|
||||||
|
|
||||||
|
- PostgreSQL, RustFS и RabbitMQ развёрнуты одиночными экземплярами и остаются точками отказа.
|
||||||
|
- При удалении volumes учебные данные и история наблюдаемости теряются.
|
||||||
|
- Трассы API и worker не связаны единым W3C-контекстом через брокер.
|
||||||
|
- Нет отдельной SQL-инструментации с DB client spans.
|
||||||
|
- Метрики глубины и возраста очереди, насыщения ресурсов и end-to-end времени до `ready` не реализованы.
|
||||||
|
- Alert rules и автоматическая проверка их переходов в firing/resolved не реализованы.
|
||||||
|
- Не проводилось отдельное испытание недоступности backend-телеметрии и измерение потерь.
|
||||||
|
- Метрики worker могут сбрасываться при перезапуске процесса; для анализа необходимы временные ряды и события восстановления.
|
||||||
|
- Не выполнялись длительные нагрузочные испытания и измерения производительности с фиксированными hardware limits.
|
||||||
|
|
||||||
|
**Следующие шаги:**
|
||||||
|
|
||||||
|
1. Добавить передачу W3C trace context через RabbitMQ.
|
||||||
|
2. Инструментировать запросы к PostgreSQL и S3 отдельными spans.
|
||||||
|
3. Добавить метрики очереди и насыщения ресурсов.
|
||||||
|
4. Добавить alert rules с порогами, окнами и runbook.
|
||||||
|
5. Провести повторяемые нагрузочные испытания с фиксацией CPU/RAM, RPS и p50/p95/p99.
|
||||||
|
6. Протестировать частичную недоступность Loki, Tempo и Collector и измерить потери телеметрии.
|
||||||
|
|
||||||
|
**Вывод:** реализована работающая подсистема наблюдаемости на основе OpenTelemetry, Prometheus, Loki, Tempo и Grafana. Подтверждён экспорт трёх сигналов, обнаружение реальных трасс по связанным логам, работоспособность dashboard и восстановление системы после двух контролируемых отказов. Заявленный уровень — оценка 3.
|
||||||
+260
@@ -0,0 +1,260 @@
|
|||||||
|
|
||||||
|
services:
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v3.5.0
|
||||||
|
command:
|
||||||
|
- --config.file=/etc/prometheus/prometheus.yml
|
||||||
|
- --storage.tsdb.path=/prometheus
|
||||||
|
- --storage.tsdb.retention.time=7d
|
||||||
|
volumes:
|
||||||
|
- ./observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- prometheus_data:/prometheus
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:9090:9090"
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.5.5
|
||||||
|
command:
|
||||||
|
- -config.file=/etc/loki/local-config.yaml
|
||||||
|
volumes:
|
||||||
|
- ./observability/loki.yaml:/etc/loki/local-config.yaml:ro
|
||||||
|
- loki_data:/loki
|
||||||
|
user: "0"
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
tempo:
|
||||||
|
image: grafana/tempo:2.8.2
|
||||||
|
command:
|
||||||
|
- -config.file=/etc/tempo.yaml
|
||||||
|
volumes:
|
||||||
|
- ./observability/tempo.yaml:/etc/tempo.yaml:ro
|
||||||
|
- tempo_data:/var/tempo
|
||||||
|
user: "0"
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:12.1.1
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: admin
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
|
volumes:
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
- ./observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:3000:3000"
|
||||||
|
depends_on:
|
||||||
|
- prometheus
|
||||||
|
- loki
|
||||||
|
- tempo
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
otel-collector:
|
||||||
|
image: otel/opentelemetry-collector-contrib:0.136.0
|
||||||
|
command:
|
||||||
|
- --config=/etc/otelcol-contrib/config.yaml
|
||||||
|
volumes:
|
||||||
|
- ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:14317:4317"
|
||||||
|
- "127.0.0.1:14318:4318"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD
|
||||||
|
- /otelcol-contrib
|
||||||
|
- components
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
rabbitmq:
|
||||||
|
image: rabbitmq:4.1.8-management
|
||||||
|
hostname: rabbitmq
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?Set RABBITMQ_USER in .env}
|
||||||
|
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- rabbitmq_data:/var/lib/rabbitmq
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 15
|
||||||
|
start_period: 15s
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
db:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
minio:
|
||||||
|
image: rustfs/rustfs:1.0.3
|
||||||
|
command: ["/data"]
|
||||||
|
environment:
|
||||||
|
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env}
|
||||||
|
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env}
|
||||||
|
RUSTFS_ADDRESS: ":9000"
|
||||||
|
volumes:
|
||||||
|
- minio_data:/data
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
s3-init:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "src.s3_setup"]
|
||||||
|
environment: &s3_env
|
||||||
|
S3_ENDPOINT: http://minio:9000
|
||||||
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||||
|
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_started
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "src.migrate"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
api1:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_SERVICE_NAME: messenger-api
|
||||||
|
OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*"
|
||||||
|
RABBITMQ_HOST: rabbitmq
|
||||||
|
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||||
|
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||||
|
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||||
|
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
<<: *s3_env
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
stop_grace_period: 15s
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
api2:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_SERVICE_NAME: messenger-api
|
||||||
|
OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: "/health/.*"
|
||||||
|
RABBITMQ_HOST: rabbitmq
|
||||||
|
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||||
|
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||||
|
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||||
|
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
<<: *s3_env
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
stop_grace_period: 15s
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
|
||||||
|
dispatcher:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "src.dispatcher"]
|
||||||
|
environment:
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||||
|
RABBITMQ_HOST: rabbitmq
|
||||||
|
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||||
|
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||||
|
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||||
|
depends_on:
|
||||||
|
rabbitmq:
|
||||||
|
condition: service_healthy
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 15s
|
||||||
|
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "src.worker"]
|
||||||
|
environment:
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_SERVICE_NAME: messenger-worker
|
||||||
|
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger
|
||||||
|
S3_ENDPOINT: http://minio:9000
|
||||||
|
S3_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||||
|
S3_SECRET_KEY: ${S3_SECRET_KEY}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
RABBITMQ_HOST: rabbitmq
|
||||||
|
RABBITMQ_USER: ${RABBITMQ_USER}
|
||||||
|
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD}
|
||||||
|
RABBITMQ_QUEUE: ${RABBITMQ_QUEUE:-image_jobs}
|
||||||
|
IMAGE_MAX_ATTEMPTS: ${IMAGE_MAX_ATTEMPTS:-5}
|
||||||
|
depends_on:
|
||||||
|
rabbitmq:
|
||||||
|
condition: service_healthy
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 20s
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.29-alpine
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:8080:80"
|
||||||
|
volumes:
|
||||||
|
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
depends_on:
|
||||||
|
api1:
|
||||||
|
condition: service_healthy
|
||||||
|
api2:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
|
minio_data:
|
||||||
|
rabbitmq_data:
|
||||||
|
prometheus_data:
|
||||||
|
loki_data:
|
||||||
|
tempo_data:
|
||||||
|
grafana_data:
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||||
|
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 22 minutes ago Up 22 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 22 minutes ago Up 22 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 27 minutes ago Up 27 minutes (healthy) 5432/tcp
|
||||||
|
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 27 minutes ago Up 27 minutes 8080/tcp
|
||||||
|
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 9 minutes ago Up 15 seconds 127.0.0.1:3000->3000/tcp
|
||||||
|
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 44 minutes ago Up 44 minutes 3100/tcp
|
||||||
|
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 16 minutes ago Exited (0) 16 minutes ago
|
||||||
|
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 27 minutes ago Up 27 minutes 9000-9001/tcp
|
||||||
|
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 27 minutes ago Up 27 minutes 127.0.0.1:8080->80/tcp
|
||||||
|
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 41 minutes ago Up 41 minutes (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||||
|
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 44 minutes ago Up 44 minutes 127.0.0.1:9090->9090/tcp
|
||||||
|
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 27 minutes ago Up 27 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||||
|
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 16 minutes ago Exited (0) 16 minutes ago
|
||||||
|
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 44 minutes ago Up 44 minutes
|
||||||
|
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 16 minutes ago Up 16 minutes 8080/tcp
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 260 KiB |
@@ -0,0 +1,200 @@
|
|||||||
|
api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats HTTP/1.1" 201 Created
|
||||||
|
api2-1 | 2026-11-21 16:13:27,824 INFO event=http_request_completed trace_id=1889717945c3eddd806a8e37da7368d7 span_id=778a902e8dfd6c54 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:37470 - "POST /api/v1/chats/4b9eb2b4-44e4-4bb3-863b-4943b9c37c70/messages HTTP/1.1" 201 Created
|
||||||
|
api2-1 | INFO: 127.0.0.1:54924 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: Shutting down
|
||||||
|
api2-1 | INFO: Waiting for application shutdown.
|
||||||
|
api2-1 | INFO: Application shutdown complete.
|
||||||
|
api2-1 | INFO: Finished server process [1]
|
||||||
|
api2-1 | INFO: Started server process [1]
|
||||||
|
api2-1 | INFO: Waiting for application startup.
|
||||||
|
api2-1 | INFO: Application startup complete.
|
||||||
|
api2-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit)
|
||||||
|
api2-1 | INFO: 127.0.0.1:45734 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44306 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44322 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:37222 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:37234 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59306 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59346 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:46812 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:46838 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57764 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:60662 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | 2026-11-21 16:14:40,820 INFO event=http_request_completed trace_id=853e8a1ac0c8acd7182a241ca8c17990 span_id=05519f4c759960a2 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||||
|
api2-1 | 2026-11-21 16:14:41,864 INFO event=http_request_completed trace_id=3bb041b87253ffd0bc2b24360061cb7e span_id=3815f86eb8e0f940 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:33638 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:60712 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:41942 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:41974 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:50386 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:50402 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:39396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:39416 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:33268 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:33276 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57776 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57788 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:47430 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:47450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:42750 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:42762 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49882 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:40420 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59716 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59740 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49390 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:52874 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:52888 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:34432 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:50824 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:50842 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44010 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44022 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:42162 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:42192 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:52500 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:52514 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:43736 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:43756 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:34588 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:34616 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57026 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:57052 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:60044 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:60050 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59624 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:59646 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:41186 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | 2026-11-21 16:18:29,812 INFO event=http_request_completed trace_id=1756efab020974bc3f9555ce40099e7d span_id=30bfd1f2baa385c0 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/users HTTP/1.1" 201 Created
|
||||||
|
api2-1 | 2026-11-21 16:18:35,246 INFO event=http_request_completed trace_id=8fdf394cb3bda0228a70e17cddcaa622 span_id=3da8e0fea00100e0 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:56222 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||||
|
api2-1 | 2026-11-21 16:18:43,293 INFO event=http_request_completed trace_id=60c1a9ff611169aa6530d289c1cf7075 span_id=c5c5694d6f107123 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||||
|
api2-1 | 2026-11-21 16:18:47,408 INFO event=http_request_completed trace_id=e79c6299dbfb2c31e3d1bbd58c7a253e span_id=0f6ee54ea39b9385 HTTP request completed
|
||||||
|
api2-1 | INFO: 172.20.0.14:43672 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||||
|
api2-1 | INFO: 127.0.0.1:33068 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:33090 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38568 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38598 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44470 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:44482 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:39106 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:39116 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49932 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49962 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:55630 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:55634 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:54914 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: Shutting down
|
||||||
|
api1-1 | INFO: Waiting for application shutdown.
|
||||||
|
api1-1 | INFO: Application shutdown complete.
|
||||||
|
api1-1 | INFO: Finished server process [1]
|
||||||
|
api1-1 | INFO: Started server process [1]
|
||||||
|
api1-1 | INFO: Waiting for application startup.
|
||||||
|
api1-1 | INFO: Application startup complete.
|
||||||
|
api1-1 | INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit)
|
||||||
|
api1-1 | INFO: 172.20.0.14:39646 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | 2026-11-21 16:13:39,393 INFO event=http_request_completed trace_id=22e54b0e6d69077f18bfdb7c4e02661d span_id=45ddb3c5048edc9f HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:39646 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:45730 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44304 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44318 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:37220 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:37226 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59320 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59332 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:46808 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:46822 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57752 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57796 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:60666 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | 2026-11-21 16:14:41,842 INFO event=http_request_completed trace_id=c58c902e3393684ca4f780fd7f70e520 span_id=a07ee23af91c0bf7 HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07 HTTP/1.1" 200 OK
|
||||||
|
api1-1 | 2026-11-21 16:14:41,907 INFO event=http_request_completed trace_id=4bcc0c6f19ddf75a5fd7c5e364d9e32b span_id=99fff1d1ee4ace8a HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:40864 - "GET /api/v1/attachments/b98c6927-b6d5-4b9b-b0c1-7caf0d5a4c07/content?variant=thumbnail HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:60724 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:41948 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:41962 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:50380 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:50396 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38278 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38280 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38540 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:38560 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:32862 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:32890 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api2-1 | INFO: 127.0.0.1:49622 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:39382 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:39402 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:33254 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:33272 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57766 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57780 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:47440 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:47448 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:42758 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:42770 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49880 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:40414 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:40426 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59714 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59724 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49378 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49410 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:52872 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:52904 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:34444 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:34450 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:50832 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:50840 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44004 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44036 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:42176 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:42178 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:52498 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:52518 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:43746 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:43758 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:34594 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:34600 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57012 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:57038 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:60030 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:60054 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59630 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:59654 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:41172 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | 2026-11-21 16:18:29,864 INFO event=http_request_completed trace_id=951f6fcc6a32d838b2dc5fa1e2d7343a span_id=776bbf9a8e2b986a HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:48674 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created
|
||||||
|
api1-1 | 2026-11-21 16:18:37,223 INFO event=http_request_completed trace_id=1f8dc7db901d925f6b15fb2348b1a048 span_id=d2e85dfd82bc72b7 HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:33554 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||||
|
api1-1 | 2026-11-21 16:18:45,331 INFO event=http_request_completed trace_id=2afc8659a56936ef96018bc515cd6e61 span_id=88f8963e2289dcfc HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:50422 - "POST /api/v1/auth/sessions HTTP/1.1" 503 Service Unavailable
|
||||||
|
api1-1 | 2026-11-21 16:18:55,183 INFO event=http_request_completed trace_id=3d6852b9fb0e7047f085b8c239515e9c span_id=2802fa3fff95e680 HTTP request completed
|
||||||
|
api1-1 | INFO: 172.20.0.14:39552 - "POST /api/v1/auth/sessions HTTP/1.1" 201 Created
|
||||||
|
api1-1 | INFO: 127.0.0.1:33084 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:33098 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38580 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38592 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44456 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:44476 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:39102 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:39112 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49946 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49978 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:55624 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:55638 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38268 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38290 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38552 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:38574 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:32866 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:32876 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
|
api1-1 | INFO: 127.0.0.1:49606 - "GET /health/ready HTTP/1.1" 200 OK
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||||
|
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 6 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 6 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up About a minute (healthy) 5432/tcp
|
||||||
|
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp
|
||||||
|
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp
|
||||||
|
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp
|
||||||
|
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) 5 minutes ago
|
||||||
|
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp
|
||||||
|
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp
|
||||||
|
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||||
|
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp
|
||||||
|
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 7 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||||
|
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) 5 minutes ago
|
||||||
|
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days
|
||||||
|
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up 5 minutes 8080/tcp
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 247 KiB |
@@ -0,0 +1,80 @@
|
|||||||
|
db-1 | 2026-11-19 10:59:29.991 UTC [1] LOG: listening on IPv6 address "::", port 5432
|
||||||
|
db-1 | 2026-11-19 10:59:29.995 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
|
||||||
|
db-1 | 2026-11-19 10:59:30.000 UTC [57] LOG: database system was shut down at 2026-11-19 10:59:29 UTC
|
||||||
|
db-1 | 2026-11-19 10:59:30.004 UTC [1] LOG: database system is ready to accept connections
|
||||||
|
db-1 | 2026-11-19 11:01:46.203 UTC [451] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:01:46.203 UTC [451] DETAIL: Key (username)=(u_alice_f0123aa15331) already exists.
|
||||||
|
db-1 | 2026-11-19 11:01:46.203 UTC [451] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:04:31.001 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-19 11:04:48.288 UTC [55] LOG: checkpoint complete: wrote 175 buffers (1.1%); 0 WAL file(s) added, 0 removed, 0 recycled; write=17.233 s, sync=0.045 s, total=17.288 s; sync files=104, longest=0.002 s, average=0.001 s; distance=803 kB, estimate=803 kB; lsn=0/19E7700, redo lsn=0/19E76C8
|
||||||
|
db-1 | 2026-11-19 11:04:50.938 UTC [988] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:04:50.938 UTC [988] DETAIL: Key (username)=(u_alice_dff3cf8f9360) already exists.
|
||||||
|
db-1 | 2026-11-19 11:04:50.938 UTC [988] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:09:31.281 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-19 11:09:37.044 UTC [55] LOG: checkpoint complete: wrote 58 buffers (0.4%); 0 WAL file(s) added, 0 removed, 0 recycled; write=5.713 s, sync=0.021 s, total=5.763 s; sync files=43, longest=0.002 s, average=0.001 s; distance=14 kB, estimate=724 kB; lsn=0/19EB220, redo lsn=0/19EB1E8
|
||||||
|
db-1 | 2026-11-19 11:11:03.824 UTC [2110] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:11:03.824 UTC [2110] DETAIL: Key (username)=(u_alice_a072f61fcaf9) already exists.
|
||||||
|
db-1 | 2026-11-19 11:11:03.824 UTC [2110] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:11:24.481 UTC [2231] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:11:24.481 UTC [2231] DETAIL: Key (username)=(u_alice_ed51a8803fc1) already exists.
|
||||||
|
db-1 | 2026-11-19 11:11:24.481 UTC [2231] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:14:31.133 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-19 11:14:34.469 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.019 s, total=3.336 s; sync files=34, longest=0.003 s, average=0.001 s; distance=30 kB, estimate=655 kB; lsn=0/19F2DB0, redo lsn=0/19F2D78
|
||||||
|
db-1 | 2026-11-19 11:27:17.811 UTC [4991] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:27:17.811 UTC [4991] DETAIL: Key (username)=(u_alice_5c60fc1beaf6) already exists.
|
||||||
|
db-1 | 2026-11-19 11:27:17.811 UTC [4991] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:29:31.636 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-19 11:29:34.995 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.022 s, total=3.359 s; sync files=34, longest=0.003 s, average=0.001 s; distance=32 kB, estimate=592 kB; lsn=0/19FB020, redo lsn=0/19FAFE8
|
||||||
|
db-1 | 2026-11-19 11:31:47.459 UTC [5815] ERROR: duplicate key value violates unique constraint "users_username_key"
|
||||||
|
db-1 | 2026-11-19 11:31:47.459 UTC [5815] DETAIL: Key (username)=(u_alice_afb8c8b7058a) already exists.
|
||||||
|
db-1 | 2026-11-19 11:31:47.459 UTC [5815] STATEMENT:
|
||||||
|
db-1 | INSERT INTO users
|
||||||
|
db-1 | (id, username, display_name, password_hash)
|
||||||
|
db-1 | VALUES ($1, $2, $3, $4)
|
||||||
|
db-1 | RETURNING id, username, display_name, created_at
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-19 11:34:32.002 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-19 11:34:35.335 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.309 s, sync=0.016 s, total=3.334 s; sync files=34, longest=0.002 s, average=0.001 s; distance=38 kB, estimate=537 kB; lsn=0/1A04AD0, redo lsn=0/1A04A98
|
||||||
|
db-1 | 2026-11-21 16:15:11.638 UTC [55] LOG: checkpoint starting: time
|
||||||
|
db-1 | 2026-11-21 16:15:14.988 UTC [55] LOG: checkpoint complete: wrote 34 buffers (0.2%); 0 WAL file(s) added, 0 removed, 0 recycled; write=3.319 s, sync=0.016 s, total=3.351 s; sync files=34, longest=0.002 s, average=0.001 s; distance=40 kB, estimate=487 kB; lsn=0/1A0EB68, redo lsn=0/1A0EB30
|
||||||
|
db-1 | 2026-11-21 16:18:29.948 UTC [1] LOG: received fast shutdown request
|
||||||
|
db-1 | 2026-11-21 16:18:29.950 UTC [1] LOG: aborting any active transactions
|
||||||
|
db-1 | 2026-11-21 16:18:29.956 UTC [1] LOG: background worker "logical replication launcher" (PID 60) exited with exit code 1
|
||||||
|
db-1 | 2026-11-21 16:18:29.957 UTC [55] LOG: shutting down
|
||||||
|
db-1 | 2026-11-21 16:18:29.958 UTC [55] LOG: checkpoint starting: shutdown immediate
|
||||||
|
db-1 | 2026-11-21 16:18:29.972 UTC [55] LOG: checkpoint complete: wrote 8 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.002 s, sync=0.005 s, total=0.016 s; sync files=8, longest=0.002 s, average=0.001 s; distance=13 kB, estimate=440 kB; lsn=0/1A12070, redo lsn=0/1A12070
|
||||||
|
db-1 | 2026-11-21 16:18:29.979 UTC [1] LOG: database system is shut down
|
||||||
|
db-1 |
|
||||||
|
db-1 | PostgreSQL Database directory appears to contain a database; Skipping initialization
|
||||||
|
db-1 |
|
||||||
|
db-1 | 2026-11-21 16:18:49.717 UTC [1] LOG: starting PostgreSQL 16.14 on x86_64-pc-linux-musl, compiled by gcc (Alpine 15.2.0) 15.2.0, 64-bit
|
||||||
|
db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432
|
||||||
|
db-1 | 2026-11-21 16:18:49.718 UTC [1] LOG: listening on IPv6 address "::", port 5432
|
||||||
|
db-1 | 2026-11-21 16:18:49.722 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
|
||||||
|
db-1 | 2026-11-21 16:18:49.728 UTC [29] LOG: database system was shut down at 2026-11-21 16:18:29 UTC
|
||||||
|
db-1 | 2026-11-21 16:18:49.735 UTC [1] LOG: database system is ready to accept connections
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
Baseline: authentication HTTP 201
|
||||||
|
INCIDENT START: stopping PostgreSQL
|
||||||
|
Container mtusi2026lab6-db-1 Stopping
|
||||||
|
Container mtusi2026lab6-db-1 Stopped
|
||||||
|
Probe 1: status=TimeoutError, duration=5.005s
|
||||||
|
Probe 2: status=503, duration=0.046s
|
||||||
|
Probe 3: status=503, duration=4.071s
|
||||||
|
Probe 4: status=503, duration=0.038s
|
||||||
|
Probe 5: status=503, duration=0.076s
|
||||||
|
INCIDENT END: restoring PostgreSQL
|
||||||
|
Container mtusi2026lab6-db-1 Created
|
||||||
|
Container mtusi2026lab6-db-1 Starting
|
||||||
|
Container mtusi2026lab6-db-1 Started
|
||||||
|
Container mtusi2026lab6-db-1 Waiting
|
||||||
|
Container mtusi2026lab6-db-1 Healthy
|
||||||
|
Recovery: HTTP 201, 0.058s
|
||||||
|
Failed requests during incident: 5/5
|
||||||
|
PASS: database failure observed and recovered
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): Normal shutdown
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1d1c0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1d1c0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Closing connection (200): 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,594 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,595 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 53380), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,595 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||||
|
dispatcher-1 | 2026-11-19 11:27:18,595 INFO Published job 6d77b086-944f-447d-8156-81995b5c562b
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,858 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,859 INFO Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,859 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,861 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,861 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,861 INFO Created channel=1
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing connection (200): Normal shutdown
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,867 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1df10> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing connection (200): 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 51124), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,868 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||||
|
dispatcher-1 | 2026-11-19 11:31:47,869 INFO Published job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,871 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,875 INFO Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,878 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,893 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,894 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,895 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,899 INFO Created channel=1
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing connection (200): Normal shutdown
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,918 INFO Closing channel (200): 'Normal shutdown' on <Channel number=1 OPEN conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,920 INFO Received <Channel.CloseOk> on <Channel number=1 CLOSING conn=<SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x70b37ef1e660> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,920 INFO Closing connection (200): 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Aborting transport connection: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO _AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=None; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Deactivating transport: state=1; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=None; pending-error=ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Stack terminated due to ConnectionClosedByClient: (200) 'Normal shutdown'
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,921 INFO Closing transport socket and unlinking: state=3; <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.20.0.12', 57084), raddr=('172.20.0.7', 5672)>
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,922 INFO User-initiated close: result=BlockingConnection__OnClosedArgs(connection=<SelectConnection CLOSED transport=None params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>, error=ConnectionClosedByClient: (200) 'Normal shutdown')
|
||||||
|
dispatcher-1 | 2026-11-21 16:13:27,923 INFO Published job e687ea16-ef44-416e-a96f-15dfac07230e
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||||
|
mtusi2026lab6-api1-1 mtusi2026lab6-api1 "uvicorn src.main:ap…" api1 2 days ago Up 2 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-api2-1 mtusi2026lab6-api2 "uvicorn src.main:ap…" api2 2 days ago Up 2 minutes (healthy) 8080/tcp
|
||||||
|
mtusi2026lab6-db-1 postgres:16-alpine "docker-entrypoint.s…" db 2 days ago Up 2 days (healthy) 5432/tcp
|
||||||
|
mtusi2026lab6-dispatcher-1 mtusi2026lab6-dispatcher "python -m src.dispa…" dispatcher 2 days ago Up 2 days 8080/tcp
|
||||||
|
mtusi2026lab6-grafana-1 grafana/grafana:12.1.1 "/run.sh" grafana 2 days ago Up 2 days 127.0.0.1:3000->3000/tcp
|
||||||
|
mtusi2026lab6-loki-1 grafana/loki:3.5.5 "/usr/bin/loki -conf…" loki 2 days ago Up 2 days 3100/tcp
|
||||||
|
mtusi2026lab6-migrate-1 mtusi2026lab6-migrate "python -m src.migra…" migrate 2 days ago Exited (0) About a minute ago
|
||||||
|
mtusi2026lab6-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 2 days ago Up 2 days 9000-9001/tcp
|
||||||
|
mtusi2026lab6-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 2 days ago Up 2 days 127.0.0.1:8080->80/tcp
|
||||||
|
mtusi2026lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.136.0 "/otelcol-contrib --…" otel-collector 2 days ago Up 2 days (healthy) 55679/tcp, 127.0.0.1:14317->4317/tcp, 127.0.0.1:14318->4318/tcp
|
||||||
|
mtusi2026lab6-prometheus-1 prom/prometheus:v3.5.0 "/bin/prometheus --c…" prometheus 2 days ago Up 2 days 127.0.0.1:9090->9090/tcp
|
||||||
|
mtusi2026lab6-rabbitmq-1 rabbitmq:4.1.8-management "docker-entrypoint.s…" rabbitmq 2 days ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||||
|
mtusi2026lab6-s3-init-1 mtusi2026lab6-s3-init "python -m src.s3_se…" s3-init 2 days ago Exited (0) About a minute ago
|
||||||
|
mtusi2026lab6-tempo-1 grafana/tempo:2.8.2 "/tempo -config.file…" tempo 2 days ago Up 2 days
|
||||||
|
mtusi2026lab6-worker-1 mtusi2026lab6-worker "python -m src.worker" worker 2 days ago Up About a minute 8080/tcp
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 226 KiB |
@@ -0,0 +1,26 @@
|
|||||||
|
worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||||
|
worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 50892), raddr=('172.20.0.7', 5672)>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1
|
||||||
|
worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs
|
||||||
|
worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled
|
||||||
|
worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled
|
||||||
|
worker-1 | 2026-11-19 11:27:18,636 INFO Job 6d77b086-944f-447d-8156-81995b5c562b ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:27:18,643 INFO event=image_job_handled trace_id=6d39def6a9347b9766c7d9820480cbea span_id=91110edf3e0e5f53 Image job handled
|
||||||
|
worker-1 | 2026-11-19 11:31:47,910 INFO Job c9c31ec9-a3cf-4f9c-8f5c-387e1f3c81e7 ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:31:47,918 INFO event=image_job_handled trace_id=5a9611a8564da15253517786b3b87c35 span_id=692dee1a1ba56089 Image job handled
|
||||||
|
worker-1 | 2026-11-21 16:14:41,241 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||||
|
worker-1 | 2026-11-21 16:14:41,242 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 37148), raddr=('172.20.0.7', 5672)>
|
||||||
|
worker-1 | 2026-11-21 16:14:41,242 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||||
|
worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-21 16:14:41,249 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-21 16:14:41,249 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x77dfab6da4b0> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-21 16:14:41,250 INFO Created channel=1
|
||||||
|
worker-1 | 2026-11-21 16:14:41,253 INFO Worker listening on image_jobs
|
||||||
|
worker-1 | 2026-11-21 16:14:41,323 INFO Job e687ea16-ef44-416e-a96f-15dfac07230e ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-21 16:14:41,331 INFO event=image_job_handled trace_id=b768e258011bde0b8084a6665f1067d1 span_id=bee607ec28cf87ab Image job handled
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 241 KiB |
@@ -0,0 +1,60 @@
|
|||||||
|
Stopping worker...
|
||||||
|
Container mtusi2026lab6-worker-1 Stopping
|
||||||
|
Container mtusi2026lab6-worker-1 Stopped
|
||||||
|
Upload: 202 queued
|
||||||
|
Message created while worker is stopped
|
||||||
|
Restarting RabbitMQ and API...
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Restarting
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Started
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Running
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||||
|
Container mtusi2026lab6-api2-1 Restarting
|
||||||
|
Container mtusi2026lab6-api1-1 Restarting
|
||||||
|
Container mtusi2026lab6-api1-1 Started
|
||||||
|
Container mtusi2026lab6-api2-1 Started
|
||||||
|
Queued attachment survived restart
|
||||||
|
INCIDENT: worker stopped for 60 seconds
|
||||||
|
Starting worker...
|
||||||
|
Container mtusi2026lab6-db-1 Running
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Running
|
||||||
|
Container mtusi2026lab6-minio-1 Running
|
||||||
|
Container mtusi2026lab6-migrate-1 Created
|
||||||
|
Container mtusi2026lab6-s3-init-1 Created
|
||||||
|
Container mtusi2026lab6-worker-1 Created
|
||||||
|
Container mtusi2026lab6-s3-init-1 Starting
|
||||||
|
Container mtusi2026lab6-db-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Started
|
||||||
|
Container mtusi2026lab6-db-1 Healthy
|
||||||
|
Container mtusi2026lab6-migrate-1 Starting
|
||||||
|
Container mtusi2026lab6-migrate-1 Started
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||||
|
Container mtusi2026lab6-worker-1 Starting
|
||||||
|
Container mtusi2026lab6-worker-1 Started
|
||||||
|
Attachment: ready
|
||||||
|
Original: SHA-256 verified
|
||||||
|
Thumbnail: JPEG 256x160 verified
|
||||||
|
PASS: durable worker recovery
|
||||||
|
Container mtusi2026lab6-db-1 Running
|
||||||
|
Container mtusi2026lab6-migrate-1 Created
|
||||||
|
Container mtusi2026lab6-minio-1 Running
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Running
|
||||||
|
Container mtusi2026lab6-s3-init-1 Created
|
||||||
|
Container mtusi2026lab6-worker-1 Running
|
||||||
|
Container mtusi2026lab6-db-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Starting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Started
|
||||||
|
Container mtusi2026lab6-db-1 Healthy
|
||||||
|
Container mtusi2026lab6-migrate-1 Starting
|
||||||
|
Container mtusi2026lab6-migrate-1 Started
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Waiting
|
||||||
|
Container mtusi2026lab6-migrate-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Waiting
|
||||||
|
Container mtusi2026lab6-s3-init-1 Exited
|
||||||
|
Container mtusi2026lab6-rabbitmq-1 Healthy
|
||||||
|
Container mtusi2026lab6-migrate-1 Exited
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"status": "success",
|
||||||
|
"data": [
|
||||||
|
"messenger_http_server_active_requests",
|
||||||
|
"messenger_http_server_duration_milliseconds_bucket",
|
||||||
|
"messenger_http_server_duration_milliseconds_count",
|
||||||
|
"messenger_http_server_duration_milliseconds_sum",
|
||||||
|
"messenger_http_server_request_size_bytes_bucket",
|
||||||
|
"messenger_http_server_request_size_bytes_count",
|
||||||
|
"messenger_http_server_request_size_bytes_sum",
|
||||||
|
"messenger_http_server_response_size_bytes_bucket",
|
||||||
|
"messenger_http_server_response_size_bytes_count",
|
||||||
|
"messenger_http_server_response_size_bytes_sum",
|
||||||
|
"messenger_messenger_http_requests_total",
|
||||||
|
"messenger_messenger_worker_job_duration_seconds_bucket",
|
||||||
|
"messenger_messenger_worker_job_duration_seconds_count",
|
||||||
|
"messenger_messenger_worker_job_duration_seconds_sum",
|
||||||
|
"messenger_messenger_worker_jobs_processed_total",
|
||||||
|
"messenger_target_info",
|
||||||
|
"scrape_duration_seconds",
|
||||||
|
"scrape_samples_post_metric_relabeling",
|
||||||
|
"scrape_samples_scraped",
|
||||||
|
"scrape_series_added",
|
||||||
|
"up"
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
Service: messenger-api
|
||||||
|
Event: HTTP request completed
|
||||||
|
Trace ID: c3ea28d96d7c536e222d8e4c7cba2727
|
||||||
|
Span ID: 3be2550ec51156e2
|
||||||
|
Tempo spans: 4
|
||||||
|
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||||
|
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||||
|
Span: POST /api/v1/auth/sessions http send kind: SPAN_KIND_INTERNAL
|
||||||
|
Span: POST /api/v1/auth/sessions kind: SPAN_KIND_SERVER
|
||||||
|
PASS: Loki log linked to Tempo trace
|
||||||
|
|
||||||
|
Service: messenger-worker
|
||||||
|
Event: Image job handled
|
||||||
|
Trace ID: af16dbcb7b7ca25b06d39902dc48afd8
|
||||||
|
Span ID: 644d3cda73387e3f
|
||||||
|
Tempo spans: 1
|
||||||
|
Span: image_job.process kind: SPAN_KIND_CONSUMER
|
||||||
|
PASS: Loki log linked to Tempo trace
|
||||||
|
|
||||||
|
PASS: API and worker trace correlation
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
worker-1 | 2026-11-19 11:10:44,315 INFO Pika version 1.3.2 connecting to ('172.20.0.7', 5672)
|
||||||
|
worker-1 | 2026-11-19 11:10:44,315 INFO Socket connected: <socket.socket fd=6, family=2, type=1, proto=6, laddr=('172.20.0.13', 50892), raddr=('172.20.0.7', 5672)>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,316 INFO Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0x71816d825d00> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||||
|
worker-1 | 2026-11-19 11:10:44,318 INFO Created channel=1
|
||||||
|
worker-1 | 2026-11-19 11:10:44,342 INFO Worker listening on image_jobs
|
||||||
|
worker-1 | 2026-11-19 11:11:04,434 INFO Job a95f84e1-cd98-4fcd-ae9c-a5782b252fe3 ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:11:04,441 INFO event=image_job_handled trace_id=af16dbcb7b7ca25b06d39902dc48afd8 span_id=644d3cda73387e3f Image job handled
|
||||||
|
worker-1 | 2026-11-19 11:11:25,616 INFO Job 1d011797-2efd-4bf7-88b9-c2c0e455119c ready, thumbnail 256x160
|
||||||
|
worker-1 | 2026-11-19 11:11:25,623 INFO event=image_job_handled trace_id=53f30d09a4f2c578521c8bf075e35b88 span_id=93d63eec0255aa53 Image job handled
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
username VARCHAR(32) NOT NULL UNIQUE,
|
||||||
|
display_name VARCHAR(100) NOT NULL,
|
||||||
|
password_hash TEXT NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
|
||||||
|
CONSTRAINT username_format
|
||||||
|
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
CONSTRAINT display_name_length
|
||||||
|
CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
CONSTRAINT display_name_not_blank
|
||||||
|
CHECK (length(btrim(display_name)) > 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS chats (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
title VARCHAR(100) NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
|
||||||
|
CONSTRAINT title_not_blank
|
||||||
|
CHECK (length(btrim(title)) > 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS chat_members (
|
||||||
|
chat_id UUID NOT NULL
|
||||||
|
REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
user_id UUID NOT NULL
|
||||||
|
REFERENCES users(id),
|
||||||
|
position INTEGER NOT NULL,
|
||||||
|
|
||||||
|
PRIMARY KEY (chat_id, user_id),
|
||||||
|
UNIQUE (chat_id, position)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS messages (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
chat_id UUID NOT NULL
|
||||||
|
REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
sender_id UUID NOT NULL
|
||||||
|
REFERENCES users(id),
|
||||||
|
text TEXT NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
|
||||||
|
CONSTRAINT message_length
|
||||||
|
CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
CONSTRAINT message_not_blank
|
||||||
|
CHECK (length(btrim(text)) > 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS sessions (
|
||||||
|
token_hash CHAR(64) PRIMARY KEY,
|
||||||
|
user_id UUID NOT NULL
|
||||||
|
REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
expires_at TIMESTAMPTZ NOT NULL,
|
||||||
|
revoked BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_chat_members_user
|
||||||
|
ON chat_members(user_id, chat_id);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_chats_order
|
||||||
|
ON chats(created_at, id);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_history
|
||||||
|
ON messages(chat_id, created_at, id);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_sessions_user
|
||||||
|
ON sessions(user_id);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_sessions_expiry
|
||||||
|
ON sessions(expires_at);
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||||
|
user_id UUID NOT NULL REFERENCES users(id),
|
||||||
|
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||||
|
idempotency_key VARCHAR(128) NOT NULL,
|
||||||
|
request_hash CHAR(64) NOT NULL,
|
||||||
|
message_id UUID NOT NULL REFERENCES messages(id),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
|
||||||
|
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_idempotency_created
|
||||||
|
ON message_idempotency(created_at);
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS attachments (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||||
|
uploader_id UUID NOT NULL REFERENCES users(id),
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
content_type TEXT NOT NULL,
|
||||||
|
size_bytes BIGINT NOT NULL,
|
||||||
|
sha256 CHAR(64) NOT NULL,
|
||||||
|
object_key TEXT NOT NULL UNIQUE,
|
||||||
|
width INTEGER NOT NULL,
|
||||||
|
height INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'ready',
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
|
||||||
|
CONSTRAINT attachment_size_valid
|
||||||
|
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||||
|
|
||||||
|
CONSTRAINT attachment_dimensions_valid
|
||||||
|
CHECK (width > 0 AND height > 0),
|
||||||
|
|
||||||
|
CONSTRAINT attachment_status_valid
|
||||||
|
CHECK (status IN ('ready', 'failed'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_attachments_chat
|
||||||
|
ON attachments(chat_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||||
|
message_id UUID NOT NULL REFERENCES messages(id),
|
||||||
|
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||||
|
|
||||||
|
PRIMARY KEY (message_id, attachment_id)
|
||||||
|
);
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
|
||||||
|
-- Lab4: messages with attachments may have empty text.
|
||||||
|
-- The API checks that a message contains text or attachments.
|
||||||
|
|
||||||
|
ALTER TABLE messages
|
||||||
|
DROP CONSTRAINT IF EXISTS message_length;
|
||||||
|
|
||||||
|
ALTER TABLE messages
|
||||||
|
DROP CONSTRAINT IF EXISTS message_not_blank;
|
||||||
|
|
||||||
|
ALTER TABLE messages
|
||||||
|
ADD CONSTRAINT message_length
|
||||||
|
CHECK (char_length(text) <= 4000);
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
-- Allow asynchronous attachment states.
|
||||||
|
ALTER TABLE attachments
|
||||||
|
DROP CONSTRAINT IF EXISTS attachment_status_valid;
|
||||||
|
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD CONSTRAINT attachment_status_valid
|
||||||
|
CHECK (
|
||||||
|
status IN (
|
||||||
|
'queued',
|
||||||
|
'processing',
|
||||||
|
'ready',
|
||||||
|
'failed'
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Staging object stored in the private S3 bucket.
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD COLUMN IF NOT EXISTS staging_key TEXT;
|
||||||
|
|
||||||
|
-- Machine-readable final error.
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD COLUMN IF NOT EXISTS error_code TEXT;
|
||||||
|
|
||||||
|
-- Last metadata update timestamp.
|
||||||
|
ALTER TABLE attachments
|
||||||
|
ADD COLUMN IF NOT EXISTS updated_at
|
||||||
|
TIMESTAMPTZ NOT NULL DEFAULT NOW();
|
||||||
|
|
||||||
|
-- Worker jobs and durable publication intent.
|
||||||
|
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
attachment_id UUID NOT NULL UNIQUE
|
||||||
|
REFERENCES attachments(id),
|
||||||
|
|
||||||
|
status TEXT NOT NULL DEFAULT 'queued',
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||||
|
|
||||||
|
published_at TIMESTAMPTZ,
|
||||||
|
started_at TIMESTAMPTZ,
|
||||||
|
completed_at TIMESTAMPTZ,
|
||||||
|
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
|
||||||
|
CONSTRAINT attachment_job_status_valid
|
||||||
|
CHECK (
|
||||||
|
status IN (
|
||||||
|
'queued',
|
||||||
|
'processing',
|
||||||
|
'ready',
|
||||||
|
'failed'
|
||||||
|
)
|
||||||
|
),
|
||||||
|
|
||||||
|
CONSTRAINT attachment_job_attempts_valid
|
||||||
|
CHECK (
|
||||||
|
attempts >= 0
|
||||||
|
AND max_attempts BETWEEN 1 AND 20
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending
|
||||||
|
ON attachment_jobs(status, next_attempt_at);
|
||||||
|
|
||||||
|
-- Image variants created by the worker.
|
||||||
|
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||||
|
attachment_id UUID NOT NULL
|
||||||
|
REFERENCES attachments(id),
|
||||||
|
|
||||||
|
variant_name TEXT NOT NULL,
|
||||||
|
object_key TEXT NOT NULL,
|
||||||
|
content_type TEXT NOT NULL,
|
||||||
|
size_bytes BIGINT NOT NULL,
|
||||||
|
sha256 CHAR(64) NOT NULL,
|
||||||
|
width INTEGER NOT NULL,
|
||||||
|
height INTEGER NOT NULL,
|
||||||
|
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
|
||||||
|
PRIMARY KEY (attachment_id, variant_name),
|
||||||
|
|
||||||
|
CONSTRAINT variant_name_valid
|
||||||
|
CHECK (variant_name IN ('thumbnail')),
|
||||||
|
|
||||||
|
CONSTRAINT variant_size_valid
|
||||||
|
CHECK (size_bytes > 0),
|
||||||
|
|
||||||
|
CONSTRAINT variant_dimensions_valid
|
||||||
|
CHECK (width > 0 AND height > 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachment_variants_key
|
||||||
|
ON attachment_variants(object_key);
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
events {
|
||||||
|
worker_connections 1024;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||||
|
resolver_timeout 2s;
|
||||||
|
|
||||||
|
upstream messenger_api {
|
||||||
|
zone messenger_api 64k;
|
||||||
|
|
||||||
|
server api1:8080 resolve max_fails=1 fail_timeout=2s;
|
||||||
|
server api2:8080 resolve max_fails=1 fail_timeout=2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
client_max_body_size 11m;
|
||||||
|
listen 80;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://messenger_api;
|
||||||
|
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
|
||||||
|
proxy_connect_timeout 2s;
|
||||||
|
proxy_read_timeout 30s;
|
||||||
|
proxy_send_timeout 30s;
|
||||||
|
|
||||||
|
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||||
|
proxy_next_upstream_tries 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
{
|
||||||
|
"uid": "messenger-lab6",
|
||||||
|
"title": "Messenger Lab6 - Observability",
|
||||||
|
"tags": ["lab6", "messenger", "opentelemetry"],
|
||||||
|
"timezone": "browser",
|
||||||
|
"schemaVersion": 41,
|
||||||
|
"version": 1,
|
||||||
|
"refresh": "5s",
|
||||||
|
"time": {
|
||||||
|
"from": "now-30m",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"annotations": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "API - Requests per second",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0, "y": 0, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum(rate(messenger_messenger_http_requests_total[2m]))",
|
||||||
|
"legendFormat": "All business requests"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "reqps",
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "API - HTTP 5xx error ratio",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12, "y": 0, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "(100 * sum(rate(messenger_messenger_http_requests_total{http_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(messenger_messenger_http_requests_total[2m])), 0.000001)) or vector(0)",
|
||||||
|
"legendFormat": "5xx / all business requests"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "API - HTTP p95 duration",
|
||||||
|
"description": "OTel FastAPI duration histogram is in milliseconds; result converted to seconds.",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0, "y": 8, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_server_duration_milliseconds_bucket[5m]))) / 1000",
|
||||||
|
"legendFormat": "p95 HTTP duration"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s",
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "Worker - Jobs by outcome",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12, "y": 8, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum by (outcome) (increase(messenger_messenger_worker_jobs_processed_total[5m]))",
|
||||||
|
"legendFormat": "{{outcome}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Worker - Processing p95 duration",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0, "y": 16, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_messenger_worker_job_duration_seconds_bucket[5m])))",
|
||||||
|
"legendFormat": "p95 job duration"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s",
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Worker - Completed job attempts",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12, "y": 16, "w": 12, "h": 8
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum(messenger_messenger_worker_jobs_processed_total)",
|
||||||
|
"legendFormat": "Processing attempts"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {
|
||||||
|
"values": false,
|
||||||
|
"calcs": ["lastNotNull"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"title": "API - Structured logs",
|
||||||
|
"type": "logs",
|
||||||
|
"datasource": {
|
||||||
|
"type": "loki",
|
||||||
|
"uid": "loki"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0, "y": 24, "w": 24, "h": 9
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "{service_name=\"messenger-api\"}",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": true,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"sortOrder": "Descending"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"title": "Worker - Structured logs",
|
||||||
|
"type": "logs",
|
||||||
|
"datasource": {
|
||||||
|
"type": "loki",
|
||||||
|
"uid": "loki"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0, "y": 33, "w": 24, "h": 9
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "{service_name=\"messenger-worker\"}",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": true,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"sortOrder": "Descending"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: Messenger Lab6
|
||||||
|
orgId: 1
|
||||||
|
folder: Messenger Lab6
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
editable: false
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
uid: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
derivedFields:
|
||||||
|
- name: TraceID
|
||||||
|
matcherRegex: 'trace_id=([a-f0-9]{32})'
|
||||||
|
datasourceUid: tempo
|
||||||
|
url: '$${__value.raw}'
|
||||||
|
|
||||||
|
- name: Tempo
|
||||||
|
uid: tempo
|
||||||
|
type: tempo
|
||||||
|
access: proxy
|
||||||
|
url: http://tempo:3200
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
filterByTraceID: true
|
||||||
|
filterBySpanID: false
|
||||||
|
nodeGraph:
|
||||||
|
enabled: true
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
|
||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: "2024-01-01"
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
storage_config:
|
||||||
|
filesystem:
|
||||||
|
directory: /loki/chunks
|
||||||
|
|
||||||
|
tsdb_shipper:
|
||||||
|
active_index_directory: /loki/index
|
||||||
|
cache_location: /loki/index_cache
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
allow_structured_metadata: true
|
||||||
|
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
processors:
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 256
|
||||||
|
spike_limit_mib: 64
|
||||||
|
|
||||||
|
batch:
|
||||||
|
timeout: 5s
|
||||||
|
send_batch_size: 512
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
prometheus:
|
||||||
|
endpoint: 0.0.0.0:8889
|
||||||
|
namespace: messenger
|
||||||
|
resource_to_telemetry_conversion:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
otlp/tempo:
|
||||||
|
endpoint: tempo:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
queue_size: 1000
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
otlphttp/loki:
|
||||||
|
endpoint: http://loki:3100/otlp
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
queue_size: 1000
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: 0.0.0.0:13133
|
||||||
|
|
||||||
|
service:
|
||||||
|
extensions:
|
||||||
|
- health_check
|
||||||
|
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlp/tempo]
|
||||||
|
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [prometheus]
|
||||||
|
|
||||||
|
logs:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/loki]
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 5s
|
||||||
|
evaluation_interval: 5s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: otel-collector
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- otel-collector:8889
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3200
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
ingester:
|
||||||
|
trace_idle_period: 10s
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
compaction:
|
||||||
|
block_retention: 48h
|
||||||
|
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
|
local:
|
||||||
|
path: /var/tempo/traces
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
fastapi==0.115.12
|
||||||
|
uvicorn==0.34.2
|
||||||
|
psycopg[binary]==3.2.6
|
||||||
|
argon2-cffi==23.1.0
|
||||||
|
boto3==1.40.70
|
||||||
|
Pillow==11.2.1
|
||||||
|
pika==1.3.2
|
||||||
|
opentelemetry-api==1.38.0
|
||||||
|
opentelemetry-sdk==1.38.0
|
||||||
|
opentelemetry-exporter-otlp-proto-http==1.38.0
|
||||||
|
opentelemetry-instrumentation-fastapi==0.59b0
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
echo "Stopping API and image processing..."
|
||||||
|
docker compose stop api1 api2 worker dispatcher
|
||||||
|
|
||||||
|
echo "Restarting persistent storage and broker..."
|
||||||
|
docker compose restart db minio rabbitmq
|
||||||
|
|
||||||
|
echo "Waiting for PostgreSQL and RabbitMQ..."
|
||||||
|
docker compose up -d --wait db rabbitmq
|
||||||
|
|
||||||
|
echo "Checking S3 bucket..."
|
||||||
|
docker compose run --rm --no-deps s3-init
|
||||||
|
|
||||||
|
echo "Starting dispatcher and worker..."
|
||||||
|
docker compose up -d dispatcher worker
|
||||||
|
|
||||||
|
echo "Starting API replicas..."
|
||||||
|
docker compose up -d --wait api1 api2
|
||||||
|
|
||||||
|
echo "Restart complete. Persistent volumes preserved."
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then
|
||||||
|
echo "TARGET_INSTANCE_ID is not set" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for service in api1 api2; do
|
||||||
|
instance_id=$(
|
||||||
|
docker compose exec -T "$service" python -c '
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
response = urllib.request.urlopen(
|
||||||
|
"http://127.0.0.1:8080/health/live",
|
||||||
|
timeout=2
|
||||||
|
)
|
||||||
|
|
||||||
|
print(response.headers.get("X-Instance-Id", ""))
|
||||||
|
' 2>/dev/null
|
||||||
|
) || continue
|
||||||
|
|
||||||
|
if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then
|
||||||
|
echo "Stopping $service"
|
||||||
|
docker compose stop -t 2 "$service"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Target API instance not found" >&2
|
||||||
|
exit 1
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
BASE_URL = "http://127.0.0.1:8080"
|
||||||
|
|
||||||
|
|
||||||
|
def docker(*args):
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "compose", *args],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def request(method, path, body=None, headers=None):
|
||||||
|
request_headers = dict(headers or {})
|
||||||
|
|
||||||
|
if isinstance(body, dict):
|
||||||
|
body = json.dumps(body).encode("utf-8")
|
||||||
|
request_headers["Content-Type"] = "application/json"
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASE_URL + path,
|
||||||
|
data=body,
|
||||||
|
headers=request_headers,
|
||||||
|
method=method,
|
||||||
|
)
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=5) as response:
|
||||||
|
response.read()
|
||||||
|
return response.status, time.monotonic() - started
|
||||||
|
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
exc.read()
|
||||||
|
return exc.code, time.monotonic() - started
|
||||||
|
|
||||||
|
except (urllib.error.URLError, TimeoutError) as exc:
|
||||||
|
return type(exc).__name__, time.monotonic() - started
|
||||||
|
|
||||||
|
|
||||||
|
def login_header(username, password):
|
||||||
|
encoded = base64.b64encode(
|
||||||
|
f"{username}:{password}".encode("utf-8")
|
||||||
|
).decode("ascii")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"Authorization": f"Basic {encoded}"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_recovery(headers):
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
status, duration = request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
body=b"",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
if status == 201:
|
||||||
|
print(
|
||||||
|
f"Recovery: HTTP 201, {duration:.3f}s",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
raise RuntimeError(
|
||||||
|
"Authentication did not recover after DB restart"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
suffix = uuid.uuid4().hex[:12]
|
||||||
|
|
||||||
|
username = "lab6_db_" + suffix
|
||||||
|
password = "IncidentTest_" + suffix
|
||||||
|
|
||||||
|
status, duration = request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/users",
|
||||||
|
body={
|
||||||
|
"username": username,
|
||||||
|
"display_name": "DB incident test",
|
||||||
|
"password": password,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if status != 201:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Cannot create test user: HTTP {status}"
|
||||||
|
)
|
||||||
|
|
||||||
|
headers = login_header(username, password)
|
||||||
|
|
||||||
|
status, _ = request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
body=b"",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
if status != 201:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Baseline login failed: HTTP {status}"
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Baseline: authentication HTTP 201", flush=True)
|
||||||
|
|
||||||
|
observed_failures = 0
|
||||||
|
|
||||||
|
print("INCIDENT START: stopping PostgreSQL", flush=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
docker("stop", "db")
|
||||||
|
|
||||||
|
for attempt in range(1, 6):
|
||||||
|
status, duration = request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
body=b"",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"Probe {attempt}: status={status}, "
|
||||||
|
f"duration={duration:.3f}s",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if status != 201:
|
||||||
|
observed_failures += 1
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
print(
|
||||||
|
"INCIDENT END: restoring PostgreSQL",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
docker("up", "-d", "--wait", "db")
|
||||||
|
|
||||||
|
wait_for_recovery(headers)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"Failed requests during incident: "
|
||||||
|
f"{observed_failures}/5",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if observed_failures == 0:
|
||||||
|
raise RuntimeError(
|
||||||
|
"The database incident did not affect requests"
|
||||||
|
)
|
||||||
|
|
||||||
|
print(
|
||||||
|
"PASS: database failure observed and recovered",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
sql() {
|
||||||
|
docker compose exec -T db \
|
||||||
|
psql -U messenger -d messenger -At -c "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Finding completed image job..."
|
||||||
|
|
||||||
|
JOB_ID=$(sql "
|
||||||
|
SELECT j.id
|
||||||
|
FROM attachment_jobs j
|
||||||
|
JOIN attachment_variants v
|
||||||
|
ON v.attachment_id = j.attachment_id
|
||||||
|
WHERE j.status = 'ready'
|
||||||
|
AND v.variant_name = 'thumbnail'
|
||||||
|
ORDER BY j.created_at DESC
|
||||||
|
LIMIT 1;
|
||||||
|
")
|
||||||
|
|
||||||
|
if [[ -z "$JOB_ID" ]]; then
|
||||||
|
echo "FAIL: no completed image job found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Job: $JOB_ID"
|
||||||
|
|
||||||
|
STATE_QUERY="
|
||||||
|
SELECT
|
||||||
|
j.status || '|' ||
|
||||||
|
j.attempts || '|' ||
|
||||||
|
v.sha256 || '|' ||
|
||||||
|
(
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM attachment_variants
|
||||||
|
WHERE attachment_id = j.attachment_id
|
||||||
|
)
|
||||||
|
FROM attachment_jobs j
|
||||||
|
JOIN attachment_variants v
|
||||||
|
ON v.attachment_id = j.attachment_id
|
||||||
|
WHERE j.id = '$JOB_ID'
|
||||||
|
AND v.variant_name = 'thumbnail';
|
||||||
|
"
|
||||||
|
|
||||||
|
BEFORE=$(sql "$STATE_QUERY")
|
||||||
|
|
||||||
|
echo "Before: $BEFORE"
|
||||||
|
echo "Publishing the same job twice..."
|
||||||
|
|
||||||
|
docker compose run --rm --no-deps api1 \
|
||||||
|
python -c '
|
||||||
|
import sys
|
||||||
|
from src.queue import publish_job
|
||||||
|
|
||||||
|
job_id = sys.argv[1]
|
||||||
|
publish_job(job_id)
|
||||||
|
publish_job(job_id)
|
||||||
|
print("Two duplicate messages published")
|
||||||
|
' "$JOB_ID"
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
AFTER=$(sql "$STATE_QUERY")
|
||||||
|
|
||||||
|
echo "After: $AFTER"
|
||||||
|
|
||||||
|
if [[ "$BEFORE" != "$AFTER" ]]; then
|
||||||
|
echo "FAIL: completed job changed after redelivery"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "PASS: duplicate delivery did not change completed job"
|
||||||
|
echo "PASS: thumbnail checksum and variant count preserved"
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
LOKI = "http://loki:3100"
|
||||||
|
TEMPO = "http://tempo:3200"
|
||||||
|
|
||||||
|
TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$")
|
||||||
|
SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$")
|
||||||
|
|
||||||
|
|
||||||
|
def get_json(url):
|
||||||
|
with urllib.request.urlopen(url, timeout=15) as response:
|
||||||
|
return json.load(response)
|
||||||
|
|
||||||
|
|
||||||
|
def find_log(service, event):
|
||||||
|
params = urllib.parse.urlencode({
|
||||||
|
"query": f'{{service_name="{service}"}}',
|
||||||
|
"start": str(time.time_ns() - 6 * 3600 * 10**9),
|
||||||
|
"end": str(time.time_ns()),
|
||||||
|
"limit": 500,
|
||||||
|
"direction": "BACKWARD",
|
||||||
|
})
|
||||||
|
|
||||||
|
payload = get_json(
|
||||||
|
f"{LOKI}/loki/api/v1/query_range?{params}"
|
||||||
|
)
|
||||||
|
|
||||||
|
streams = payload.get("data", {}).get("result", [])
|
||||||
|
|
||||||
|
for stream in streams:
|
||||||
|
labels = stream.get("stream", {})
|
||||||
|
|
||||||
|
for entry in stream.get("values", []):
|
||||||
|
if len(entry) < 2:
|
||||||
|
continue
|
||||||
|
|
||||||
|
message = entry[1]
|
||||||
|
metadata = entry[2] if len(entry) >= 3 else {}
|
||||||
|
|
||||||
|
if event not in message and (
|
||||||
|
metadata.get("event") != event
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
trace_id = (
|
||||||
|
metadata.get("trace_id")
|
||||||
|
or labels.get("trace_id")
|
||||||
|
)
|
||||||
|
|
||||||
|
span_id = (
|
||||||
|
metadata.get("span_id")
|
||||||
|
or labels.get("span_id")
|
||||||
|
)
|
||||||
|
|
||||||
|
if not trace_id or not TRACE_PATTERN.fullmatch(
|
||||||
|
trace_id
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not span_id or not SPAN_PATTERN.fullmatch(
|
||||||
|
span_id
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
return {
|
||||||
|
"service": service,
|
||||||
|
"event": event,
|
||||||
|
"message": message,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"span_id": span_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def verify_trace(log, expected_kind):
|
||||||
|
trace_id = log["trace_id"]
|
||||||
|
|
||||||
|
payload = get_json(
|
||||||
|
f"{TEMPO}/api/traces/{trace_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
batches = payload.get("batches", [])
|
||||||
|
spans = []
|
||||||
|
|
||||||
|
for batch in batches:
|
||||||
|
for scope in batch.get("scopeSpans", []):
|
||||||
|
spans.extend(scope.get("spans", []))
|
||||||
|
|
||||||
|
if not spans:
|
||||||
|
raise AssertionError(
|
||||||
|
f"No spans found for {trace_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
kinds = [str(span.get("kind", "")) for span in spans]
|
||||||
|
|
||||||
|
if not any(expected_kind in kind for kind in kinds):
|
||||||
|
raise AssertionError(
|
||||||
|
f"Expected span kind {expected_kind}; got {kinds}"
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Service: {log['service']}")
|
||||||
|
print(f"Event: {log['event']}")
|
||||||
|
print(f"Trace ID: {trace_id}")
|
||||||
|
print(f"Span ID: {log['span_id']}")
|
||||||
|
print(f"Tempo spans: {len(spans)}")
|
||||||
|
|
||||||
|
for span in spans:
|
||||||
|
print(
|
||||||
|
" Span:",
|
||||||
|
span.get("name"),
|
||||||
|
"kind:",
|
||||||
|
span.get("kind"),
|
||||||
|
)
|
||||||
|
|
||||||
|
print("PASS: Loki log linked to Tempo trace")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cases = [
|
||||||
|
(
|
||||||
|
"messenger-api",
|
||||||
|
"HTTP request completed",
|
||||||
|
"SERVER",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"messenger-worker",
|
||||||
|
"Image job handled",
|
||||||
|
"CONSUMER",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
for service, event, kind in cases:
|
||||||
|
log = find_log(service, event)
|
||||||
|
|
||||||
|
if log is None:
|
||||||
|
raise AssertionError(
|
||||||
|
f"No correlated log found for {service}"
|
||||||
|
)
|
||||||
|
|
||||||
|
verify_trace(log, kind)
|
||||||
|
|
||||||
|
print("PASS: API and worker trace correlation")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,256 @@
|
|||||||
|
import os
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
import zlib
|
||||||
|
import struct
|
||||||
|
|
||||||
|
|
||||||
|
BASE = "http://127.0.0.1:8080"
|
||||||
|
|
||||||
|
|
||||||
|
def docker(*args):
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "compose", *args],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def api(method, path, token=None, body=None, headers=None):
|
||||||
|
request_headers = dict(headers or {})
|
||||||
|
|
||||||
|
if token:
|
||||||
|
request_headers["Authorization"] = f"Bearer {token}"
|
||||||
|
|
||||||
|
if isinstance(body, dict):
|
||||||
|
body = json.dumps(body).encode("utf-8")
|
||||||
|
request_headers["Content-Type"] = "application/json"
|
||||||
|
|
||||||
|
request = urllib.request.Request(
|
||||||
|
BASE + path,
|
||||||
|
data=body,
|
||||||
|
headers=request_headers,
|
||||||
|
method=method,
|
||||||
|
)
|
||||||
|
|
||||||
|
with urllib.request.urlopen(request, timeout=10) as response:
|
||||||
|
data = response.read()
|
||||||
|
return (
|
||||||
|
response.status,
|
||||||
|
json.loads(data) if data and
|
||||||
|
"application/json" in response.headers.get(
|
||||||
|
"Content-Type", ""
|
||||||
|
) else data,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def png_chunk(name, data):
|
||||||
|
value = name + data
|
||||||
|
return (
|
||||||
|
struct.pack(">I", len(data))
|
||||||
|
+ value
|
||||||
|
+ struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_png():
|
||||||
|
width, height = 320, 200
|
||||||
|
scanline = b"\x00" + bytes([220, 80, 40]) * width
|
||||||
|
|
||||||
|
return (
|
||||||
|
b"\x89PNG\r\n\x1a\n"
|
||||||
|
+ png_chunk(
|
||||||
|
b"IHDR",
|
||||||
|
struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0),
|
||||||
|
)
|
||||||
|
+ png_chunk(
|
||||||
|
b"IDAT", zlib.compress(scanline * height)
|
||||||
|
)
|
||||||
|
+ png_chunk(b"IEND", b"")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_ready():
|
||||||
|
for _ in range(90):
|
||||||
|
try:
|
||||||
|
status, data = api("GET", "/health/ready")
|
||||||
|
if status == 200 and data.get("status") == "ok":
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
raise RuntimeError("API did not become ready")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
suffix = uuid.uuid4().hex[:12]
|
||||||
|
username = "lab5_" + suffix
|
||||||
|
password = "Lab5TestPassword_" + suffix
|
||||||
|
|
||||||
|
print("Stopping worker...", flush=True)
|
||||||
|
docker("stop", "worker")
|
||||||
|
wait_ready()
|
||||||
|
|
||||||
|
status, user = api(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/users",
|
||||||
|
body={
|
||||||
|
"username": username,
|
||||||
|
"display_name": "Lab5 recovery test",
|
||||||
|
"password": password,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert status == 201
|
||||||
|
|
||||||
|
credentials = base64.b64encode(
|
||||||
|
f"{username}:{password}".encode()
|
||||||
|
).decode()
|
||||||
|
|
||||||
|
status, session = api(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/auth/sessions",
|
||||||
|
headers={"Authorization": f"Basic {credentials}"},
|
||||||
|
body=b"",
|
||||||
|
)
|
||||||
|
assert status == 201
|
||||||
|
token = session["token"]
|
||||||
|
|
||||||
|
status, chat = api(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/chats",
|
||||||
|
token,
|
||||||
|
{
|
||||||
|
"title": "Worker recovery",
|
||||||
|
"member_ids": [user["id"]],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert status == 201
|
||||||
|
|
||||||
|
original = make_png()
|
||||||
|
path = (
|
||||||
|
f"/api/v1/chats/{chat['id']}/attachments"
|
||||||
|
"?filename=recovery.png"
|
||||||
|
)
|
||||||
|
|
||||||
|
status, attachment = api(
|
||||||
|
"POST",
|
||||||
|
path,
|
||||||
|
token,
|
||||||
|
original,
|
||||||
|
{"Content-Type": "image/png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert status == 202, f"Expected 202, got {status}"
|
||||||
|
assert attachment["status"] == "queued"
|
||||||
|
attachment_id = attachment["id"]
|
||||||
|
print("Upload: 202 queued", flush=True)
|
||||||
|
|
||||||
|
status, message = api(
|
||||||
|
"POST",
|
||||||
|
f"/api/v1/chats/{chat['id']}/messages",
|
||||||
|
token,
|
||||||
|
{"text": "", "attachment_ids": [attachment_id]},
|
||||||
|
{"Idempotency-Key": "recovery-" + suffix},
|
||||||
|
)
|
||||||
|
assert status == 201
|
||||||
|
assert attachment_id in message["attachment_ids"]
|
||||||
|
print("Message created while worker is stopped", flush=True)
|
||||||
|
|
||||||
|
status, metadata = api(
|
||||||
|
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||||
|
)
|
||||||
|
assert status == 200 and metadata["status"] == "queued"
|
||||||
|
|
||||||
|
print("Restarting RabbitMQ and API...", flush=True)
|
||||||
|
docker("restart", "rabbitmq")
|
||||||
|
docker("up", "-d", "--wait", "rabbitmq")
|
||||||
|
docker("restart", "api1", "api2")
|
||||||
|
wait_ready()
|
||||||
|
|
||||||
|
status, metadata = api(
|
||||||
|
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||||
|
)
|
||||||
|
assert metadata["status"] == "queued"
|
||||||
|
print("Queued attachment survived restart", flush=True)
|
||||||
|
|
||||||
|
pause_seconds = int(
|
||||||
|
os.getenv("LAB6_WORKER_PAUSE_SECONDS", "60")
|
||||||
|
)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"INCIDENT: worker stopped for {pause_seconds} seconds",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
time.sleep(pause_seconds)
|
||||||
|
|
||||||
|
print("Starting worker...", flush=True)
|
||||||
|
docker("up", "-d", "worker")
|
||||||
|
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
status, metadata = api(
|
||||||
|
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||||
|
)
|
||||||
|
|
||||||
|
if metadata["status"] == "ready":
|
||||||
|
break
|
||||||
|
|
||||||
|
if metadata["status"] == "failed":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Processing failed: {metadata.get('error_code')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
time.sleep(1)
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Attachment did not become ready")
|
||||||
|
|
||||||
|
status, downloaded = api(
|
||||||
|
"GET",
|
||||||
|
f"/api/v1/attachments/{attachment_id}/content",
|
||||||
|
token,
|
||||||
|
)
|
||||||
|
assert status == 200
|
||||||
|
assert downloaded == original
|
||||||
|
assert hashlib.sha256(downloaded).hexdigest() == (
|
||||||
|
hashlib.sha256(original).hexdigest()
|
||||||
|
)
|
||||||
|
|
||||||
|
status, thumbnail = api(
|
||||||
|
"GET",
|
||||||
|
f"/api/v1/attachments/{attachment_id}/content"
|
||||||
|
"?variant=thumbnail",
|
||||||
|
token,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert status == 200
|
||||||
|
assert thumbnail.startswith(b"\xff\xd8")
|
||||||
|
assert thumbnail.endswith(b"\xff\xd9")
|
||||||
|
|
||||||
|
variants = metadata["variants"]
|
||||||
|
assert any(
|
||||||
|
v["name"] == "thumbnail"
|
||||||
|
and v["width"] == 256
|
||||||
|
and v["height"] == 160
|
||||||
|
for v in variants
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Attachment: ready", flush=True)
|
||||||
|
print("Original: SHA-256 verified", flush=True)
|
||||||
|
print("Thumbnail: JPEG 256x160 verified", flush=True)
|
||||||
|
print("PASS: durable worker recovery", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
finally:
|
||||||
|
# Restore the worker even when the test fails.
|
||||||
|
docker("up", "-d", "worker")
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from fastapi import Request
|
||||||
|
from opentelemetry import trace
|
||||||
|
from opentelemetry.instrumentation.fastapi import (
|
||||||
|
FastAPIInstrumentor,
|
||||||
|
)
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
|
||||||
|
from src.telemetry import setup_telemetry
|
||||||
|
|
||||||
|
|
||||||
|
def configure_api_observability(app):
|
||||||
|
tracer, meter, logger = setup_telemetry(
|
||||||
|
"messenger-api"
|
||||||
|
)
|
||||||
|
|
||||||
|
request_counter = meter.create_counter(
|
||||||
|
name="messenger_http_requests",
|
||||||
|
unit="{request}",
|
||||||
|
description="Completed business HTTP requests",
|
||||||
|
)
|
||||||
|
|
||||||
|
class ObservabilityMiddleware(BaseHTTPMiddleware):
|
||||||
|
async def dispatch(self, request: Request, call_next):
|
||||||
|
# Health checks should not distort business RPS.
|
||||||
|
if request.url.path.startswith("/health/"):
|
||||||
|
return await call_next(request)
|
||||||
|
|
||||||
|
|
||||||
|
request_id = uuid.uuid4().hex
|
||||||
|
started = time.monotonic()
|
||||||
|
status_code = 500
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await call_next(request)
|
||||||
|
status_code = response.status_code
|
||||||
|
|
||||||
|
# Preserve the request ID produced by the existing
|
||||||
|
# API error handler. It must match the JSON body.
|
||||||
|
existing_id = response.headers.get("X-Request-Id")
|
||||||
|
|
||||||
|
if existing_id:
|
||||||
|
request_id = existing_id
|
||||||
|
else:
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
finally:
|
||||||
|
duration = time.monotonic() - started
|
||||||
|
|
||||||
|
# FastAPI provides a route template after routing.
|
||||||
|
route = request.scope.get("route")
|
||||||
|
|
||||||
|
if route is not None:
|
||||||
|
route_template = getattr(
|
||||||
|
route, "path", None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
route_template = None
|
||||||
|
|
||||||
|
# Do not use raw paths or dynamic IDs as labels.
|
||||||
|
if not route_template:
|
||||||
|
route_template = "unmatched"
|
||||||
|
|
||||||
|
method = request.method
|
||||||
|
|
||||||
|
if method not in (
|
||||||
|
"GET", "POST", "PUT",
|
||||||
|
"PATCH", "DELETE", "HEAD", "OPTIONS"
|
||||||
|
):
|
||||||
|
method = "OTHER"
|
||||||
|
|
||||||
|
status_class = f"{status_code // 100}xx"
|
||||||
|
|
||||||
|
attributes = {
|
||||||
|
"http.request.method": method,
|
||||||
|
"http.route": route_template,
|
||||||
|
"http.status_class": status_class,
|
||||||
|
}
|
||||||
|
|
||||||
|
request_counter.add(
|
||||||
|
1,
|
||||||
|
attributes=attributes,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The span is created by FastAPIInstrumentor.
|
||||||
|
span = trace.get_current_span()
|
||||||
|
|
||||||
|
if span.is_recording():
|
||||||
|
span.set_attribute(
|
||||||
|
"messenger.request_id",
|
||||||
|
request_id,
|
||||||
|
)
|
||||||
|
span.set_attribute(
|
||||||
|
"messenger.status_class",
|
||||||
|
status_class,
|
||||||
|
)
|
||||||
|
|
||||||
|
# No Authorization header, body, filename,
|
||||||
|
# chat ID, user ID or raw URL in telemetry.
|
||||||
|
logger.info(
|
||||||
|
"HTTP request completed",
|
||||||
|
extra={
|
||||||
|
"event": "http_request_completed",
|
||||||
|
"request_id": request_id,
|
||||||
|
"http_method": method,
|
||||||
|
"http_route": route_template,
|
||||||
|
"status_code": status_code,
|
||||||
|
"duration_ms": round(
|
||||||
|
duration * 1000, 3
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add our middleware first so it executes within
|
||||||
|
# the span created by the OTel ASGI middleware.
|
||||||
|
app.add_middleware(ObservabilityMiddleware)
|
||||||
|
|
||||||
|
FastAPIInstrumentor.instrument_app(
|
||||||
|
app,
|
||||||
|
excluded_urls=r".*/health/.*",
|
||||||
|
)
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
|
||||||
|
import io
|
||||||
|
import re
|
||||||
|
import uuid
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
|
||||||
|
|
||||||
|
MAX_BYTES = 10 * 1024 * 1024
|
||||||
|
MAX_PIXELS = 40_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def valid_filename(filename):
|
||||||
|
if not isinstance(filename, str):
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not 1 <= len(filename) <= 255:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if filename.strip() == "":
|
||||||
|
return False
|
||||||
|
|
||||||
|
if "/" in filename or "\\" in filename:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if any(ord(ch) < 32 or ord(ch) == 127 for ch in filename):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_image(data, declared_mime):
|
||||||
|
if len(data) > MAX_BYTES:
|
||||||
|
raise ValueError("too_large")
|
||||||
|
|
||||||
|
if not data:
|
||||||
|
raise ValueError("invalid_image")
|
||||||
|
|
||||||
|
if declared_mime not in ("image/png", "image/jpeg"):
|
||||||
|
raise ValueError("invalid_image")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with Image.open(io.BytesIO(data)) as image:
|
||||||
|
image_format = image.format
|
||||||
|
|
||||||
|
width, height = image.size
|
||||||
|
|
||||||
|
if width * height > MAX_PIXELS:
|
||||||
|
raise ValueError("too_many_pixels")
|
||||||
|
|
||||||
|
if image_format not in ("PNG", "JPEG"):
|
||||||
|
raise ValueError("invalid_image")
|
||||||
|
|
||||||
|
actual_mime = (
|
||||||
|
"image/png"
|
||||||
|
if image_format == "PNG"
|
||||||
|
else "image/jpeg"
|
||||||
|
)
|
||||||
|
|
||||||
|
if declared_mime != actual_mime:
|
||||||
|
raise ValueError("invalid_image")
|
||||||
|
|
||||||
|
image.verify()
|
||||||
|
|
||||||
|
except Image.DecompressionBombError:
|
||||||
|
raise ValueError("too_many_pixels")
|
||||||
|
except (
|
||||||
|
UnidentifiedImageError,
|
||||||
|
OSError,
|
||||||
|
SyntaxError,
|
||||||
|
EOFError,
|
||||||
|
):
|
||||||
|
raise ValueError("invalid_image")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"content_type": actual_mime,
|
||||||
|
"width": width,
|
||||||
|
"height": height,
|
||||||
|
"size_bytes": len(data),
|
||||||
|
"sha256": hashlib.sha256(data).hexdigest(),
|
||||||
|
}
|
||||||
|
|
||||||
|
def attachment_json(row, variants=None):
|
||||||
|
from datetime import timezone
|
||||||
|
|
||||||
|
def timestamp(value):
|
||||||
|
return value.astimezone(timezone.utc).isoformat(
|
||||||
|
timespec="microseconds"
|
||||||
|
).replace("+00:00", "Z")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": str(row["id"]),
|
||||||
|
"chat_id": str(row["chat_id"]),
|
||||||
|
"owner_id": str(row["uploader_id"]),
|
||||||
|
"filename": row["filename"],
|
||||||
|
"content_type": row["content_type"],
|
||||||
|
"size_bytes": row["size_bytes"],
|
||||||
|
"sha256": row["sha256"],
|
||||||
|
"status": row["status"],
|
||||||
|
"variants": variants if variants is not None else [],
|
||||||
|
"error_code": row["error_code"],
|
||||||
|
"created_at": timestamp(row["created_at"]),
|
||||||
|
"updated_at": timestamp(row["updated_at"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
def object_key():
|
||||||
|
return f"originals/{uuid.uuid4()}"
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
from src.queue import publish_job
|
||||||
|
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(message)s",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger("image-dispatcher")
|
||||||
|
|
||||||
|
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch_one():
|
||||||
|
with psycopg.connect(
|
||||||
|
DATABASE_URL,
|
||||||
|
row_factory=dict_row,
|
||||||
|
) as conn:
|
||||||
|
|
||||||
|
# Only publish due, unpublished jobs.
|
||||||
|
# The row lock also prevents two dispatchers
|
||||||
|
# from selecting the same job simultaneously.
|
||||||
|
job = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id
|
||||||
|
FROM attachment_jobs
|
||||||
|
WHERE status = 'queued'
|
||||||
|
AND published_at IS NULL
|
||||||
|
AND next_attempt_at <= NOW()
|
||||||
|
ORDER BY created_at, id
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
""",
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
if job is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
job_id = str(job["id"])
|
||||||
|
|
||||||
|
# RabbitMQ publisher confirms are enabled.
|
||||||
|
# An exception rolls back this DB transaction.
|
||||||
|
publish_job(job_id)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET published_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Published job %s", job_id)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logger.info("Dispatcher started")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
if not dispatch_one():
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"Dispatch failed; retrying"
|
||||||
|
)
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+1277
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
|||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
|
||||||
|
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||||
|
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations():
|
||||||
|
with psycopg.connect(DATABASE_URL) as conn:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
# Prevent simultaneous migration runs.
|
||||||
|
cur.execute("SELECT pg_advisory_xact_lock(2026002)")
|
||||||
|
|
||||||
|
cur.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version TEXT PRIMARY KEY,
|
||||||
|
applied_at TIMESTAMPTZ NOT NULL
|
||||||
|
DEFAULT clock_timestamp()
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
for file in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||||
|
cur.execute(
|
||||||
|
"SELECT 1 FROM schema_migrations WHERE version = %s",
|
||||||
|
(file.name,)
|
||||||
|
)
|
||||||
|
|
||||||
|
if cur.fetchone():
|
||||||
|
print(f"Already applied: {file.name}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
sql = file.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
for statement in sql.split(";"):
|
||||||
|
if statement.strip():
|
||||||
|
cur.execute(statement)
|
||||||
|
|
||||||
|
cur.execute(
|
||||||
|
"INSERT INTO schema_migrations (version) VALUES (%s)",
|
||||||
|
(file.name,)
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Applied migration: {file.name}")
|
||||||
|
|
||||||
|
print("Migrations completed")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run_migrations()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pika
|
||||||
|
|
||||||
|
|
||||||
|
RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq")
|
||||||
|
RABBITMQ_USER = os.environ["RABBITMQ_USER"]
|
||||||
|
RABBITMQ_PASSWORD = os.environ["RABBITMQ_PASSWORD"]
|
||||||
|
RABBITMQ_QUEUE = os.getenv(
|
||||||
|
"RABBITMQ_QUEUE",
|
||||||
|
"image_jobs",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connect():
|
||||||
|
credentials = pika.PlainCredentials(
|
||||||
|
RABBITMQ_USER,
|
||||||
|
RABBITMQ_PASSWORD,
|
||||||
|
)
|
||||||
|
|
||||||
|
parameters = pika.ConnectionParameters(
|
||||||
|
host=RABBITMQ_HOST,
|
||||||
|
port=5672,
|
||||||
|
credentials=credentials,
|
||||||
|
heartbeat=30,
|
||||||
|
blocked_connection_timeout=15,
|
||||||
|
connection_attempts=3,
|
||||||
|
retry_delay=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
return pika.BlockingConnection(parameters)
|
||||||
|
|
||||||
|
|
||||||
|
def declare_queue(channel):
|
||||||
|
channel.queue_declare(
|
||||||
|
queue=RABBITMQ_QUEUE,
|
||||||
|
durable=True,
|
||||||
|
arguments={
|
||||||
|
"x-queue-type": "quorum",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def publish_job(job_id):
|
||||||
|
connection = connect()
|
||||||
|
|
||||||
|
try:
|
||||||
|
channel = connection.channel()
|
||||||
|
declare_queue(channel)
|
||||||
|
|
||||||
|
# Wait for confirmation from RabbitMQ.
|
||||||
|
channel.confirm_delivery()
|
||||||
|
|
||||||
|
payload = json.dumps({
|
||||||
|
"job_id": str(uuid.UUID(str(job_id))),
|
||||||
|
}).encode("utf-8")
|
||||||
|
|
||||||
|
channel.basic_publish(
|
||||||
|
exchange="",
|
||||||
|
routing_key=RABBITMQ_QUEUE,
|
||||||
|
body=payload,
|
||||||
|
properties=pika.BasicProperties(
|
||||||
|
delivery_mode=2,
|
||||||
|
content_type="application/json",
|
||||||
|
message_id=str(job_id),
|
||||||
|
),
|
||||||
|
mandatory=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
if connection.is_open:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_job(body):
|
||||||
|
payload = json.loads(body)
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError("Invalid job payload")
|
||||||
|
|
||||||
|
job_id = payload.get("job_id")
|
||||||
|
|
||||||
|
if not isinstance(job_id, str):
|
||||||
|
raise ValueError("Missing job_id")
|
||||||
|
|
||||||
|
return str(uuid.UUID(job_id))
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
bucket = os.environ["S3_BUCKET"]
|
||||||
|
|
||||||
|
client = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||||
|
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||||
|
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||||
|
region_name="us-east-1",
|
||||||
|
config=Config(
|
||||||
|
signature_version="s3v4",
|
||||||
|
s3={"addressing_style": "path"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
for attempt in range(30):
|
||||||
|
try:
|
||||||
|
client.head_bucket(Bucket=bucket)
|
||||||
|
print(f"Bucket already exists: {bucket}", flush=True)
|
||||||
|
break
|
||||||
|
|
||||||
|
except ClientError as exc:
|
||||||
|
code = exc.response["Error"]["Code"]
|
||||||
|
|
||||||
|
if code in ("404", "NoSuchBucket"):
|
||||||
|
try:
|
||||||
|
client.create_bucket(Bucket=bucket)
|
||||||
|
print(f"Bucket created: {bucket}", flush=True)
|
||||||
|
break
|
||||||
|
except Exception as create_error:
|
||||||
|
print(
|
||||||
|
f"Waiting for S3: {create_error}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(f"Waiting for S3: {exc}", flush=True)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"Waiting for S3: {exc}", flush=True)
|
||||||
|
|
||||||
|
if attempt == 29:
|
||||||
|
raise RuntimeError("Cannot initialize S3 bucket")
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
# No public bucket policy is configured.
|
||||||
|
print(f"Private S3 bucket is ready: {bucket}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
|
||||||
|
from opentelemetry import metrics, trace
|
||||||
|
from opentelemetry._logs import set_logger_provider
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
|
||||||
|
OTLPLogExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
||||||
|
OTLPMetricExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||||
|
OTLPSpanExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import (
|
||||||
|
BatchLogRecordProcessor,
|
||||||
|
)
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import (
|
||||||
|
PeriodicExportingMetricReader,
|
||||||
|
)
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import (
|
||||||
|
BatchSpanProcessor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
SERVICE_VERSION = "lab6-1.0"
|
||||||
|
ENVIRONMENT = "lab"
|
||||||
|
|
||||||
|
OTLP_ENDPOINT = os.getenv(
|
||||||
|
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||||
|
"http://otel-collector:4318",
|
||||||
|
).rstrip("/")
|
||||||
|
|
||||||
|
|
||||||
|
class TraceContextFilter(logging.Filter):
|
||||||
|
def filter(self, record):
|
||||||
|
context = trace.get_current_span().get_span_context()
|
||||||
|
|
||||||
|
if context.is_valid:
|
||||||
|
record.trace_id = format(
|
||||||
|
context.trace_id, "032x"
|
||||||
|
)
|
||||||
|
record.span_id = format(
|
||||||
|
context.span_id, "016x"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
record.trace_id = "-"
|
||||||
|
record.span_id = "-"
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def setup_telemetry(service_name):
|
||||||
|
"""
|
||||||
|
Initialize OTLP traces, metrics and logs.
|
||||||
|
|
||||||
|
Do not record request bodies, Authorization
|
||||||
|
headers, message text or image bytes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
resource = Resource.create(
|
||||||
|
{
|
||||||
|
"service.name": service_name,
|
||||||
|
"service.version": SERVICE_VERSION,
|
||||||
|
"service.instance.id": os.getenv(
|
||||||
|
"HOSTNAME",
|
||||||
|
socket.gethostname(),
|
||||||
|
),
|
||||||
|
"deployment.environment.name": ENVIRONMENT,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Distributed traces -> Collector -> Tempo
|
||||||
|
tracer_provider = TracerProvider(resource=resource)
|
||||||
|
|
||||||
|
trace_exporter = OTLPSpanExporter(
|
||||||
|
endpoint=f"{OTLP_ENDPOINT}/v1/traces",
|
||||||
|
timeout=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
tracer_provider.add_span_processor(
|
||||||
|
BatchSpanProcessor(
|
||||||
|
trace_exporter,
|
||||||
|
max_queue_size=1024,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
trace.set_tracer_provider(tracer_provider)
|
||||||
|
|
||||||
|
# Metrics -> Collector -> Prometheus
|
||||||
|
metric_exporter = OTLPMetricExporter(
|
||||||
|
endpoint=f"{OTLP_ENDPOINT}/v1/metrics",
|
||||||
|
timeout=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
metric_reader = PeriodicExportingMetricReader(
|
||||||
|
metric_exporter,
|
||||||
|
export_interval_millis=5000,
|
||||||
|
export_timeout_millis=3000,
|
||||||
|
)
|
||||||
|
|
||||||
|
meter_provider = MeterProvider(
|
||||||
|
resource=resource,
|
||||||
|
metric_readers=[metric_reader],
|
||||||
|
)
|
||||||
|
|
||||||
|
metrics.set_meter_provider(meter_provider)
|
||||||
|
|
||||||
|
# Structured logs -> Collector -> Loki
|
||||||
|
logger_provider = LoggerProvider(resource=resource)
|
||||||
|
|
||||||
|
log_exporter = OTLPLogExporter(
|
||||||
|
endpoint=f"{OTLP_ENDPOINT}/v1/logs",
|
||||||
|
timeout=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger_provider.add_log_record_processor(
|
||||||
|
BatchLogRecordProcessor(
|
||||||
|
log_exporter,
|
||||||
|
max_queue_size=1024,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
set_logger_provider(logger_provider)
|
||||||
|
|
||||||
|
# Dedicated application logger.
|
||||||
|
# We intentionally do not forward arbitrary
|
||||||
|
# library logs or sensitive request data.
|
||||||
|
logger = logging.getLogger("messenger.telemetry")
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.propagate = False
|
||||||
|
|
||||||
|
for handler in list(logger.handlers):
|
||||||
|
logger.removeHandler(handler)
|
||||||
|
|
||||||
|
context_filter = TraceContextFilter()
|
||||||
|
|
||||||
|
console_handler = logging.StreamHandler()
|
||||||
|
console_handler.addFilter(context_filter)
|
||||||
|
console_handler.setFormatter(
|
||||||
|
logging.Formatter(
|
||||||
|
"%(asctime)s %(levelname)s "
|
||||||
|
"event=%(event)s "
|
||||||
|
"trace_id=%(trace_id)s "
|
||||||
|
"span_id=%(span_id)s "
|
||||||
|
"%(message)s"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.addHandler(console_handler)
|
||||||
|
|
||||||
|
otlp_handler = LoggingHandler(
|
||||||
|
level=logging.INFO,
|
||||||
|
logger_provider=logger_provider,
|
||||||
|
)
|
||||||
|
otlp_handler.addFilter(context_filter)
|
||||||
|
logger.addHandler(otlp_handler)
|
||||||
|
|
||||||
|
tracer = trace.get_tracer(service_name, SERVICE_VERSION)
|
||||||
|
meter = metrics.get_meter(service_name, SERVICE_VERSION)
|
||||||
|
|
||||||
|
return tracer, meter, logger
|
||||||
+576
@@ -0,0 +1,576 @@
|
|||||||
|
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
import pika
|
||||||
|
import psycopg
|
||||||
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||||
|
from src.telemetry import setup_telemetry
|
||||||
|
from botocore.config import Config
|
||||||
|
from PIL import Image, ImageOps
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
from src.queue import (
|
||||||
|
RABBITMQ_QUEUE,
|
||||||
|
connect,
|
||||||
|
declare_queue,
|
||||||
|
parse_job,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(message)s",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger("image-worker")
|
||||||
|
|
||||||
|
worker_tracer, worker_meter, worker_logger = setup_telemetry(
|
||||||
|
"messenger-worker"
|
||||||
|
)
|
||||||
|
|
||||||
|
jobs_processed = worker_meter.create_counter(
|
||||||
|
name="messenger_worker_jobs_processed",
|
||||||
|
unit="{job}",
|
||||||
|
description="Image processing job attempts by outcome",
|
||||||
|
)
|
||||||
|
|
||||||
|
job_duration = worker_meter.create_histogram(
|
||||||
|
name="messenger_worker_job_duration",
|
||||||
|
unit="s",
|
||||||
|
description="Image processing duration in seconds",
|
||||||
|
)
|
||||||
|
|
||||||
|
DATABASE_URL = os.environ["DATABASE_URL"]
|
||||||
|
S3_BUCKET = os.environ["S3_BUCKET"]
|
||||||
|
MAX_BYTES = 10 * 1024 * 1024
|
||||||
|
|
||||||
|
s3 = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||||
|
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||||
|
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||||
|
region_name="us-east-1",
|
||||||
|
config=Config(
|
||||||
|
signature_version="s3v4",
|
||||||
|
s3={"addressing_style": "path"},
|
||||||
|
connect_timeout=3,
|
||||||
|
read_timeout=15,
|
||||||
|
retries={"max_attempts": 1},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidImage(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def make_thumbnail(original):
|
||||||
|
try:
|
||||||
|
with Image.open(io.BytesIO(original)) as image:
|
||||||
|
if image.format not in ("PNG", "JPEG"):
|
||||||
|
raise InvalidImage("Unsupported image")
|
||||||
|
|
||||||
|
if image.width * image.height > 40_000_000:
|
||||||
|
raise InvalidImage("Too many pixels")
|
||||||
|
|
||||||
|
image = ImageOps.exif_transpose(image)
|
||||||
|
image.thumbnail(
|
||||||
|
(256, 256),
|
||||||
|
Image.Resampling.LANCZOS,
|
||||||
|
)
|
||||||
|
|
||||||
|
# JPEG does not support transparency.
|
||||||
|
rgba = image.convert("RGBA")
|
||||||
|
|
||||||
|
background = Image.new(
|
||||||
|
"RGB",
|
||||||
|
rgba.size,
|
||||||
|
(255, 255, 255),
|
||||||
|
)
|
||||||
|
|
||||||
|
background.paste(
|
||||||
|
rgba,
|
||||||
|
mask=rgba.getchannel("A"),
|
||||||
|
)
|
||||||
|
|
||||||
|
output = io.BytesIO()
|
||||||
|
|
||||||
|
background.save(
|
||||||
|
output,
|
||||||
|
format="JPEG",
|
||||||
|
quality=85,
|
||||||
|
optimize=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return (
|
||||||
|
output.getvalue(),
|
||||||
|
background.width,
|
||||||
|
background.height,
|
||||||
|
)
|
||||||
|
|
||||||
|
except (OSError, SyntaxError, EOFError) as exc:
|
||||||
|
raise InvalidImage("Invalid image") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def get_staging_object(key):
|
||||||
|
response = s3.get_object(
|
||||||
|
Bucket=S3_BUCKET,
|
||||||
|
Key=key,
|
||||||
|
)
|
||||||
|
|
||||||
|
stream = response["Body"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = stream.read(MAX_BYTES + 1)
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
if not data or len(data) > MAX_BYTES:
|
||||||
|
raise InvalidImage("Invalid staging size")
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def process_job(job_id):
|
||||||
|
"""
|
||||||
|
Use a PostgreSQL row lock to prevent concurrent
|
||||||
|
processing of the same logical job.
|
||||||
|
|
||||||
|
The transaction commits before RabbitMQ ACK.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with psycopg.connect(
|
||||||
|
DATABASE_URL,
|
||||||
|
row_factory=dict_row,
|
||||||
|
) as conn:
|
||||||
|
|
||||||
|
job = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT j.*, a.staging_key,
|
||||||
|
a.object_key, a.sha256,
|
||||||
|
a.size_bytes, a.status AS attachment_status
|
||||||
|
FROM attachment_jobs j
|
||||||
|
JOIN attachments a
|
||||||
|
ON a.id = j.attachment_id
|
||||||
|
WHERE j.id = %s
|
||||||
|
FOR UPDATE OF j, a
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
if job is None:
|
||||||
|
logger.info("Unknown job %s, ignoring", job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
if job["status"] == "ready":
|
||||||
|
logger.info("Job %s already completed", job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
if job["status"] == "failed":
|
||||||
|
logger.info("Job %s already failed", job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
if job["next_attempt_at"] is not None:
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
if job["next_attempt_at"] > datetime.now(timezone.utc):
|
||||||
|
logger.info("Job %s is not due yet", job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
attempts = job["attempts"] + 1
|
||||||
|
|
||||||
|
if attempts > job["max_attempts"]:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET status = 'failed',
|
||||||
|
completed_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'failed',
|
||||||
|
error_code = 'processing_failed',
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job["attachment_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET status = 'processing',
|
||||||
|
attempts = %s,
|
||||||
|
started_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(attempts, job_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'processing',
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job["attachment_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
original = get_staging_object(
|
||||||
|
job["staging_key"]
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
len(original) != job["size_bytes"]
|
||||||
|
or hashlib.sha256(original).hexdigest()
|
||||||
|
!= job["sha256"]
|
||||||
|
):
|
||||||
|
raise InvalidImage("Staging checksum mismatch")
|
||||||
|
|
||||||
|
thumbnail, width, height = make_thumbnail(
|
||||||
|
original
|
||||||
|
)
|
||||||
|
|
||||||
|
attachment_id = str(job["attachment_id"])
|
||||||
|
|
||||||
|
# Stable keys make duplicate delivery safe.
|
||||||
|
original_key = job["object_key"]
|
||||||
|
thumb_key = (
|
||||||
|
f"thumbnails/{attachment_id}.jpg"
|
||||||
|
)
|
||||||
|
|
||||||
|
s3.put_object(
|
||||||
|
Bucket=S3_BUCKET,
|
||||||
|
Key=original_key,
|
||||||
|
Body=original,
|
||||||
|
ContentType="image/png"
|
||||||
|
if original[:8] == b"\x89PNG\r\n\x1a\n"
|
||||||
|
else "image/jpeg",
|
||||||
|
)
|
||||||
|
|
||||||
|
s3.put_object(
|
||||||
|
Bucket=S3_BUCKET,
|
||||||
|
Key=thumb_key,
|
||||||
|
Body=thumbnail,
|
||||||
|
ContentType="image/jpeg",
|
||||||
|
)
|
||||||
|
|
||||||
|
thumb_sha256 = hashlib.sha256(
|
||||||
|
thumbnail
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO attachment_variants
|
||||||
|
(
|
||||||
|
attachment_id,
|
||||||
|
variant_name,
|
||||||
|
object_key,
|
||||||
|
content_type,
|
||||||
|
size_bytes,
|
||||||
|
sha256,
|
||||||
|
width,
|
||||||
|
height
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
%s, 'thumbnail', %s, 'image/jpeg',
|
||||||
|
%s, %s, %s, %s
|
||||||
|
)
|
||||||
|
ON CONFLICT (
|
||||||
|
attachment_id, variant_name
|
||||||
|
)
|
||||||
|
DO UPDATE SET
|
||||||
|
object_key = EXCLUDED.object_key,
|
||||||
|
size_bytes = EXCLUDED.size_bytes,
|
||||||
|
sha256 = EXCLUDED.sha256,
|
||||||
|
width = EXCLUDED.width,
|
||||||
|
height = EXCLUDED.height
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
job["attachment_id"],
|
||||||
|
thumb_key,
|
||||||
|
len(thumbnail),
|
||||||
|
thumb_sha256,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'ready',
|
||||||
|
error_code = NULL,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job["attachment_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET status = 'ready',
|
||||||
|
completed_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Job %s ready, thumbnail %sx%s",
|
||||||
|
job_id,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception(
|
||||||
|
"Processing failed for job %s: %s",
|
||||||
|
job_id,
|
||||||
|
type(exc).__name__,
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
attempts >= job["max_attempts"]
|
||||||
|
or isinstance(exc, InvalidImage)
|
||||||
|
):
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET status = 'failed',
|
||||||
|
completed_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'failed',
|
||||||
|
error_code = 'processing_failed',
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job["attachment_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachment_jobs
|
||||||
|
SET status = 'queued',
|
||||||
|
next_attempt_at =
|
||||||
|
NOW() + INTERVAL '5 seconds',
|
||||||
|
published_at = NULL,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE attachments
|
||||||
|
SET status = 'queued',
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job["attachment_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_job_status(job_id):
|
||||||
|
with psycopg.connect(DATABASE_URL) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT status
|
||||||
|
FROM attachment_jobs
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(job_id,),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
return row[0] if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def on_message(channel, method, properties, body):
|
||||||
|
try:
|
||||||
|
job_id = parse_job(body)
|
||||||
|
except (ValueError, TypeError, KeyError):
|
||||||
|
logger.warning("Invalid queue message")
|
||||||
|
channel.basic_ack(
|
||||||
|
delivery_tag=method.delivery_tag
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
attributes = {
|
||||||
|
"messaging.system": "rabbitmq",
|
||||||
|
"messaging.destination.name": RABBITMQ_QUEUE,
|
||||||
|
"messaging.operation.name": "process",
|
||||||
|
"messenger.job_type": "image_thumbnail",
|
||||||
|
}
|
||||||
|
|
||||||
|
# A real consumer span around the actual job processing.
|
||||||
|
with worker_tracer.start_as_current_span(
|
||||||
|
"image_job.process",
|
||||||
|
kind=SpanKind.CONSUMER,
|
||||||
|
attributes=attributes,
|
||||||
|
) as span:
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
|
||||||
|
try:
|
||||||
|
previous_status = get_job_status(job_id)
|
||||||
|
|
||||||
|
process_job(job_id)
|
||||||
|
|
||||||
|
current_status = get_job_status(job_id)
|
||||||
|
duration = time.monotonic() - started
|
||||||
|
|
||||||
|
if previous_status in ("ready", "failed"):
|
||||||
|
outcome = "duplicate"
|
||||||
|
|
||||||
|
elif current_status == "ready":
|
||||||
|
outcome = "success"
|
||||||
|
|
||||||
|
elif current_status == "failed":
|
||||||
|
outcome = "failed"
|
||||||
|
|
||||||
|
elif current_status == "queued":
|
||||||
|
outcome = "retry"
|
||||||
|
|
||||||
|
else:
|
||||||
|
outcome = "ignored"
|
||||||
|
|
||||||
|
span.set_attribute(
|
||||||
|
"messenger.job.outcome",
|
||||||
|
outcome,
|
||||||
|
)
|
||||||
|
|
||||||
|
if outcome == "failed":
|
||||||
|
span.set_status(
|
||||||
|
Status(StatusCode.ERROR)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Count actual processing attempts, not
|
||||||
|
# duplicate deliveries of completed jobs.
|
||||||
|
if outcome in ("success", "failed", "retry"):
|
||||||
|
metric_attributes = {
|
||||||
|
"job_type": "image_thumbnail",
|
||||||
|
"outcome": outcome,
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs_processed.add(
|
||||||
|
1,
|
||||||
|
attributes=metric_attributes,
|
||||||
|
)
|
||||||
|
|
||||||
|
job_duration.record(
|
||||||
|
duration,
|
||||||
|
attributes=metric_attributes,
|
||||||
|
)
|
||||||
|
|
||||||
|
worker_logger.info(
|
||||||
|
"Image job handled",
|
||||||
|
extra={
|
||||||
|
"event": "image_job_handled",
|
||||||
|
"job_type": "image_thumbnail",
|
||||||
|
"outcome": outcome,
|
||||||
|
"duration_seconds": round(duration, 4),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# ACK after process_job has committed the
|
||||||
|
# PostgreSQL transaction.
|
||||||
|
channel.basic_ack(
|
||||||
|
delivery_tag=method.delivery_tag
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
span.record_exception(exc)
|
||||||
|
span.set_status(Status(StatusCode.ERROR))
|
||||||
|
|
||||||
|
worker_logger.error(
|
||||||
|
"Image job processing interrupted",
|
||||||
|
extra={
|
||||||
|
"event": "image_job_interrupted",
|
||||||
|
"job_type": "image_thumbnail",
|
||||||
|
"error_type": type(exc).__name__,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.exception(
|
||||||
|
"Unexpected failure for job %s",
|
||||||
|
job_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
channel.basic_nack(
|
||||||
|
delivery_tag=method.delivery_tag,
|
||||||
|
requeue=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
connection = connect()
|
||||||
|
channel = connection.channel()
|
||||||
|
|
||||||
|
declare_queue(channel)
|
||||||
|
|
||||||
|
channel.basic_qos(
|
||||||
|
prefetch_count=1
|
||||||
|
)
|
||||||
|
|
||||||
|
channel.basic_consume(
|
||||||
|
queue=RABBITMQ_QUEUE,
|
||||||
|
on_message_callback=on_message,
|
||||||
|
auto_ack=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Worker listening on %s",
|
||||||
|
RABBITMQ_QUEUE,
|
||||||
|
)
|
||||||
|
|
||||||
|
channel.start_consuming()
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"Worker disconnected, reconnecting"
|
||||||
|
)
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
if (
|
||||||
|
"connection" in locals()
|
||||||
|
and connection.is_open
|
||||||
|
):
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user