3 Commits
117 changed files with 59608 additions and 30 deletions
+6
View File
@@ -18,3 +18,9 @@ __pycache__
node_modules
evidence
.DS_Store
infra/secrets
Защита.md
ЗАЩИТА.md
защита.md
+16 -4
View File
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
@@ -19,12 +19,24 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
# The current consumer is sequential; this documents capacity, not a tuning knob.
WORKER_CONCURRENCY=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_TIMEOUT_SECONDS=30
JOB_TOTAL_TIMEOUT_SECONDS=90
JOB_LEASE_SECONDS=120
STAGING_RETENTION_SECONDS=604800
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Actual Compose overrides (all other application settings have defaults).
LB_PORT=8080
LB_STATS_PORT=8404
GRAFANA_PORT=3000
PROMETHEUS_PORT=9090
LOKI_PORT=3100
TEMPO_PORT=3200
# If another lab uses 8080/8404, choose e.g. 8086/8406 locally.
+5
View File
@@ -0,0 +1,5 @@
* text=auto
*.sh text eol=lf
*.py text eol=lf
*.yaml text eol=lf
Dockerfile* text eol=lf
+20
View File
@@ -0,0 +1,20 @@
name: Application smoke
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: sh scripts/init-secrets.sh
- run: docker compose up -d --build --wait --wait-timeout 240
- run: docker compose --profile tools run --rm -T --no-deps tester make check test
- run: docker compose exec -T api python - < student-tests/observability_unit.py
- run: python scripts/observe_lab6.py --baseline
- if: always()
run: docker compose down
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
# Local defense preparation
/защита.md
/Защита.md
/ЗАЩИТА.md
+24
View File
@@ -0,0 +1,24 @@
# Messenger lab5: one image for API, publisher, worker, migrations and bootstrap.
FROM python:3.12.7-slim
WORKDIR /app
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app ./app
COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/
COPY migrations ./migrations
RUN useradd --create-home --uid 10001 appuser
USER appuser
EXPOSE 8080
# Exec form: python is PID 1 and receives SIGTERM from "docker stop" itself.
# (lab2 used "sh -c uvicorn ...": sh stayed PID 1, uvicorn never got SIGTERM
# and was killed together with the container instead of draining.)
CMD ["python", "-m", "app.server"]
+288 -22
View File
@@ -1,37 +1,303 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Автор: Максимова Мария, БВТ2303. Целевая оценка: 4.
## Перенос lab5
Исходники перенесены из соседнего репозитория `mtusi2026lab5`, версия
`19cbacd`. Перенесены приложение, миграции 001–005, Dockerfile, Compose,
операторские скрипты, собственные проверки. Контракт и публичные тесты lab6
сохранены. Доказательства старой лабораторной не выдаются за результаты lab6.
Архитектура: HAProxy → две реплики FastAPI → PostgreSQL/Valkey и приватное
S3 (SeaweedFS). До 202 сохраняются S3 staging и транзакционный outbox в БД.
Publisher доставляет задания в RabbitMQ, worker публикует original,
thumbnail JPEG 256×256, crop JPEG 128×128 и optimized WebP 1280×1280.
Применяются at least once, publisher confirms, ack после commit,
lease с fencing token, ограниченные retry с jitter и журнал ошибок.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine/Compose v2 (Linux containers), доступ к registry/PyPI,
около 8 GiB RAM у Docker и Python 3.9+ для stdlib HTTP-проверок.
Образ приложения: Python 3.12.7; PostgreSQL 16.4, Valkey 8.0.1,
SeaweedFS 3.97, RabbitMQ 3.13.7, HAProxy 3.0.11. Python-зависимости
закреплены в `requirements.txt`. `Dockerfile` общий для API/publisher/worker.
## Архитектура и решения
Windows PowerShell:
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```powershell
powershell -ExecutionPolicy Bypass -File scripts/init-secrets.ps1
Copy-Item .env.example .env
# Если заняты 8080/8404, изменить LB_PORT/LB_STATS_PORT в .env.
docker compose config --quiet
docker compose up -d --build --wait --wait-timeout 240
docker compose --profile tools build tester
docker compose --profile tools run --rm -T --no-deps tester make check test
$env:BASE_URL = 'http://localhost:8080' # либо выбранный LB_PORT
python -X utf8 scripts/observe_lab6.py --baseline
python -X utf8 scripts/observe_lab6.py --chain-with-retry
python -X utf8 scripts/observe_lab6.py --incidents
```
## Доказательства критериев
Linux/macOS: `sh scripts/init-secrets.sh`, `cp .env.example .env`, те же
команды Compose; `BASE_URL=http://localhost:8080 python3 scripts/observe_lab6.py ...`.
Unit: `docker compose exec -T api python - < student-tests/observability_unit.py`
(на PowerShell: `Get-Content -Raw student-tests/observability_unit.py | docker compose exec -T api python -`).
Для полной схемы: `python -m venv .venv`, установить `requirements-dev.txt`
в venv, затем `python -X utf8 scripts/validate_openapi.py` из этого venv.
Собственные проверки изображений/восстановления: `python -X utf8 scripts/test_lab5.py`
(без `--quick` выполняются намеренные отказы). Не запускать сценарии отказов параллельно.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Init-скрипты создают случайные локальные credentials в `infra/secrets/`,
существующие сохраняют; файлы исключены из Git и Docker build context.
`migrate` до запуска API последовательно применяет миграции 001–006 под
блокировкой и журналирует их; повторный запуск пропускает уже применённые.
`s3-init` создаёт приватный bucket и проверяет права. Ручной seed не нужен.
`--wait` проверяет health основных компонентов; baseline дополнительно
ожидает поступление реальных данных в backend.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Безопасная остановка: `docker compose down`. Данные сохраняются в volumes.
**Только для намеренного сброса учебных данных:** `docker compose down -v`.
Эта команда удалит и бизнес-данные, и сохранённую телеметрию.
В текущем прогоне используются локальные порты API **8086**, LB stats **8406**,
Grafana **3000**, Prometheus **9090**, Loki **3100**, Tempo **3200**.
## Эксперименты и отказы
## Наблюдаемость: уровень 3
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
API, worker и publisher используют OpenTelemetry Python SDK 1.28.2,
OTLP HTTP/protobuf → Collector contrib 0.111.0 → Prometheus 2.55.0,
Loki 3.2.1 (native OTLP), Tempo 2.6.1 → Grafana 11.2.2.
Полные конфигурации и импортируемый dashboard находятся в `infra/`.
Данные backend и состояние Grafana сохраняются в именованных volumes.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Метрики SDK `messenger.http.requests` и `messenger.jobs.processed` — counters;
`messenger.http.duration` и `messenger.jobs.duration` — histograms в секундах.
Prometheus добавляет `_total` к counters и `_seconds_bucket/sum/count` к
histograms. Labels HTTP: method, route template, status_class; jobs:
job_type и outcome. Resource attributes: service.name, service.version=1.6.0,
service.instance.id, deployment.environment.name=lab. Сервисы называются
messenger-api, messenger-worker и messenger-publisher.
## Ограничения и следующие шаги
API log `http.request.complete` связывается с ответом по X-Request-Id и
X-Trace-Id. Worker log `job.complete` связывается с реальным CONSUMER span
по trace_id/span_id. В логах есть timestamp/severity/event и service resource.
Технические job/attachment ID разрешены только в traces/logs синтетического
стенда; метрики не используют идентификаторы объектов.
SDK экспортирует только явные allowlist-события, без текстов сообщений,
паролей, токенов, SQL и exception text. Uvicorn access log выключен.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Grafana доступна только через loopback http://localhost:3000 (учебный
anonymous Viewer); dashboard **Messenger — lab6**, три datasource заданы
provisioning автоматически. Prometheus/Loki/Tempo на loopback
9090/3100/3200 нужны также для воспроизводимых проверок.
OTLP порты доступны только внутри сети Compose.
Collector применяет memory_limiter (192 MiB), batch (не больше 512 records),
очереди exporters по 256 batches и retry до 30 секунд. У SDK фоновые очереди
traces/logs по 512 records, batches по 64, HTTP timeout 2 секунды, период
метрик 5 секунд. Ни API request, ни job не ожидают сетевой экспорт.
Зависимости telemetry не входят в readiness приложения.
Источники: [OTel SDK](https://opentelemetry-python.readthedocs.io/en/stable/sdk/),
[Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
[Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
Реализация и документация подготовлены с помощью Codex.
## Уровень 4: причинная связь и насыщение
Миграция 006 добавляет nullable W3C carrier в attachments/outbox без изменения
публичных схем. API сохраняет carrier в той же транзакции, что и metadata/outbox;
publisher извлекает его, создаёт PRODUCER span, записывает новый carrier в
envelope. Worker создаёт CONSUMER span с этим parent, сохраняет consumer
context при claim для lease recovery. Следующая попытка получает новый span ID
в той же трассе. Operator replay/redeliver сохраняет причинный контекст.
Транзакционная граница outbox и семантика at least once остаются прежними.
Внутренние spans: `db.acquire`, `db.query`, `db.commit`, `db.health`,
`s3.PutObject/GetObject/DeleteObject/HeadObject`, `image.generate`.
SQL span содержит только тип операции, exception — только имя класса.
GetObject измеряет получение headers; чтение streaming body продолжается
внутри вызывающей операции, поэтому этот span не является полной download latency.
| Prometheus metric | Смысл и единица |
| --- | --- |
| `messenger_queue_depth` | queued + processing в PostgreSQL, jobs; включает outbox и delayed retry |
| `messenger_queue_oldest_age_seconds` | возраст старейшей незавершённой job, секунды |
| `messenger_queue_sample_age_seconds` | возраст последнего успешного наблюдения, секунды |
| `messenger_db_pool_used` / `waiting` / `capacity` | занятые соединения / acquisition requests in progress / лимит соединений |
| `messenger_worker_busy` / `capacity` | занятая обработка / доступная ёмкость (один consumer, capacity=1) |
| `messenger_process_cpu_seconds_total` | CPU time родительского процесса, секунды |
| `messenger_process_memory_bytes` | текущая RSS родительского процесса, байты |
| `messenger_attachment_time_to_ready_seconds_*` | histogram от created_at принятого attachment до успешного COMMIT ready |
Waiting включает создание/проверку соединения, а не только очередь при
исчерпанном pool. Метрики CPU/RSS не включают подпроцесс декодера; контейнер
дополнительно ограничен Docker. Backlog измеряет publisher независимо от worker,
sample age позволяет обнаружить устаревшее наблюдение при отказе БД.
`WORKER_CONCURRENCY` из логических рекомендаций skeleton не используется:
реальный worker последователен, prefetch=1; scale worker потребует отдельной проверки.
Dashboard фильтрует worker resources по heartbeat не старше 15 секунд,
затем суммирует busy/capacity. Так старые process resources после restart
не завышают ёмкость и не показывают мёртвый busy worker. Collector удаляет
не обновляемые серии через 5 минут; heartbeat делает эту задержку явной.
Пример p95 API в секундах (для p50/p99 меняется только первый аргумент):
```promql
histogram_quantile(0.95,
sum by (le) (rate(messenger_http_duration_seconds_bucket{
service_name="messenger-api",route!~"/health/.*"}[5m])))
```
Сначала объединяются buckets реплик с одинаковыми границами, потом вычисляется
percentile. Buckets SDK: 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5,
5, 10, 30, 60, 120, +Inf секунд. Это приближённая оценка, не среднее p95.
Все запросы, легенды и единицы находятся в импортируемом dashboard JSON.
SLI доступности = 1 − (5xx / все бизнес-ответы), health исключены, 4xx не
считаются отказом сервера. Учебный SLO: ≥99% за скользящие 5 минут.
Дополнительный SLO: ≥95% успешных ready-завершений за ≤60 секунд в окне 5 минут
по histogram; он условен по завершениям, pending/failed отдельно контролируются
backlog/errors. При отсутствии трафика или завершений выполнение SLO не доказано.
Alert high error rate: >5% за 1 минуту непрерывно 10 секунд. Queue lag:
age >15 секунд при depth >0 и свежем sample <10 секунд непрерывно 10 секунд.
Оценка каждые 5 секунд; короткие пороги выбраны для защиты, а не production SLA.
Правила — `infra/prometheus/alerts.yaml`, диагностика —
[RUNBOOK](infra/otel/RUNBOOK.md). Доставка уведомлений наружу не используется.
## Конфигурация и границы устойчивости
`.env.example` содержит логические рекомендации skeleton. Реально Compose
передаёт DATABASE_URL_FILE, CACHE_URL_FILE, BROKER_URL_FILE и S3 *_FILE через
secrets; сессии кэшируются в Valkey, источник истины PostgreSQL. DB pool
10+5 соединений на процесс, pool/connect timeout 3 секунды, statement timeout
5 секунд. Upload: ≤10 MiB и ≤40M pixels; spool RAM 1 MiB на upload,
не более 8 concurrent uploads на API. Worker: image timeout 30 секунд,
total job timeout 90, lease 120, максимум 5 попыток, exponential backoff+jitter.
Thumbnail JPEG quality=85, crop quality=85, optimized WebP quality=80,
metadata EXIF удаляется после применения orientation; original сохраняется побайтно.
Порты, API_REPLICAS, SESSION_TTL_SECONDS, IDEMPOTENCY_TTL_SECONDS,
JOB_* и RETRY_BASE_SECONDS можно переопределить через `.env` по `compose.yaml`.
Для локальных OTLP-ресурсов выбраны имена messenger-api/worker/publisher;
endpoint и resource заданы явно в Compose. `OTEL_SDK_DISABLED=true` отключает SDK.
API readiness зависит от PostgreSQL и drain; S3 влияет на вложения, а broker,
worker и telemetry не выключают весь синхронный путь сообщений.
Две API-реплики выдерживают остановку одной. Postgres/S3/broker volumes
переживают перезапуск контейнеров; outbox, publisher confirms, ack после commit,
идемпотентная обработка и fencing token защищают от окон потери/дублей.
Один Docker-хост, один LB, DB, S3, broker и Collector — точки отказа.
Это не кластер с независимыми узлами и не backup; отказ диска/хоста здесь
не покрыт. HTTP разрешён только для локального синтетического стенда.
Telemetry экспортируется фоново с ограниченной памятью. При недоступности
backend Collector повторяет до 30 секунд, затем отбрасывает batch; при заполнении
очереди возможны дополнительные потери. SDK counters export_failures и
dropped_records отражают неуспешные exports; overflow SDK queue отдельно
не подсчитывается. Collector send_failed — число неуспешных попыток с records,
не число уникально потерянных spans. Отказ наблюдаемости не должен останавливать
бизнес-записи. Семидневный retention задан для всех трёх сигналов;
Prometheus дополнительно ограничен 512 MB, остальные backend ограничены
контейнерами и свободным местом учебного Docker-диска.
## Методика и результаты проверки
Проверки выполнены 21.11.2026 на Windows/Docker Desktop 29.3.0, Linux
containers, Intel Core Ultra 5 125U. Docker видит 14 CPU и 7 965 700 096 bytes
RAM. Лимиты ресурсов каждого сервиса закреплены в Compose; это один хост.
Телеметрия включена, sampling полный; данные синтетические.
Результаты сняты из рабочего дерева уровня 4 после коммита `d0a062e` и
включены в третий коммит. История: `git log -4 --oneline`.
Профиль нагрузки: публичный `tests/load.py`, closed loop POST messages,
concurrency=4, длительность 20 секунд, один прогон, без автоматических retry.
Отдельный длительный прогрев не выполнялся; до нагрузки стенд работал
несколько минут и прошёл smoke. Создано **1645 сообщений**, ошибок **0**,
успешные RPS **81.154**, p50 **38.796 ms**, p95 **114.354 ms**, p99 **194.742 ms**.
Это client-side latency успешных записей; ошибки учитываются отдельно,
closed loop имеет coordinated omission. Этот прогон не доказывает максимальную
пропускную способность или выполнение SLO за длительный период.
Данные: [load-summary.json](evidence/load-summary.json),
[команда/stdout](evidence/load.txt).
За тот же интервал Prometheus зафиксировал максимальную RSS одного API
128 040 960 bytes, worker 89 513 984, publisher 89 305 088. Максимальный
15-секундный rate CPU одного API 0.389 CPU cores, worker 0.001,
publisher 0.021. Это дискретные samples родительских процессов, не суммарные
CPU/RAM всех контейнеров; image worker подпроцесс не включён.
[Сырые запросы и range samples](evidence/load-resources.json).
| Критерий | Команда / сценарий | Доказательство |
| --- | --- | --- |
| Контракт и публичный smoke с telemetry | `python -X utf8 scripts/test_lab6.py` | [make-check-test.txt](evidence/make-check-test.txt) |
| Полная OpenAPI / JSON Schema | venv Python, `scripts/validate_openapi.py` | [validate.txt](evidence/validate.txt) |
| SDK, корреляция, W3C, отсутствие утечек | `student-tests/observability_unit.py` в API image | [5 tests](evidence/observability-unit.txt) |
| Метрики HTTP / jobs, реальные API и worker logs/traces | `scripts/observe_lab6.py --baseline` | [baseline](evidence/observability-baseline.json), [trace](evidence/baseline-api-trace.json), [API logs](evidence/baseline-api-logs.json), [worker logs](evidence/baseline-worker-logs.json) |
| Устойчивый upload, изображения и лимиты lab5 | `scripts/test_lab5.py --quick` | [images](evidence/images.txt), [decoder limits](evidence/limits.txt) |
| Сохранность после restart | public persistence с `scripts/restart.sh` | [persistence.txt](evidence/persistence.txt) |
| Смена API-реплики без потери сессии/данных | public failover с `scripts/stop-one.sh` | [failover.txt](evidence/failover.txt) |
| Outbox gap, crash до commit / до ack, retry exhaustion, replay и duplicate no-op | `scripts/test_lab5.py` (полный прогон с telemetry) | [recovery.txt](evidence/recovery.txt) |
Метрики/логи/трассы из evidence запрошены у настоящих backend; это не
статические фиктивные примеры. JSON dashboard и provisioning находятся в Git,
а не только в скриншоте. Содержимое вывода и JSON проверяется на текущие
синтетические secret markers и запрещённые telemetry keys/metric labels.
Собственная проверка oversized upload отправляет заведомо чрезмерный
Content-Length и проверяет ранний 413 до передачи body: urllib может потерять
ранний ответ, продолжая писать уже отклонённые 10 MiB. Публичные тесты не менялись.
## Контролируемые отказы и диагностика
Общие данные: [observability-incidents.json](evidence/observability-incidents.json).
Сценарии восстанавливают остановленные сервисы в `finally`; worker запускается
через точные Docker container IDs, чтобы Compose не поднял одновременно
искусственно остановленную зависимость. Перед проверкой alerts ожидаются
пустой backlog и inactive alert, чтобы не принять старое срабатывание за новое.
| Воздействие | Наблюдение и причина | Восстановление |
| --- | --- | --- |
| Worker остановлен, приняты четыре upload (09:03:12–09:04:10 UTC) | Растут depth/oldest age при свежем publisher sample. `MessengerQueueLag` activeAt 09:03:35, затем firing. API продолжает выдавать 202; причина — отсутствие consumer, не длительность HTTP | После старта все четыре ready, оригиналы побайтно и SHA-256 совпали, thumbnail 256×160. Alert resolved в 09:04:10; полный сценарий 58 секунд |
| PostgreSQL остановлен (08:58:31–09:00:26 UTC) | 18 прямых бизнес-запросов в две реальные API-реплики дали 503 unavailable. `MessengerHighErrorRate` activeAt 08:58:55, firing. DB error span и связанный log указывают на зависимость | БД запущена с тем же volume, readiness восстановился, сообщение записано. Alert resolved 09:00:25 после выхода ошибок из минутного окна |
| Tempo и Loki остановлены на 65 секунд | 339 успешных записей сообщений; exporter send_failed растёт, очереди не превышают capacity=256, максимальная Collector RSS 204 MiB < контейнерного лимита 256 MiB | После старта и опустошения очередей новый upload снова имеет связанный API/worker trace и logs. Проверены все 339 контрольных trace ID и SERVER spans: отсутствующих 0; отсутствующих контрольных API logs 5 |
Для первого инцидента отдельный чистый прогон:
[worker-lag-isolated.txt](evidence/worker-lag-isolated.txt).
Для БД: [trace](evidence/incident-db-trace.json) и
[logs](evidence/incident-db-logs.json). Запросы идут прямо в API-контейнеры,
потому что после отрицательного readiness HAProxy сам возвращает 503;
такой ответ LB не следует выдавать за ошибку, измеренную SDK API.
Значение alert `activeAt` — начало pending, не точное время перехода firing;
само состояние firing сохранено в JSON.
Потери измерены по известным request/trace IDs контрольных бизнес-записей
после восстановления и опустошения очередей, с дополнительной проверкой
наличия SERVER span. Это количество отсутствующих контрольных событий,
а не всех внутренних spans системы. Отсутствующий log при сохранённой
бизнес-записи показывает допустимую потерю наблюдений при ограниченном retry.
[Сырые контрольные logs](evidence/telemetry-loss-control-logs.json) и список IDs
с результатами поиска находятся в evidence. Нулевая потеря этих traces
не означает гарантию сохранности telemetry при любом отказе.
Отдельно воспроизведены S3 outage, retry и lease recovery:
[summary](evidence/observability-retry.json),
[полная трасса](evidence/retry-api-trace.json),
[worker logs](evidence/retry-worker-logs.json).
Upload `8da6897d-7ad0-4a47-92a1-63e8820c6fb3` имеет общий trace
`94af84b9ec1663416ac05526995fbd91`; видны error попытки 1/2,
duplicate redelivery и success попытки 3. Проверен путь parent каждого
consumer через producer до исходного SERVER span, а также DB/S3 children.
Оригинал 111817 bytes сохранился, thumbnail имеет размер 256×160.
Lease 120 секунд объясняет длительное восстановление после прерывания
worker: бюджет smoke 60 секунд относится к свободному исправному worker,
не к восстановлению аварийной попытки.
После массового чтения накопленных трасс настройки Tempo уточнены:
GOMEMLIMIT=256 MiB, контейнер 512 MiB / 1 CPU. Итоговая конфигурация сохраняет
ограниченную память и выдержала повторный поиск всех контрольных SERVER spans.
Нагрузочный прогон и первоначальный backend outage сняты до этого изменения;
бизнес-код и настройки Collector при этом не менялись.
Трек оценки 5 не заявлен. Нет сравнительного исследования sampling,
доказанной production ёмкости или отказоустойчивого кластера хранилищ.
View File
+114
View File
@@ -0,0 +1,114 @@
"""Low-cardinality saturation/backlog signals and safe dependency spans (lab6/4)."""
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
import os
import resource
import time
from opentelemetry import metrics
from opentelemetry.metrics import Observation
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.telemetry import span
meter = metrics.get_meter("messenger")
waiting = 0
busy = 0
snapshot: dict[str, float] = {}
ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
retries = meter.create_counter("messenger.jobs.retries", unit="{retry}")
class ObservedSession(AsyncSession):
async def execute(self, statement, params=None, **kwargs):
global waiting
# Number of acquisition requests in progress (includes new connects).
# Acquisition is separate from query execution to diagnose pool waits.
waiting += 1
try:
with span("db.acquire", {"db.system": "postgresql"}):
await self.connection()
finally:
waiting -= 1
operation = str(statement).lstrip().split(None, 1)[0].upper()
if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "SET"}:
operation = "OTHER"
with span("db.query", {"db.system": "postgresql", "db.operation": operation}):
return await super().execute(statement, params, **kwargs)
async def commit(self):
with span("db.commit", {"db.system": "postgresql"}):
return await super().commit()
def install_gauges(engine) -> None:
pool = engine.sync_engine.pool
meter.create_observable_gauge("messenger.db.pool.used",
callbacks=[lambda _: [Observation(pool.checkedout())]])
meter.create_observable_gauge("messenger.db.pool.waiting",
callbacks=[lambda _: [Observation(waiting)]])
meter.create_observable_gauge("messenger.db.pool.capacity",
callbacks=[lambda _: [Observation(pool.size() + pool._max_overflow)]])
meter.create_observable_gauge("messenger.worker.busy",
callbacks=[lambda _: [Observation(busy)]])
capacity = 1 if os.getenv("OTEL_SERVICE_NAME") == "messenger-worker" else 0
meter.create_observable_gauge("messenger.worker.capacity",
callbacks=[lambda _: [Observation(capacity)]])
if capacity:
meter.create_observable_gauge("messenger.worker.heartbeat", unit="s",
callbacks=[lambda _: [Observation(time.time())]])
meter.create_observable_counter("messenger.process.cpu", unit="s",
callbacks=[lambda _: [Observation(time.process_time())]])
def rss(_):
# Linux /proc resident pages, current RSS (not ru_maxrss high watermark).
try:
with open("/proc/self/statm", encoding="ascii") as statm:
value = int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
except OSError:
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
return [Observation(value)]
meter.create_observable_gauge("messenger.process.memory", unit="By", callbacks=[rss])
if os.getenv("OTEL_SERVICE_NAME") == "messenger-publisher":
for name, unit, key in (("depth", "", "depth"),
("oldest_age", "s", "oldest_age"),
("sample_age", "s", "sample_age")):
def observe(_, key=key):
if "sampled_at" not in snapshot:
return []
value = (time.monotonic() - snapshot["sampled_at"]
if key == "sample_age" else snapshot[key])
return [Observation(value, {"queue": "attachments"})]
meter.create_observable_gauge("messenger.queue." + name, unit=unit,
callbacks=[observe])
async def backlog_loop(session_factory) -> None:
# Authoritative logical queue = queued + processing, including durable
# outbox and delayed retries. Remains observable when worker is stopped.
while True:
try:
async with session_factory() as db:
result = await db.execute(text("""
SELECT count(*) AS depth,
COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS oldest_age
FROM attachments WHERE status IN ('queued','processing')
"""))
row = result.mappings().one()
snapshot.update(depth=float(row["depth"]), oldest_age=float(row["oldest_age"]),
sampled_at=time.monotonic())
except asyncio.CancelledError:
raise
except Exception:
# Retain last sample; sample_age makes stale observations explicit.
pass
await asyncio.sleep(2)
def observe_ready(row) -> None:
elapsed = (datetime.now(timezone.utc) - row["created_at"]).total_seconds()
ready_duration.record(max(0, elapsed), {"job_type": "attachment.process.v1"})
+467
View File
@@ -0,0 +1,467 @@
"""Attachments: durable upload to staging, queued metadata + outbox (lab5).
Upload, in this order (all checks before the bytes are stored):
1. filename (400) -> chat membership (404) -> Content-Type header (415)
-> Content-Length (413 without reading the body);
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
3. the body is read in chunks into a SpooledTemporaryFile (only
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
4. content validation in a thread (app.images): format by magic bytes,
pixel limit from the header (413), integrity (415);
5. upload intent COMMIT -> staging S3 PUT -> one transaction "delete intent +
insert queued attachment + outbox" -> 202. Worker produces all variants.
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
in the background):
- PUT failed or timed out: the outcome is unknown - S3 may still finish
writing the object AFTER our DELETE. So the object is deleted best
effort, but the intent is KEPT; the janitor deletes the object again
after UPLOAD_GRACE_SECONDS and only then the intent;
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
and only the answer was lost. Under the intent's row lock: no intent ->
the row is committed, keep the object; else delete object, then intent;
- the process died after PUT: the intent stays; the janitor deletes the
object and the intent after UPLOAD_GRACE_SECONDS.
So an object without a row exists only while its intent exists, and an
object of a committed row is never deleted.
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import random
import tempfile
import unicodedata
from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import quote
from uuid import UUID, uuid4
from fastapi import Request
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import ClientDisconnect
from app import repositories
from app.config import settings
from app.db import SessionLocal
from app.errors import (
ApiError,
Conflict,
InvalidRequest,
NotFound,
PayloadTooLarge,
RateLimited,
Unavailable,
UnsupportedMediaType,
)
from app.images import InvalidImage, TooManyPixels, validate_image
from app.storage import ObjectMissing, StorageUnavailable, storage
logger = logging.getLogger("messenger.attachments")
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
# Characters that can turn a display name into a path or hide its real
# extension: separators, NUL/control characters, bidi overrides etc.
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
# Full integrity checks are CPU work too. In particular, progressive JPEG
# retains decoder coefficients even with draft(); never run eight at once.
validation_slot = asyncio.Semaphore(1)
# Strong references to background compensation tasks (asyncio keeps only
# weak ones); if the process dies first, the intent stays for the janitor.
_compensations: set[asyncio.Task] = set()
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": row["id"],
"chat_id": row["chat_id"],
"owner_id": row["owner_id"],
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [
{key: value for key, value in variant.items() if key != "object_key"}
for variant in _variants(row)
],
"error_code": row["error_code"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
def _variants(row: dict[str, Any]) -> list[dict[str, Any]]:
value = row.get("variants") or []
return json.loads(value) if isinstance(value, str) else value
def validate_filename(filename: str) -> str:
if not 1 <= len(filename) <= 255 or not filename.strip():
raise InvalidRequest("filename must be 1-255 characters, not blank.")
if "/" in filename or "\\" in filename:
raise InvalidRequest("filename must not contain path separators.")
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
raise InvalidRequest("filename must not contain control characters.")
if filename.strip(". ") == "":
raise InvalidRequest("filename must not be a relative path.")
return filename
def declared_content_type(request: Request) -> str:
raw = request.headers.get("Content-Type", "")
media_type = raw.split(";", 1)[0].strip().lower()
if media_type not in ALLOWED_CONTENT_TYPES:
raise UnsupportedMediaType(
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
)
return media_type
def _too_large(message: str) -> PayloadTooLarge:
error = PayloadTooLarge(message)
# The rest of the body is not read: the connection cannot be reused.
error.headers["Connection"] = "close"
return error
def check_declared_length(request: Request) -> None:
raw = request.headers.get("Content-Length")
if raw is None:
return # chunked: the limit is enforced while reading
if not raw.isdigit():
raise InvalidRequest("Invalid Content-Length.")
if int(raw) > settings.max_upload_bytes:
raise _too_large("File is larger than 10 MiB.")
async def _receive_body(request: Request, spool) -> tuple[int, str]:
"""Stream the body into the spool; return (size, sha256)."""
digest = hashlib.sha256()
size = 0
try:
async with asyncio.timeout(settings.upload_timeout_seconds):
async for chunk in request.stream():
if not chunk:
continue
size += len(chunk)
if size > settings.max_upload_bytes:
raise _too_large("File is larger than 10 MiB.")
digest.update(chunk)
spool.write(chunk)
except TimeoutError:
raise ApiError(
400,
"invalid_request",
"Upload was not completed in time.",
{"Connection": "close"},
) from None
except ClientDisconnect:
# Nobody will read this answer; nothing was stored anywhere.
logger.info("upload interrupted by the client after %d bytes", size)
raise InvalidRequest("Upload was interrupted.") from None
return size, digest.hexdigest()
async def _acquire_slot() -> None:
try:
async with asyncio.timeout(settings.upload_slot_wait_seconds):
await upload_slots.acquire()
except TimeoutError:
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
async def _discard_after_failed_put(object_key: str) -> None:
"""Best effort only; the intent stays for the janitor (late PUT)."""
try:
await storage.delete(object_key)
except StorageUnavailable:
logger.warning("compensation: S3 delete failed, janitor will retry")
async def _compensate_failed_commit(object_key: str) -> None:
"""Delete the object only if its upload really did not commit."""
try:
async with SessionLocal() as db:
row = await repositories.lock_upload_intent(db, object_key)
if row is None:
# COMMIT succeeded after all (or the janitor finished it).
await db.rollback()
return
if not row["committed"]:
try:
await storage.delete(object_key)
except StorageUnavailable:
await db.rollback()
logger.warning("compensation: S3 delete failed, janitor will retry")
return
await repositories.delete_upload_intent(db, object_key)
except Exception as exc:
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
async def upload_attachment(
request: Request,
db: AsyncSession,
*,
chat_id: UUID,
filename: str,
current_user: dict[str, Any],
) -> dict[str, Any]:
validate_filename(filename)
chat = await repositories.get_accessible_chat(
db, chat_id=chat_id, user_id=current_user["id"]
)
if chat is None:
raise NotFound()
# Do not keep a pooled connection idle in a transaction while the
# client uploads up to 10 MiB.
await db.rollback()
content_type = declared_content_type(request)
check_declared_length(request)
await _acquire_slot()
try:
with tempfile.SpooledTemporaryFile(
max_size=settings.upload_spool_memory_bytes
) as spool:
size, sha256 = await _receive_body(request, spool)
if size == 0:
raise InvalidRequest("Request body with image bytes is required.")
try:
async with validation_slot:
validation = asyncio.create_task(asyncio.to_thread(
validate_image, spool, content_type, settings.max_image_pixels
))
try:
info = await asyncio.shield(validation)
except asyncio.CancelledError:
# A thread cannot be killed: keep its spool and the
# validation slot alive until it has really stopped.
await asyncio.gather(validation, return_exceptions=True)
raise
except TooManyPixels:
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
except InvalidImage as exc:
raise UnsupportedMediaType(str(exc)) from None
return await _store(
db,
spool=spool,
chat_id=chat_id,
owner_id=current_user["id"],
filename=filename,
content_type=info.content_type,
size=size,
sha256=sha256,
width=info.width,
height=info.height,
)
finally:
upload_slots.release()
async def _store(
db: AsyncSession,
*,
spool,
chat_id: UUID,
owner_id: UUID,
filename: str,
content_type: str,
size: int,
sha256: str,
width: int,
height: int,
) -> dict[str, Any]:
attachment_id = uuid4()
# Server-generated random key, unrelated to the filename and to the
# public attachment ID.
object_key = f"staging/{uuid4().hex}"
await repositories.create_upload_intent(
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
)
try:
await storage.put(object_key, spool, size, content_type)
except StorageUnavailable as exc:
logger.warning("S3 PUT failed: %s", exc)
_in_background(_discard_after_failed_put(object_key))
raise Unavailable() from None
if settings.fault_injection == "crash_after_s3_put":
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
os._exit(1)
try:
if settings.fault_injection == "db_error_after_s3_put":
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
row = await repositories.finalize_attachment(
db,
attachment_id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
content_type=content_type,
size_bytes=size,
sha256=sha256,
width=width,
height=height,
object_key=object_key,
)
except Exception as exc:
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
_in_background(_compensate_failed_commit(object_key))
raise Unavailable() from None
if row is None:
# The janitor already claimed this intent (upload slower than the
# grace period): the object is (being) deleted, report a failure.
logger.warning("upload intent expired before COMMIT")
raise Unavailable()
return _to_attachment(row)
async def get_attachment(
db: AsyncSession,
*,
attachment_id: UUID,
current_user: dict[str, Any],
) -> dict[str, Any]:
row = await repositories.get_accessible_attachment(
db, attachment_id=attachment_id, user_id=current_user["id"]
)
if row is None:
raise NotFound()
return _to_attachment(row)
async def open_attachment_content(
db: AsyncSession,
*,
attachment_id: UUID,
variant: str,
current_user: dict[str, Any],
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
row = await repositories.get_accessible_attachment(
db, attachment_id=attachment_id, user_id=current_user["id"]
)
if row is None:
raise NotFound()
await db.rollback()
if row["status"] != "ready":
raise Conflict("Attachment content is not ready.")
content = row
if variant != "original":
content = next((item for item in _variants(row) if item["name"] == variant), None)
if content is None:
raise NotFound()
try:
chunks, length = await storage.open_download(content["object_key"])
except StorageUnavailable as exc:
logger.warning("S3 GET failed: %s", exc)
raise Unavailable() from None
except ObjectMissing:
# Metadata says "ready" but the bytes are gone: an inconsistency,
# not a user error (logged as ERROR).
logger.error("object of attachment %s is missing in S3", attachment_id)
raise
if length != content["size_bytes"]:
logger.error("object of attachment %s has a wrong size", attachment_id)
raise ObjectMissing(content["object_key"])
headers = {
"Content-Type": content["content_type"],
"Content-Length": str(length),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
"ETag": f'"{content["sha256"]}"',
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
}
return chunks, headers
def _in_background(compensation) -> None:
"""Do not keep the client waiting for compensation: with S3 down each
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
anyway."""
task = asyncio.create_task(compensation)
_compensations.add(task)
task.add_done_callback(_compensations.discard)
# --- Janitor ----------------------------------------------------------------
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
"""Delete objects of upload intents older than the grace period.
One intent per transaction: lock it (SKIP LOCKED), delete the object in
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
stop: the intent stays and the next run retries.
"""
cleaned = 0
for _ in range(limit):
async with SessionLocal() as db:
row = await repositories.lock_stale_intent(db, grace_seconds)
if row is None:
await db.rollback()
break
if not row["committed"]:
try:
await storage.delete(row["object_key"])
except StorageUnavailable:
await db.rollback()
logger.warning("janitor: S3 unavailable, will retry")
break
await repositories.delete_upload_intent(db, row["object_key"])
cleaned += 1
logger.warning(
"janitor: removed abandoned upload %s (object %s)",
row["object_key"],
"kept, row committed" if row["committed"] else "deleted",
)
return cleaned
async def janitor_loop() -> None:
while True:
# Jitter: replicas do not all wake up at the same moment.
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
try:
await cleanup_stale_uploads(settings.upload_grace_seconds)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning("janitor run failed: %s", type(exc).__name__)
+179
View File
@@ -0,0 +1,179 @@
"""Valkey session cache.
PostgreSQL is the source of truth for sessions. Valkey only keeps a short
lived copy of "token hash -> user" to avoid a SQL lookup on every request:
- key: messenger:session:<sha256(token)>, the raw token is never stored;
- value TTL: min(SESSION_CACHE_TTL_SECONDS, remaining session lifetime),
and expires_at is checked again on every read;
- logout writes a "revoked" tombstone (SET ... EX, overwrite), while cache
population uses SET ... NX, so a concurrent request cannot resurrect a
revoked session in the cache;
- any Valkey error or timeout is a cache miss: requests go to PostgreSQL and
the cache is skipped for CACHE_RETRY_AFTER_SECONDS. After such an error,
and on every API process start, the cache is flushed before its next use,
because an invalidation may have been lost while Valkey was unreachable
(also by a previous API process).
"""
from __future__ import annotations
import asyncio
import json
import logging
import math
import time
from datetime import datetime, timezone
from typing import Any
from uuid import UUID
from redis.asyncio import ConnectionPool, Redis
from redis.exceptions import RedisError
from app.config import settings
logger = logging.getLogger("messenger.cache")
KEY_PREFIX = "messenger:session:"
REVOKED = "revoked"
class Revoked:
"""Marker returned when the cache knows the session was revoked."""
REVOKED_SESSION = Revoked()
def _key(token_hash: str) -> str:
return KEY_PREFIX + token_hash
def _serialize(user: dict[str, Any], expires_at: datetime) -> str:
return json.dumps(
{
"id": str(user["id"]),
"username": user["username"],
"display_name": user["display_name"],
"created_at": user["created_at"].isoformat(),
"expires_at": expires_at.isoformat(),
},
separators=(",", ":"),
)
def _deserialize(raw: str) -> tuple[dict[str, Any], datetime]:
payload = json.loads(raw)
user = {
"id": UUID(payload["id"]),
"username": payload["username"],
"display_name": payload["display_name"],
"created_at": datetime.fromisoformat(payload["created_at"]),
}
return user, datetime.fromisoformat(payload["expires_at"])
class SessionCache:
def __init__(self, url: str | None) -> None:
self._client: Redis | None = None
self._unavailable_until = 0.0
# A process never trusts a cache it has not flushed itself: an
# earlier process may have lost an invalidation (logout while
# Valkey was unreachable) and that knowledge died with it.
# Cleared only after a successful FLUSHDB.
self._needs_flush = True
if url:
pool = ConnectionPool.from_url(
url,
max_connections=settings.cache_max_connections,
socket_timeout=settings.cache_timeout_seconds,
socket_connect_timeout=settings.cache_timeout_seconds,
decode_responses=True,
)
self._client = Redis(connection_pool=pool)
@property
def enabled(self) -> bool:
return self._client is not None
async def _run(self, operation):
"""Run one cache operation; return (ok, result)."""
if self._client is None:
return False, None
if time.monotonic() < self._unavailable_until:
return False, None
try:
async with asyncio.timeout(settings.cache_timeout_seconds * 2):
if self._needs_flush:
await self._client.flushdb()
self._needs_flush = False
logger.warning("session cache flushed before use")
return True, await operation(self._client)
except (RedisError, OSError, TimeoutError) as exc:
self._unavailable_until = (
time.monotonic() + settings.cache_retry_after_seconds
)
self._needs_flush = True
logger.warning(
"session cache unavailable, falling back to PostgreSQL: %s",
type(exc).__name__,
)
return False, None
async def get(
self,
token_hash: str,
) -> dict[str, Any] | Revoked | None:
ok, raw = await self._run(lambda client: client.get(_key(token_hash)))
if not ok or raw is None:
return None
if raw == REVOKED:
return REVOKED_SESSION
try:
user, expires_at = _deserialize(raw)
except (ValueError, KeyError, TypeError):
return None
if expires_at <= datetime.now(timezone.utc):
return None
return user
async def put(
self,
token_hash: str,
user: dict[str, Any],
expires_at: datetime,
) -> None:
remaining = (expires_at - datetime.now(timezone.utc)).total_seconds()
ttl = min(settings.session_cache_ttl_seconds, math.floor(remaining))
if ttl < 1:
return
value = _serialize(user, expires_at)
await self._run(
lambda client: client.set(_key(token_hash), value, ex=ttl, nx=True)
)
async def revoke(self, token_hash: str) -> None:
await self._run(
lambda client: client.set(
_key(token_hash),
REVOKED,
ex=settings.session_cache_ttl_seconds,
)
)
async def close(self) -> None:
if self._client is not None:
await self._client.aclose()
session_cache = SessionCache(settings.cache_url)
+180
View File
@@ -0,0 +1,180 @@
from dataclasses import dataclass
from pathlib import Path
import os
import socket
def read_value(name: str, default: str | None = None) -> str:
file_name = os.getenv(f"{name}_FILE")
if file_name:
return Path(file_name).read_text(encoding="utf-8").strip()
value = os.getenv(name, default)
if value is None:
raise RuntimeError(f"{name} is required.")
return value
def read_positive_int(name: str, default: str) -> int:
value = int(os.getenv(name, default))
if value < 1:
raise RuntimeError(f"{name} must be positive.")
return value
def read_positive_float(name: str, default: str) -> float:
value = float(os.getenv(name, default))
if value <= 0:
raise RuntimeError(f"{name} must be positive.")
return value
@dataclass(frozen=True)
class Settings:
app_host: str
app_port: int
database_url: str
session_ttl_seconds: int
# PostgreSQL connection pool and timeouts.
db_pool_size: int
db_max_overflow: int
db_pool_timeout_seconds: float
db_connect_timeout_seconds: float
db_statement_timeout_ms: int
# Valkey is a session cache; PostgreSQL stays the source of truth.
cache_url: str | None
session_cache_ttl_seconds: int
cache_timeout_seconds: float
cache_max_connections: int
cache_retry_after_seconds: float
# Lab3: X-Instance-Id is the container hostname (short Docker ID), stable
# for the process lifetime; scripts/stop-one.sh maps it to a container.
instance_id: str
idempotency_ttl_seconds: int
# Graceful shutdown: readiness answers 503 for shutdown_drain_seconds so
# the LB drops this replica, then uvicorn closes the listener and waits
# for in-flight requests up to shutdown_timeout_seconds.
shutdown_drain_seconds: float
shutdown_timeout_seconds: int
# Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private
# S3 bucket; the API uses its own restricted credentials (not root).
s3_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
s3_object_prefix: str
s3_connect_timeout_seconds: float
s3_read_timeout_seconds: float
# Deadline of one whole S3 call (DNS + retries included), below the LB
# "timeout server" (10 s): a stopped S3 container makes Docker DNS hang
# ~8 s per lookup, which connect_timeout does not cover.
s3_operation_timeout_seconds: float
# PUT gets operation timeout + size / this rate: 10 MiB under load is
# slower than a HEAD. HAProxy allows 40 s for an upload POST.
s3_put_min_bytes_per_second: int
max_upload_bytes: int
max_image_pixels: int
# One upload may not take unlimited RAM/time: the body is spooled to a
# temporary file (only upload_spool_memory_bytes stay in RAM), receiving
# it is limited by upload_timeout_seconds, and at most
# max_concurrent_uploads uploads per replica run at once.
upload_spool_memory_bytes: int
upload_timeout_seconds: float
max_concurrent_uploads: int
upload_slot_wait_seconds: float
# Janitor: an upload intent older than upload_grace_seconds means the
# request died between S3 PUT and COMMIT; its object is deleted.
upload_grace_seconds: int
janitor_interval_seconds: float
# Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put
fault_injection: str
def load_settings() -> Settings:
session_store = os.getenv("SESSION_STORE", "postgres").lower()
if session_store not in {"postgres", "valkey", "redis"}:
raise RuntimeError("SESSION_STORE must be postgres, valkey or redis.")
cache_url = None
if session_store != "postgres":
cache_url = read_value("CACHE_URL")
return Settings(
app_host=os.getenv("APP_HOST", "0.0.0.0"),
app_port=int(os.getenv("APP_PORT", "8080")),
database_url=read_value("DATABASE_URL"),
session_ttl_seconds=read_positive_int("SESSION_TTL_SECONDS", "3600"),
db_pool_size=read_positive_int("DB_POOL_SIZE", "10"),
db_max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "5")),
db_pool_timeout_seconds=read_positive_float("DB_POOL_TIMEOUT_SECONDS", "3"),
db_connect_timeout_seconds=read_positive_float(
"DB_CONNECT_TIMEOUT_SECONDS", "3"
),
db_statement_timeout_ms=read_positive_int("DB_STATEMENT_TIMEOUT_MS", "5000"),
cache_url=cache_url,
session_cache_ttl_seconds=read_positive_int(
"SESSION_CACHE_TTL_SECONDS", "60"
),
cache_timeout_seconds=read_positive_float("CACHE_TIMEOUT_SECONDS", "0.25"),
cache_max_connections=read_positive_int("CACHE_MAX_CONNECTIONS", "20"),
cache_retry_after_seconds=read_positive_float(
"CACHE_RETRY_AFTER_SECONDS", "5"
),
instance_id=os.getenv("INSTANCE_ID") or socket.gethostname(),
idempotency_ttl_seconds=read_positive_int(
"IDEMPOTENCY_TTL_SECONDS", "86400"
),
shutdown_drain_seconds=read_positive_float(
"SHUTDOWN_DRAIN_SECONDS", "3"
),
shutdown_timeout_seconds=read_positive_int(
"SHUTDOWN_TIMEOUT_SECONDS", "8"
),
s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"),
s3_region=os.getenv("S3_REGION", "us-east-1"),
s3_bucket=read_value("S3_BUCKET", "messenger-attachments"),
s3_access_key_id=read_value("S3_ACCESS_KEY_ID"),
s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"),
s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"),
s3_connect_timeout_seconds=read_positive_float(
"S3_CONNECT_TIMEOUT_SECONDS", "2"
),
s3_read_timeout_seconds=read_positive_float(
"S3_READ_TIMEOUT_SECONDS", "5"
),
s3_operation_timeout_seconds=read_positive_float(
"S3_OPERATION_TIMEOUT_SECONDS", "6"
),
s3_put_min_bytes_per_second=read_positive_int(
"S3_PUT_MIN_BYTES_PER_SECOND", "1048576"
),
max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"),
max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"),
upload_spool_memory_bytes=read_positive_int(
"UPLOAD_SPOOL_MEMORY_BYTES", "1048576"
),
upload_timeout_seconds=read_positive_float(
"UPLOAD_TIMEOUT_SECONDS", "60"
),
max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"),
upload_slot_wait_seconds=read_positive_float(
"UPLOAD_SLOT_WAIT_SECONDS", "10"
),
upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"),
janitor_interval_seconds=read_positive_float(
"JANITOR_INTERVAL_SECONDS", "60"
),
fault_injection=os.getenv("FAULT_INJECTION", ""),
)
settings = load_settings()
+63
View File
@@ -0,0 +1,63 @@
from collections.abc import AsyncGenerator
from sqlalchemy.ext.asyncio import (
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from app.config import settings
from app.advanced import ObservedSession, install_gauges
from app.telemetry import span
database_url = settings.database_url.replace(
"postgresql://",
"postgresql+asyncpg://",
1,
)
# Bounded pool: at most db_pool_size + db_max_overflow connections per API
# process. Waiting for a free connection, connecting and every statement are
# limited, so an unavailable database produces a fast 503 instead of a hang.
engine = create_async_engine(
database_url,
pool_pre_ping=True,
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout_seconds,
pool_recycle=1800,
connect_args={
"timeout": settings.db_connect_timeout_seconds,
"command_timeout": settings.db_statement_timeout_ms / 1000 + 1,
"server_settings": {
"statement_timeout": str(settings.db_statement_timeout_ms),
"idle_in_transaction_session_timeout": "10000",
"application_name": "messenger-api",
},
},
)
SessionLocal = async_sessionmaker(
bind=engine,
class_=ObservedSession,
expire_on_commit=False,
autoflush=False,
)
install_gauges(engine)
async def get_db() -> AsyncGenerator[AsyncSession, None]:
async with SessionLocal() as session:
yield session
async def check_database_ready() -> bool:
try:
with span("db.health", {"db.system": "postgresql"}):
async with engine.connect() as connection:
await connection.exec_driver_sql("SELECT 1")
return True
except Exception:
return False
+61
View File
@@ -0,0 +1,61 @@
"""
FastAPI dependencies for Lab2:
- Bearer authentication;
- pagination query parameters.
"""
from typing import Optional
from fastapi import Depends, Query, Request, Security
from fastapi.security import HTTPAuthorizationCredentials, HTTPBasic, HTTPBearer
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import get_db
from app.errors import Unauthorized
from app.services import current_user_from_bearer_token
# Security schemes make the "Authorize" button in /docs send real
# Authorization headers. Errors keep the contract format: auto_error=False
# and our own 401 responses.
bearer_scheme = HTTPBearer(
scheme_name="Bearer",
auto_error=False,
description="Opaque session token from POST /api/v1/auth/sessions.",
)
class BasicForDocs(HTTPBasic):
"""Documents Basic auth; the header is parsed by app.security."""
async def __call__(self, request: Request) -> None: # type: ignore[override]
return None
basic_scheme = BasicForDocs(
scheme_name="Basic",
auto_error=False,
description="Only for POST /api/v1/auth/sessions.",
)
async def require_user(
credentials: Optional[HTTPAuthorizationCredentials] = Security(
bearer_scheme
),
db: AsyncSession = Depends(get_db),
) -> dict:
if credentials is None or not credentials.credentials.strip():
raise Unauthorized("Bearer")
return await current_user_from_bearer_token(
db,
credentials.credentials.strip(),
)
def pagination_params(
limit: int = Query(default=50, ge=1, le=100),
cursor: Optional[str] = Query(default=None, min_length=1),
) -> tuple[int, Optional[str]]:
return limit, cursor
+130
View File
@@ -0,0 +1,130 @@
import re
from fastapi import Request
from fastapi.responses import JSONResponse
from uuid import uuid4
from app.config import settings
from app.lifecycle import draining
class ApiError(Exception):
def __init__(
self,
status_code: int,
code: str,
message: str,
headers: dict[str, str] | None = None,
) -> None:
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers or {}
class InvalidRequest(ApiError):
def __init__(self, message: str = "Invalid request.") -> None:
super().__init__(400, "invalid_request", message)
class Unauthorized(ApiError):
def __init__(self, scheme: str = "Bearer") -> None:
if scheme == "Basic":
challenge = 'Basic realm="messenger", charset="UTF-8"'
else:
challenge = 'Bearer realm="messenger"'
super().__init__(
401,
"unauthorized",
"Authentication is required.",
{"WWW-Authenticate": challenge},
)
class NotFound(ApiError):
def __init__(self, message: str = "Object not found.") -> None:
super().__init__(404, "not_found", message)
class Conflict(ApiError):
def __init__(self, message: str = "Conflict.") -> None:
super().__init__(409, "conflict", message)
class Unavailable(ApiError):
def __init__(self) -> None:
super().__init__(503, "unavailable", "Service is unavailable.")
class PayloadTooLarge(ApiError):
def __init__(self, message: str = "Payload is too large.") -> None:
super().__init__(413, "payload_too_large", message)
class UnsupportedMediaType(ApiError):
def __init__(
self,
message: str = "Only PNG and JPEG images are accepted.",
) -> None:
super().__init__(415, "unsupported_media_type", message)
class RateLimited(ApiError):
def __init__(self, message: str, retry_after_seconds: int = 1) -> None:
super().__init__(
429,
"rate_limited",
message,
{"Retry-After": str(retry_after_seconds)},
)
# X-Request-Id from HAProxy (or the client) is reused only if it is short and
# printable; anything else is replaced, so it cannot inject into logs.
REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def incoming_request_id(request: Request) -> str:
value = request.headers.get("X-Request-Id", "")
if REQUEST_ID_PATTERN.fullmatch(value):
return value
return str(uuid4())
def request_id(request: Request) -> str:
return getattr(request.state, "request_id", str(uuid4()))
def common_headers(request: Request) -> dict[str, str]:
"""Headers of every API response, including errors (lab3: X-Instance-Id)."""
headers = {
"X-Request-Id": request_id(request),
"X-Instance-Id": settings.instance_id,
}
if draining.is_set():
# Ask HAProxy not to reuse the connection to a replica that is stopping.
headers["Connection"] = "close"
return headers
async def api_error_handler(
request: Request,
exc: ApiError,
) -> JSONResponse:
rid = request_id(request)
headers = {
**common_headers(request),
**exc.headers,
}
return JSONResponse(
status_code=exc.status_code,
content={
"error": {
"code": exc.code,
"message": exc.message,
"request_id": rid,
}
},
headers=headers,
)
+95
View File
@@ -0,0 +1,95 @@
"""Disposable, resource-limited image decoder. No app credentials/config.
Usage: python -m app.image_worker INPUT OUTPUT_DIR. Result is manifest.json.
All input comes from a trusted staging object selected by the worker in PG.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
from pathlib import Path
import sys
import warnings
def fit_size(width: int, height: int, limit: int) -> tuple[int, int]:
largest = max(width, height)
if largest <= limit:
return width, height
return max(1, width * limit // largest), max(1, height * limit // largest)
def convert(source: Path, directory: Path) -> list[dict]:
from PIL import Image, ImageOps
Image.MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(source) as opened:
if opened.format not in {"PNG", "JPEG"}:
raise ValueError("invalid_image")
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
raise ValueError("image_too_large")
opened.load()
oriented = ImageOps.exif_transpose(opened)
# Detach metadata and use fixed output modes. The original file
# itself is never rewritten; EXIF orientation affects variants only.
image = oriented.convert("RGBA" if "A" in oriented.getbands() or
"transparency" in oriented.info else "RGB")
oriented.close()
image.info.clear()
if image.mode == "RGBA":
rgb = Image.new("RGB", image.size, "white")
rgb.paste(image, mask=image.getchannel("A"))
else:
rgb = image
variants = []
def save(name, result, filename, content_type, **options):
path = directory / filename
result.info.clear()
result.save(path, **options)
data = path.read_bytes()
variants.append({"name": name, "filename": filename, "content_type": content_type,
"width": result.width, "height": result.height,
"size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
if result is not rgb and result is not image:
result.close()
save("thumbnail", rgb.resize(fit_size(*rgb.size, 256), Image.Resampling.LANCZOS),
"thumbnail.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
save("crop", ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS,
centering=(0.5, 0.5)),
"crop.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
save("optimized", image.resize(fit_size(*image.size, 1280), Image.Resampling.LANCZOS),
"optimized.webp", "image/webp", format="WEBP", quality=80, method=4)
if rgb is not image:
rgb.close()
image.close()
return variants
def main() -> int:
# Linux container limits are inherited by every native decoder. Parent
# enforces a wall timeout and kills/reaps this process on expiration.
if sys.platform == "linux":
import resource
memory = int(os.getenv("WORKER_MEMORY_MB", "768")) * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
cpu = math.ceil(float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))) + 1
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu))
try:
source, directory = Path(sys.argv[1]), Path(sys.argv[2])
manifest = convert(source, directory)
(directory / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
return 0
except MemoryError:
return 21
except Exception:
return 20
if __name__ == "__main__":
raise SystemExit(main())
+302
View File
@@ -0,0 +1,302 @@
"""Content-based validation of uploaded PNG/JPEG images.
The declared Content-Type is never trusted: the format is detected from the
first bytes and must match it. Dimensions are read from the header and the
pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file
cannot make the API allocate gigabytes.
- PNG is validated by our own streaming reader: critical chunks, CRC of every
chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces.
Scanline sizes and filter bytes (also in Adam7 passes) are checked without
retaining any row. Memory stays O(64 KiB) for any image.
- JPEG: Pillow reads the header (lazy open); then the image is decoded with
draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded
stream (a truncated/corrupted file raises) while allocating only 1/64 of
the full-size pixel buffer. Large progressive JPEGs can still retain DCT
coefficients; the caller must bound concurrent validation and memory.
Their entropy stream is checked as well: there is no header-only bypass.
The functions are synchronous (CPU work); call them via asyncio.to_thread.
"""
from __future__ import annotations
import struct
import zlib
from dataclasses import dataclass
from typing import BinaryIO
from PIL import Image
# Our own limit is applied before Pillow decodes anything.
Image.MAX_IMAGE_PIXELS = None
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SIGNATURE = b"\xff\xd8\xff"
CHUNK = 64 * 1024
class InvalidImage(Exception):
"""Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415)."""
class TooManyPixels(Exception):
"""Decoded image would exceed MAX_IMAGE_PIXELS (-> 413)."""
@dataclass(frozen=True)
class ImageInfo:
content_type: str
width: int
height: int
def sniff_content_type(head: bytes) -> str | None:
if head.startswith(PNG_SIGNATURE):
return "image/png"
if head.startswith(JPEG_SIGNATURE):
return "image/jpeg"
return None
def validate_image(
file: BinaryIO,
declared_content_type: str,
max_pixels: int,
) -> ImageInfo:
file.seek(0)
detected = sniff_content_type(file.read(len(PNG_SIGNATURE)))
file.seek(0)
if detected is None:
raise InvalidImage("Content is not a PNG or JPEG image.")
if detected != declared_content_type:
raise InvalidImage("Content-Type does not match the image content.")
if detected == "image/png":
width, height = _validate_png(file, max_pixels)
else:
width, height = _validate_jpeg(file, max_pixels)
return ImageInfo(detected, width, height)
# --- PNG -------------------------------------------------------------------
# color type -> (channels, allowed bit depths)
_PNG_COLOR_TYPES = {
0: (1, {1, 2, 4, 8, 16}),
2: (3, {8, 16}),
3: (1, {1, 2, 4, 8}),
4: (2, {8, 16}),
6: (4, {8, 16}),
}
_ADAM7_PASSES = (
(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4),
(0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2),
)
def _read_exact(file: BinaryIO, size: int) -> bytes:
data = file.read(size)
if len(data) != size:
raise InvalidImage("Image file is truncated.")
return data
def _png_scanlines(width: int, height: int, bits_per_pixel: int,
interlace: int) -> list[tuple[int, int]]:
"""At most seven (packed row bytes, row count) pairs; no row allocation."""
if interlace == 0:
return [((width * bits_per_pixel + 7) // 8, height)]
passes = []
for x0, y0, dx, dy in _ADAM7_PASSES:
w = (width - x0 + dx - 1) // dx if width > x0 else 0
h = (height - y0 + dy - 1) // dy if height > y0 else 0
if w and h:
passes.append(((w * bits_per_pixel + 7) // 8, h))
return passes
class _PngScanlines:
"""Validate filter positions across arbitrary decompression boundaries."""
def __init__(self, passes: list[tuple[int, int]]) -> None:
self.passes = iter(passes)
self.row_bytes, self.rows_left = next(self.passes)
self.row_remaining = 0
self.expected = sum((row_bytes + 1) * count for row_bytes, count in passes)
self.produced = 0
def feed(self, data: bytes) -> None:
self.produced += len(data)
if self.produced > self.expected:
raise InvalidImage("PNG image data is larger than IHDR declares.")
position = 0
while position < len(data):
if self.row_remaining == 0:
if self.rows_left == 0:
try:
self.row_bytes, self.rows_left = next(self.passes)
except StopIteration:
raise InvalidImage("PNG has too many scanlines.") from None
if data[position] > 4:
raise InvalidImage("Invalid PNG scanline filter.")
position += 1
self.rows_left -= 1
self.row_remaining = self.row_bytes
consumed = min(self.row_remaining, len(data) - position)
self.row_remaining -= consumed
position += consumed
def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
_read_exact(file, len(PNG_SIGNATURE))
width = height = 0
scanlines = None
inflater = zlib.decompressobj()
color_type = -1
bit_depth = 0
seen_plte = False
palette_entries = 0
seen_trns = False
idat_state = "before" # before -> inside -> after
first = True
while True:
header = _read_exact(file, 8)
length, chunk_type = struct.unpack(">I4s", header)
if length > 0x7FFFFFFF:
raise InvalidImage("Invalid PNG chunk length.")
if first and chunk_type != b"IHDR":
raise InvalidImage("PNG must start with IHDR.")
if not all(65 <= ch <= 90 or 97 <= ch <= 122 for ch in chunk_type) \
or not 65 <= chunk_type[2] <= 90:
raise InvalidImage("Invalid PNG chunk type.")
if chunk_type == b"IHDR" and (not first or length != 13):
# Check before reading/buffering: an attacker cannot turn a
# bogus multi-megabyte IHDR into a large allocation.
raise InvalidImage("Invalid or duplicate PNG IHDR.")
if chunk_type[0] < 97 and chunk_type not in {b"IHDR", b"PLTE", b"IDAT", b"IEND"}:
raise InvalidImage("Unknown critical PNG chunk.")
if chunk_type == b"IEND" and length != 0:
raise InvalidImage("Invalid PNG IEND.")
if chunk_type == b"PLTE":
if seen_plte or seen_trns or idat_state != "before" or color_type in {0, 4} \
or not 3 <= length <= 768 or length % 3:
raise InvalidImage("Invalid PNG palette.")
palette_entries = length // 3
if color_type == 3 and palette_entries > 1 << bit_depth:
raise InvalidImage("PNG palette exceeds its bit depth.")
if chunk_type == b"tRNS":
valid_length = ((color_type == 0 and length == 2) or
(color_type == 2 and length == 6) or
(color_type == 3 and seen_plte and 1 <= length <= palette_entries))
if seen_trns or idat_state != "before" or not valid_length:
raise InvalidImage("Invalid PNG transparency chunk.")
seen_trns = True
crc = zlib.crc32(chunk_type)
is_idat = chunk_type == b"IDAT"
if is_idat:
if idat_state == "after":
raise InvalidImage("PNG IDAT chunks are not consecutive.")
if color_type == 3 and not seen_plte:
raise InvalidImage("PNG palette image without PLTE.")
idat_state = "inside"
elif idat_state == "inside":
idat_state = "after"
data = b""
remaining = length
while remaining:
piece = _read_exact(file, min(CHUNK, remaining))
remaining -= len(piece)
crc = zlib.crc32(piece, crc)
if is_idat:
_inflate_scanlines(inflater, piece, scanlines)
elif chunk_type == b"IHDR":
data += piece
(stored_crc,) = struct.unpack(">I", _read_exact(file, 4))
if stored_crc != crc:
raise InvalidImage("PNG chunk CRC mismatch.")
if first:
first = False
(width, height, bit_depth, color_type,
compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data)
if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF:
raise InvalidImage("Invalid PNG dimensions.")
if width * height > max_pixels:
# Before a single IDAT byte is decompressed.
raise TooManyPixels()
if color_type not in _PNG_COLOR_TYPES:
raise InvalidImage("Invalid PNG color type.")
channels, depths = _PNG_COLOR_TYPES[color_type]
if bit_depth not in depths or compression != 0 or filter_method != 0 \
or interlace not in (0, 1):
raise InvalidImage("Invalid PNG IHDR.")
scanlines = _PngScanlines(
_png_scanlines(width, height, channels * bit_depth, interlace)
)
elif chunk_type == b"PLTE":
seen_plte = True
elif chunk_type == b"IEND":
break
if idat_state == "before":
raise InvalidImage("PNG has no image data.")
# decompress(max_length) drains every unconsumed_tail above. No flush()
# is needed (and its output would not have a strict allocation bound).
if not inflater.eof or scanlines.produced != scanlines.expected:
raise InvalidImage("PNG image data is corrupted or truncated.")
return width, height
def _inflate_scanlines(inflater, data: bytes, scanlines: _PngScanlines) -> None:
"""Decompress at most 64 KiB at once, validate, then discard."""
try:
while data:
if inflater.eof:
raise InvalidImage("PNG has data after the end of the zlib stream.")
out = inflater.decompress(data, CHUNK)
scanlines.feed(out)
if inflater.unused_data:
raise InvalidImage("PNG has data after the end of the zlib stream.")
data = inflater.unconsumed_tail
except zlib.error:
raise InvalidImage("PNG image data is corrupted.") from None
# --- JPEG ------------------------------------------------------------------
def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
try:
with Image.open(file, formats=["JPEG"]) as image:
width, height = image.size
if width < 1 or height < 1:
raise InvalidImage("Invalid JPEG dimensions.")
if width * height > max_pixels:
# Header only: no pixel has been decoded yet.
raise TooManyPixels()
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
image.load()
except (InvalidImage, TooManyPixels):
raise
except Exception:
# Pillow raises OSError/SyntaxError/ValueError/... for broken data.
raise InvalidImage("JPEG image is corrupted or truncated.") from None
return width, height
+145
View File
@@ -0,0 +1,145 @@
"""Transactional outbox relay, expired-lease recovery and staging janitor.
DB COMMIT and RabbitMQ publish cannot be one transaction. Keep outbox rows
until a publisher confirm, accepting harmless duplicate delivery on crash.
"""
from __future__ import annotations
import asyncio
import logging
from sqlalchemy import text
from app.db import SessionLocal, engine
from app.pipeline import (RETENTION_SECONDS, arm_from_env, broker_channel,
crash_if_armed, envelope, publish, retry_or_fail)
from app.storage import storage
from app.telemetry import configure, emit, install_termination_handler, shutdown
from app import advanced
logger = logging.getLogger("messenger.jobs")
async def recover_leases() -> int:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT * FROM attachments WHERE status='processing' AND lease_until<now()
ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED
"""))
rows = list(result.mappings())
for row in rows:
await retry_or_fail(db, row, "lease_expired")
await db.commit()
if rows:
logger.warning("recovered expired leases count=%s", len(rows))
return len(rows)
async def cleanup_staging() -> None:
# One object transaction at a time: S3's bounded deadline is shorter
# than idle_in_transaction_session_timeout. Replay takes the same lock.
for _ in range(10):
async with SessionLocal() as db:
# Failed retention may remove five objects. Keep its row lock
# until all bounded deletes finish, so replay cannot race it.
await db.execute(text("SET LOCAL idle_in_transaction_session_timeout='60s'"))
result = await db.execute(text("""
SELECT * FROM attachments
WHERE staging_key IS NOT NULL AND NOT staging_deleted AND
(status='ready' OR (status='failed' AND updated_at <
now()-make_interval(secs => :retention)))
ORDER BY updated_at LIMIT 1 FOR UPDATE SKIP LOCKED
"""), {"retention": RETENTION_SECONDS})
row = result.mappings().first()
if row is None:
await db.rollback()
return
await storage.delete(row["staging_key"])
# Partial final objects of an exhausted job are no longer
# useful after retention; ready objects are never deleted.
if row["status"] == "failed":
prefix = row["object_key"].rsplit("/", 1)[0]
for suffix in ("original", "thumbnail.jpg", "crop.jpg", "optimized.webp"):
await storage.delete(f"{prefix}/{suffix}")
await db.execute(text("""
UPDATE attachments SET staging_deleted=true WHERE id=:id
"""), {"id": row["id"]})
await db.commit()
logger.info("staging cleaned attachment=%s", row["id"])
async def relay_one(channel) -> bool:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT o.*,a.pipeline_version FROM outbox o
JOIN attachments a ON a.id=o.attachment_id
WHERE o.published_at IS NULL AND o.available_at<=now()
ORDER BY o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED
"""))
row = result.mappings().first()
if row is None:
await db.rollback()
return False
# The upload COMMIT already exists; the pending row is untouched
# by this crash, so the restarted relay finds it automatically.
await crash_if_armed("crash_after_commit_before_publish")
await publish(channel, envelope(row))
await db.execute(text("UPDATE outbox SET published_at=now() WHERE id=:id"),
{"id": row["id"]})
await db.commit()
logger.info("published job=%s attempt=%s", row["job_id"], row["attempt"])
return True
async def main() -> None:
configure("messenger-publisher")
install_termination_handler()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
for name in ("aio_pika", "aiormq"):
logging.getLogger(name).setLevel(logging.CRITICAL)
await storage.start()
backlog = asyncio.create_task(advanced.backlog_loop(SessionLocal))
connection = None
try:
await arm_from_env("PUBLISHER_FAULT")
while True:
try:
connection, channel, _ = await broker_channel()
last_maintenance = 0.0
while True:
now = asyncio.get_running_loop().time()
if now - last_maintenance > 1:
await recover_leases()
try:
await cleanup_staging()
except Exception as exc:
# Retention is best effort. S3 failure must not
# starve outbox delivery or exhaustible retries.
logger.warning("staging cleanup deferred: %s", type(exc).__name__)
last_maintenance = now
if not await relay_one(channel):
await asyncio.sleep(0.25)
except asyncio.CancelledError:
raise
except Exception as exc:
# Exception text can contain connection URLs. Log type only.
logger.warning("relay unavailable: %s", type(exc).__name__)
if connection:
await connection.close()
connection = None
await asyncio.sleep(1)
finally:
backlog.cancel()
await asyncio.gather(backlog, return_exceptions=True)
if connection:
await connection.close()
await storage.close()
await engine.dispose()
await asyncio.to_thread(shutdown)
if __name__ == "__main__":
try:
asyncio.run(main())
except asyncio.CancelledError:
pass # SIGTERM has run finally/SDK flush; exit cleanly.
+6
View File
@@ -0,0 +1,6 @@
"""Process lifecycle state shared by the server and request handlers."""
import threading
# Set on the first SIGTERM/SIGINT. While draining, /health/ready answers 503
# and responses carry "Connection: close", but requests are still served.
draining = threading.Event()
+461
View File
@@ -0,0 +1,461 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from uuid import UUID
from typing import Literal
from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, StreamingResponse
from sqlalchemy import exc as sa_exc
from starlette.exceptions import HTTPException as StarletteHTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.attachments import (
get_attachment,
janitor_loop,
open_attachment_content,
upload_attachment,
)
from app.cache import session_cache
from app.db import check_database_ready, engine, get_db
from app.deps import basic_scheme, pagination_params, require_user
from app.errors import (
ApiError,
InvalidRequest,
Unavailable,
api_error_handler,
common_headers,
)
from app.lifecycle import draining
from app.config import settings
from app.telemetry import HttpTelemetryMiddleware, configure, shutdown
from app.models import (
Attachment,
Chat,
ChatPage,
CreateChat,
CreateMessage,
CreateUser,
Health,
Message,
MessagePage,
Session,
User,
)
from app.storage import storage
from app.services import (
create_chat,
create_message,
create_user_session,
get_chat,
list_chats,
list_messages,
logout_current_session,
register_user,
)
@asynccontextmanager
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
configure("messenger-api", settings.instance_id)
await storage.start()
janitor = asyncio.create_task(janitor_loop())
try:
yield
finally:
janitor.cancel()
await asyncio.gather(janitor, return_exceptions=True)
await storage.close()
await session_cache.close()
await engine.dispose()
await asyncio.to_thread(shutdown)
app = FastAPI(
title="MTUSI Messenger — лабораторная 6",
version="1.6.0",
lifespan=lifespan,
)
@app.exception_handler(ApiError)
async def handle_api_error(
request: Request,
exc: Exception,
) -> JSONResponse:
if not isinstance(exc, ApiError):
raise exc
return await api_error_handler(request, exc)
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(
request: Request,
exc: StarletteHTTPException,
) -> JSONResponse:
# Unknown routes/methods: the contract error format instead of
# FastAPI's default {"detail": ...}.
if exc.status_code == 404:
error = ApiError(404, "not_found", "Route not found.")
else:
error = ApiError(exc.status_code, "invalid_request", "Invalid request.")
return await api_error_handler(request, error)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(
request: Request,
_exc: RequestValidationError,
) -> JSONResponse:
return JSONResponse(
status_code=400,
content={
"error": {
"code": "invalid_request",
"message": "Invalid request.",
"request_id": common_headers(request)["X-Request-Id"],
}
},
headers=common_headers(request),
)
logger = logging.getLogger("messenger.api")
def is_dependency_failure(exc: Exception) -> bool:
"""PostgreSQL is unreachable, restarting, overloaded or timed out."""
if isinstance(
exc,
(
sa_exc.OperationalError,
sa_exc.InterfaceError,
sa_exc.TimeoutError, # connection pool exhausted
OSError,
TimeoutError,
),
):
return True
return isinstance(exc, sa_exc.DBAPIError) and exc.connection_invalidated
async def dependency_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
if not is_dependency_failure(exc):
return await unexpected_error_handler(request, exc)
logger.warning("database unavailable: %s", type(exc).__name__)
return await api_error_handler(request, Unavailable())
for _exc_class in (sa_exc.DBAPIError, sa_exc.TimeoutError, OSError, TimeoutError):
app.add_exception_handler(_exc_class, dependency_error_handler)
@app.exception_handler(Exception)
async def unexpected_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
headers = common_headers(request)
# Only the type: SQL parameters and headers may contain secrets.
logger.error("unexpected error: %s", type(exc).__name__)
return JSONResponse(
status_code=500,
content={
"error": {
"code": "internal_error",
"message": "Internal server error.",
"request_id": headers["X-Request-Id"],
}
},
headers=headers,
)
@app.get(
"/health/live",
response_model=Health,
)
async def liveness() -> dict[str, str]:
return {"status": "ok"}
@app.get(
"/health/ready",
response_model=Health,
)
async def readiness() -> dict[str, str]:
# A draining replica reports "not ready" so HAProxy stops sending new
# requests to it before the listener is closed.
# S3 is deliberately NOT checked: without it only attachments degrade
# (503), users/chats/messages keep working. If readiness depended on S3,
# an S3 outage would take every replica out of the LB.
if draining.is_set() or not await check_database_ready():
raise Unavailable()
return {"status": "ok"}
@app.post(
"/api/v1/users",
status_code=201,
response_model=User,
)
async def register_user_route(
payload: CreateUser,
db: AsyncSession = Depends(get_db),
):
return await register_user(
db,
username=payload.username,
display_name=payload.display_name,
password=payload.password,
)
@app.post(
"/api/v1/auth/sessions",
status_code=201,
response_model=Session,
)
async def login(
request: Request,
db: AsyncSession = Depends(get_db),
_basic: None = Security(basic_scheme),
):
# Contract: login takes only "Authorization: Basic", no request body.
# Any body (malformed JSON, "{}", ...) is rejected before the password check.
if await request.body():
raise InvalidRequest("POST /auth/sessions does not accept a request body.")
return await create_user_session(
db,
request.headers.get("Authorization"),
)
@app.delete(
"/api/v1/auth/sessions/current",
status_code=204,
)
async def logout(
request: Request,
db: AsyncSession = Depends(get_db),
_current_user: dict = Depends(require_user),
) -> Response:
_, _, token = request.headers["Authorization"].partition(" ")
await logout_current_session(
db,
token.strip(),
)
return Response(status_code=204)
@app.get(
"/api/v1/users/me",
response_model=User,
)
async def current_user_route(
current_user: dict = Depends(require_user),
):
return current_user
@app.post(
"/api/v1/chats",
status_code=201,
response_model=Chat,
)
async def create_chat_route(
payload: CreateChat,
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
return await create_chat(
db,
title=payload.title,
member_ids=payload.member_ids,
current_user=current_user,
)
@app.get(
"/api/v1/chats",
response_model=ChatPage,
)
async def list_chats_route(
page: tuple[int, str | None] = Depends(pagination_params),
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
limit, cursor = page
return await list_chats(
db,
limit=limit,
cursor=cursor,
current_user=current_user,
)
@app.get(
"/api/v1/chats/{chat_id}",
response_model=Chat,
)
async def get_chat_route(
chat_id: UUID,
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
return await get_chat(
db,
chat_id=chat_id,
current_user=current_user,
)
@app.post(
"/api/v1/chats/{chat_id}/messages",
status_code=201,
response_model=Message,
)
async def create_message_route(
chat_id: UUID,
payload: CreateMessage,
response: Response,
idempotency_key: str | None = Header(
default=None,
alias="Idempotency-Key",
),
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
message, replayed = await create_message(
db,
chat_id=chat_id,
text_value=payload.text,
attachment_ids=payload.attachment_ids,
idempotency_key=idempotency_key,
current_user=current_user,
)
if replayed:
response.headers["Idempotent-Replayed"] = "true"
return message
@app.get(
"/api/v1/chats/{chat_id}/messages",
response_model=MessagePage,
)
async def list_messages_route(
chat_id: UUID,
page: tuple[int, str | None] = Depends(pagination_params),
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
limit, cursor = page
return await list_messages(
db,
chat_id=chat_id,
limit=limit,
cursor=cursor,
current_user=current_user,
)
UPLOAD_BODY = {
"requestBody": {
"required": True,
"description": "Raw image bytes, not multipart and not JSON/base64.",
"content": {
"image/png": {"schema": {"type": "string", "format": "binary"}},
"image/jpeg": {"schema": {"type": "string", "format": "binary"}},
},
}
}
@app.post(
"/api/v1/chats/{chat_id}/attachments",
status_code=202,
response_model=Attachment,
openapi_extra=UPLOAD_BODY,
)
async def upload_attachment_route(
chat_id: UUID,
request: Request,
filename: str = Query(min_length=1, max_length=255),
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
return await upload_attachment(
request,
db,
chat_id=chat_id,
filename=filename,
current_user=current_user,
)
@app.get(
"/api/v1/attachments/{attachment_id}",
response_model=Attachment,
)
async def get_attachment_route(
attachment_id: UUID,
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
return await get_attachment(
db,
attachment_id=attachment_id,
current_user=current_user,
)
@app.get(
"/api/v1/attachments/{attachment_id}/content",
response_class=StreamingResponse,
responses={200: {"content": {"image/png": {}, "image/jpeg": {}, "image/webp": {}}}},
)
async def download_attachment_route(
attachment_id: UUID,
# Known variant names are accepted so that a ready attachment without
# that variant answers 404 (contract); any other value is 400.
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(
default="original"
),
current_user: dict = Depends(require_user),
db: AsyncSession = Depends(get_db),
):
chunks, headers = await open_attachment_content(
db,
attachment_id=attachment_id,
variant=variant,
current_user=current_user,
)
return StreamingResponse(
chunks,
media_type=headers.pop("Content-Type"),
headers=headers,
)
# Wrap outside FastAPI ServerErrorMiddleware to observe its final 500 response.
app = HttpTelemetryMiddleware(app)
+193
View File
@@ -0,0 +1,193 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class ApiModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateUser(ApiModel):
username: str = Field(
min_length=3,
max_length=32,
pattern=r"^[a-z0-9_]{3,32}$",
)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_is_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("username must not contain only spaces.")
return value
@field_validator("display_name")
@classmethod
def display_name_is_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError(
"display_name must not contain only spaces."
)
return value
@field_validator("password")
@classmethod
def password_is_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("password must not contain only spaces.")
return value
class User(ApiModel):
id: UUID
username: str
display_name: str
created_at: datetime
@field_validator("created_at")
@classmethod
def user_time_is_utc(cls, value: datetime) -> datetime:
if value.tzinfo is None:
raise ValueError("created_at must include UTC timezone.")
return value.astimezone(timezone.utc)
class CreateChat(ApiModel):
# Contract: 1-100 code points (lab2 allowed 255 by mistake).
title: str = Field(min_length=1, max_length=100)
# Contract: 1-100 members (minItems/maxItems in openapi.json).
member_ids: list[UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def title_is_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("title must not contain only spaces.")
return value
class Chat(ApiModel):
id: UUID
title: str
member_ids: list[UUID]
created_at: datetime
@field_validator("created_at")
@classmethod
def chat_time_is_utc(cls, value: datetime) -> datetime:
if value.tzinfo is None:
raise ValueError("created_at must include UTC timezone.")
return value.astimezone(timezone.utc)
class ChatPage(ApiModel):
items: list[Chat]
next_cursor: str | None = None
class CreateMessage(ApiModel):
# Lab4: text may be "" when the message carries attachments.
text: str = Field(max_length=4000)
# 0-10 unique IDs of attachments of the same chat; missing == [].
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids")
@classmethod
def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError("attachment_ids must be unique.")
return value
@model_validator(mode="after")
def text_or_attachments(self) -> "CreateMessage":
if not self.attachment_ids and not self.text.strip():
raise ValueError(
"text must not be blank when there are no attachments."
)
return self
class Message(ApiModel):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: list[UUID] = Field(default_factory=list)
@field_validator("created_at")
@classmethod
def message_time_is_utc(cls, value: datetime) -> datetime:
if value.tzinfo is None:
raise ValueError("created_at must include UTC timezone.")
return value.astimezone(timezone.utc)
class MessagePage(ApiModel):
items: list[Message]
next_cursor: str | None = None
class Session(ApiModel):
token: str = Field(min_length=1)
token_type: Literal["Bearer"]
expires_at: datetime
user: User
@field_validator("expires_at")
@classmethod
def expiry_time_is_utc(cls, value: datetime) -> datetime:
if value.tzinfo is None:
raise ValueError("expires_at must include UTC timezone.")
return value.astimezone(timezone.utc)
class Variant(ApiModel):
name: Literal["thumbnail", "crop", "optimized"]
content_type: Literal["image/jpeg", "image/png", "image/webp"]
size_bytes: int
width: int
height: int
sha256: str
class Attachment(ApiModel):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: Literal["image/png", "image/jpeg"]
size_bytes: int
sha256: str
status: Literal["queued", "processing", "ready", "failed"]
variants: list[Variant]
error_code: str | None
created_at: datetime
updated_at: datetime
@field_validator("created_at", "updated_at")
@classmethod
def attachment_time_is_utc(cls, value: datetime) -> datetime:
if value.tzinfo is None:
raise ValueError("timestamps must include UTC timezone.")
return value.astimezone(timezone.utc)
class Health(ApiModel):
status: Literal["ok"]
class ErrorInner(ApiModel):
code: str = Field(min_length=1)
message: str = Field(min_length=1)
request_id: str = Field(min_length=1)
class Error(ApiModel):
error: ErrorInner
+92
View File
@@ -0,0 +1,92 @@
"""Explicit local operator commands; no new public HTTP endpoints."""
from __future__ import annotations
import argparse
import asyncio
import json
from uuid import UUID
from sqlalchemy import text
from app.db import SessionLocal, engine
from app.pipeline import event
POINTS = ("crash_after_commit_before_publish", "crash_after_objects_before_commit",
"crash_after_commit_before_ack")
async def run(args) -> None:
try:
async with SessionLocal() as db:
if args.command == "fault":
await db.execute(text("""
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
ON CONFLICT(point) DO UPDATE SET remaining=1,updated_at=now()
"""), {"point": args.point})
await db.commit()
print(f"ARMED one-shot fault: {args.point}")
return
if args.command == "failed":
result = await db.execute(text("""
SELECT id,job_id,attempt,attempts_run,error_code,staging_deleted,updated_at
FROM attachments WHERE status='failed' ORDER BY updated_at
"""))
print(json.dumps([dict(row) for row in result.mappings()], default=str, indent=2))
return
result = await db.execute(text("SELECT * FROM attachments WHERE id=:id FOR UPDATE"),
{"id": args.attachment_id})
row = result.mappings().first()
if row is None or row["job_id"] is None:
raise ValueError("Unknown asynchronous attachment")
if args.command == "replay":
if row["status"] == "ready":
print("NO-OP: attachment is already ready")
return
if row["status"] != "failed":
raise ValueError("Replay is allowed only for failed attachments")
if row["staging_deleted"]:
raise ValueError("Replay source expired; upload the original again")
if not args.reason.strip() or len(args.reason) > 200:
raise ValueError("Replay reason must have 1-200 characters")
result = await db.execute(text("""
UPDATE attachments SET status='queued',attempt=attempt+1,attempts_run=0,
error_code=NULL,lease_token=NULL,lease_until=NULL,
next_attempt_at=now(),updated_at=now() WHERE id=:id RETURNING *
"""), {"id": row["id"]})
row = result.mappings().one()
await event(db, row, "operator_replay", args.reason)
else:
# Real duplicate envelope: same job ID and current attempt.
# Ready effects are checked in PG by the worker and acked.
await event(db, row, "operator_redeliver")
await db.execute(text("""
INSERT INTO outbox(job_id,attachment_id,attempt,available_at,traceparent,tracestate)
SELECT job_id,id,attempt,GREATEST(now(),next_attempt_at),traceparent,tracestate
FROM attachments WHERE id=:id
"""), {"id": row["id"]})
await db.commit()
print(f"QUEUED: {args.command} attachment={row['id']} job={row['job_id']}")
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
commands.add_parser("failed")
fault = commands.add_parser("fault")
fault.add_argument("point", choices=POINTS)
replay = commands.add_parser("replay")
replay.add_argument("attachment_id", type=UUID)
replay.add_argument("--reason", required=True)
redeliver = commands.add_parser("redeliver")
redeliver.add_argument("attachment_id", type=UUID)
args = parser.parse_args()
try:
asyncio.run(run(args))
except ValueError as exc:
parser.exit(2, f"REFUSED: {exc}\n")
if __name__ == "__main__":
main()
+182
View File
@@ -0,0 +1,182 @@
"""Shared durable-job plumbing, deliberately separate from the HTTP API."""
from __future__ import annotations
import json
import logging
import math
import os
import random
import re
from datetime import datetime
from uuid import UUID
import aio_pika
from sqlalchemy import text
from app.config import read_value
from app.db import SessionLocal
from app import advanced
from app.telemetry import emit, span
from app.trace_context import carrier, parent_context
from opentelemetry.trace import SpanKind
logger = logging.getLogger("messenger.pipeline")
QUEUE = "attachments"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120"))
IMAGE_TIMEOUT = float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
TOTAL_TIMEOUT = float(os.getenv("JOB_TOTAL_TIMEOUT_SECONDS", "90"))
RETENTION_SECONDS = int(os.getenv("STAGING_RETENTION_SECONDS", "604800"))
RETRY_BASE_SECONDS = float(os.getenv("RETRY_BASE_SECONDS", "1"))
if (MAX_ATTEMPTS < 1 or not math.isfinite(IMAGE_TIMEOUT) or IMAGE_TIMEOUT <= 0
or not 0 < TOTAL_TIMEOUT < LEASE_SECONDS):
raise RuntimeError("Invalid job limits; total timeout must be below lease.")
if not math.isfinite(RETRY_BASE_SECONDS) or RETRY_BASE_SECONDS <= 0:
raise RuntimeError("RETRY_BASE_SECONDS must be finite and positive.")
def retry_delay(attempts_run: int) -> float:
# Equal jitter: never zero, capped exponential delay, one durable due date.
upper = min(60.0, RETRY_BASE_SECONDS * 2 ** min(max(attempts_run - 1, 0), 10))
return random.uniform(upper / 2, upper)
async def broker_channel():
connection = await aio_pika.connect_robust(read_value("BROKER_URL"), timeout=5)
try:
channel = await connection.channel(publisher_confirms=True, on_return_raises=True)
await channel.set_qos(prefetch_count=1)
dead = await channel.declare_exchange("attachments.dead", aio_pika.ExchangeType.FANOUT,
durable=True)
dlq = await channel.declare_queue("attachments.dlq", durable=True)
await dlq.bind(dead)
queue = await channel.declare_queue(
QUEUE, durable=True, arguments={"x-dead-letter-exchange": "attachments.dead"}
)
return connection, channel, queue
except BaseException:
await connection.close()
raise
def envelope(row) -> dict:
job = {
"schema_version": 1, "job_id": str(row["job_id"]),
"type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]),
"pipeline_version": int(row.get("pipeline_version", 1)),
"attempt": int(row["attempt"]),
"created_at": row["created_at"].isoformat().replace("+00:00", "Z"),
}
job.update({name: row[name] for name in ("traceparent", "tracestate") if row.get(name)})
return job
def parse_envelope(raw: bytes) -> dict:
job = json.loads(raw)
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
"attempt", "created_at"}
if not isinstance(job, dict) or not required <= job.keys() or (
job.keys() - required - {"traceparent", "tracestate"}
):
raise ValueError("invalid job fields")
if type(job["schema_version"]) is not int or job["schema_version"] != 1 or job["type"] != "attachment.process.v1":
raise ValueError("unsupported job schema")
if type(job["attempt"]) is not int or job["attempt"] < 1:
raise ValueError("invalid attempt")
if type(job["pipeline_version"]) is not int or job["pipeline_version"] != 1:
raise ValueError("unsupported pipeline")
UUID(job["job_id"])
UUID(job["attachment_id"])
if datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).tzinfo is None:
raise ValueError("job timestamp must include timezone")
if "traceparent" in job and not re.fullmatch(
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", job["traceparent"]
):
raise ValueError("invalid traceparent")
if "tracestate" in job and (not isinstance(job["tracestate"], str) or
len(job["tracestate"]) > 512):
raise ValueError("invalid tracestate")
return job
async def publish(channel, job: dict) -> None:
with span("attachments publish", kind=SpanKind.PRODUCER, parent=parent_context(job), attributes={
"messaging.system": "rabbitmq", "messaging.destination.name": QUEUE,
"messaging.operation": "publish", "job_id": job["job_id"], "attempt": job["attempt"],
}):
# The confirmed envelope names this producer span as the consumer's
# parent. Durable outbox remains unchanged if confirmation is lost.
outgoing = {**job, **carrier()}
await channel.default_exchange.publish(
aio_pika.Message(json.dumps(outgoing).encode("utf-8"), content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=job["job_id"]),
routing_key=QUEUE, mandatory=True, timeout=5,
)
emit("job.published", job_id=job["job_id"], attempt=job["attempt"], queue=QUEUE)
async def event(db, row, name: str, detail: str = "") -> None:
await db.execute(text("""
INSERT INTO job_events(attachment_id,event,attempt,detail)
VALUES (:id,:event,:attempt,:detail)
"""), {"id": row["id"], "attempt": row["attempt"], "event": name, "detail": detail})
async def arm_from_env(variable: str) -> None:
point = os.getenv(variable, "")
if point:
async with SessionLocal() as db:
await db.execute(text("""
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
ON CONFLICT (point) DO NOTHING
"""), {"point": point})
await db.commit()
async def crash_if_armed(point: str) -> None:
async with SessionLocal() as db:
result = await db.execute(text("""
UPDATE fault_injections SET remaining=remaining-1,hits=hits+1,updated_at=now()
WHERE point=:point AND remaining>0 RETURNING point
"""), {"point": point})
hit = result.first() is not None
await db.commit()
if hit:
logger.error("FAULT: %s (one-shot durable hit)", point)
os._exit(70)
async def retry_or_fail(db, row, code: str, *, permanent: bool = False) -> None:
"""Caller owns the attachment row lock. Commit before acking old delivery."""
if permanent or row["attempts_run"] >= MAX_ATTEMPTS:
final_code = code if permanent else "retry_exhausted"
await db.execute(text("""
UPDATE attachments SET status='failed',error_code=:code,
lease_token=NULL,lease_until=NULL,updated_at=now()
WHERE id=:id
"""), {"id": row["id"], "code": final_code})
await event(db, row, "failed", final_code)
logger.warning("failed attachment=%s code=%s", row["id"], final_code)
return
delay = retry_delay(row["attempts_run"])
# A consumer retry continues from its active span. Lease recovery may have
# no active span, so retain the context durably recorded by the old claim.
context = carrier() or {name: row.get(name) for name in ("traceparent", "tracestate")}
await db.execute(text("""
UPDATE attachments SET status='queued',error_code=NULL,attempt=attempt+1,
lease_token=NULL,lease_until=NULL,
traceparent=:traceparent,tracestate=:tracestate,
next_attempt_at=now()+make_interval(secs => CAST(:delay AS double precision)),
updated_at=now() WHERE id=:id
"""), {"id": row["id"], "delay": delay,
"traceparent": context.get("traceparent"), "tracestate": context.get("tracestate")})
await db.execute(text("""
INSERT INTO outbox(job_id,attachment_id,attempt,available_at,traceparent,tracestate)
SELECT job_id,id,attempt,next_attempt_at,traceparent,tracestate FROM attachments WHERE id=:id
"""), {"id": row["id"]})
await event(db, row, "retry", f"{code}; delay={delay:.3f}s")
logger.info("retry attachment=%s after=%.3fs", row["id"], delay)
advanced.retries.add(1, {"job_type": "attachment.process.v1"})
emit("job.retry", job_id=str(row["job_id"]), attachment_id=str(row["id"]),
attempt=row["attempt"] + 1, outcome="retry", queue=QUEUE)
+726
View File
@@ -0,0 +1,726 @@
"""PostgreSQL repository functions for MTUSI Messenger Lab4."""
from __future__ import annotations
from collections.abc import Iterable
from datetime import datetime
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.trace_context import carrier
async def create_user(
db: AsyncSession,
*,
username: str,
display_name: str,
password_hash: str,
) -> dict[str, Any] | None:
user_id = uuid4()
try:
result = await db.execute(
text(
"""
INSERT INTO users (id, username, display_name, password_hash)
VALUES (:id, :username, :display_name, :password_hash)
RETURNING id, username, display_name, created_at
"""
),
{
"id": user_id,
"username": username,
"display_name": display_name,
"password_hash": password_hash,
},
)
await db.commit()
return dict(result.mappings().one())
except IntegrityError:
await db.rollback()
return None
async def get_user_by_username(
db: AsyncSession,
username: str,
) -> dict[str, Any] | None:
result = await db.execute(
text(
"""
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = :username
"""
),
{"username": username},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def get_user_by_id(
db: AsyncSession,
user_id: UUID,
) -> dict[str, Any] | None:
result = await db.execute(
text(
"""
SELECT id, username, display_name, created_at
FROM users
WHERE id = :user_id
"""
),
{"user_id": user_id},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def user_ids_that_exist(
db: AsyncSession,
member_ids: Iterable[UUID],
) -> set[UUID]:
values = list(member_ids)
if not values:
return set()
result = await db.execute(
text("SELECT id FROM users WHERE id = ANY(CAST(:member_ids AS uuid[]))"),
{"member_ids": values},
)
return {row.id for row in result}
async def create_session(
db: AsyncSession,
*,
user_id: UUID,
token_hash: str,
expires_at: datetime,
) -> None:
await db.execute(
text(
"""
INSERT INTO sessions (id, user_id, token_hash, expires_at)
VALUES (:id, :user_id, :token_hash, :expires_at)
"""
),
{
"id": uuid4(),
"user_id": user_id,
"token_hash": token_hash,
"expires_at": expires_at,
},
)
await db.commit()
async def get_active_session_user(
db: AsyncSession,
token_hash: str,
) -> dict[str, Any] | None:
result = await db.execute(
text(
"""
SELECT
s.id AS session_id,
s.token_hash,
s.expires_at,
u.id,
u.username,
u.display_name,
u.created_at
FROM sessions AS s
JOIN users AS u ON u.id = s.user_id
WHERE s.token_hash = :token_hash
AND s.revoked_at IS NULL
AND s.expires_at > now()
"""
),
{"token_hash": token_hash},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def revoke_active_session(
db: AsyncSession,
token_hash: str,
) -> bool:
result = await db.execute(
text(
"""
UPDATE sessions
SET revoked_at = now()
WHERE token_hash = :token_hash
AND revoked_at IS NULL
AND expires_at > now()
"""
),
{"token_hash": token_hash},
)
await db.commit()
return result.rowcount == 1
async def create_chat_with_members(
db: AsyncSession,
*,
title: str,
member_ids: Iterable[UUID],
) -> dict[str, Any]:
chat_id = uuid4()
members = list(member_ids)
try:
result = await db.execute(
text(
"""
INSERT INTO chats (id, title)
VALUES (:id, :title)
RETURNING id, title, created_at
"""
),
{"id": chat_id, "title": title},
)
chat = dict(result.mappings().one())
await db.execute(
text(
"""
INSERT INTO chat_members (chat_id, user_id)
VALUES (:chat_id, :user_id)
"""
),
[{"chat_id": chat_id, "user_id": user_id} for user_id in members],
)
await db.commit()
except Exception:
await db.rollback()
raise
chat["member_ids"] = members
return chat
async def get_accessible_chat(
db: AsyncSession,
*,
chat_id: UUID,
user_id: UUID,
) -> dict[str, Any] | None:
result = await db.execute(
text(
"""
SELECT c.id, c.title, c.created_at
FROM chats AS c
JOIN chat_members AS own_membership
ON own_membership.chat_id = c.id
AND own_membership.user_id = :user_id
WHERE c.id = :chat_id
"""
),
{"chat_id": chat_id, "user_id": user_id},
)
chat_row = result.mappings().first()
if chat_row is None:
return None
members_result = await db.execute(
text(
"""
SELECT user_id
FROM chat_members
WHERE chat_id = :chat_id
ORDER BY user_id ASC
"""
),
{"chat_id": chat_id},
)
chat = dict(chat_row)
chat["member_ids"] = [row.user_id for row in members_result]
return chat
async def list_chats_for_user(
db: AsyncSession,
*,
user_id: UUID,
limit: int,
cursor_created_at: datetime | None,
cursor_id: UUID | None,
) -> list[dict[str, Any]]:
query = """
SELECT c.id, c.title, c.created_at
FROM chats AS c
JOIN chat_members AS own_membership
ON own_membership.chat_id = c.id
AND own_membership.user_id = :user_id \
"""
params: dict[str, Any] = {
"user_id": user_id,
"limit_plus_one": limit + 1,
}
if cursor_created_at is not None and cursor_id is not None:
query += """
WHERE (c.created_at, c.id) > (:cursor_created_at, :cursor_id)
"""
params["cursor_created_at"] = cursor_created_at
params["cursor_id"] = cursor_id
query += " ORDER BY c.created_at ASC, c.id ASC LIMIT :limit_plus_one"
result = await db.execute(text(query), params)
chats = [dict(row) for row in result.mappings().all()]
if not chats:
return []
chat_ids = [chat["id"] for chat in chats]
members_result = await db.execute(
text(
"""
SELECT chat_id, user_id
FROM chat_members
WHERE chat_id = ANY(CAST(:chat_ids AS uuid[]))
ORDER BY chat_id ASC, user_id ASC
"""
),
{"chat_ids": chat_ids},
)
members_by_chat: dict[UUID, list[UUID]] = {chat_id: [] for chat_id in chat_ids}
for row in members_result:
members_by_chat[row.chat_id].append(row.user_id)
for chat in chats:
chat["member_ids"] = members_by_chat[chat["id"]]
return chats
# Attachment IDs of a message in the order of the request.
MESSAGE_ATTACHMENT_IDS = """
COALESCE(
(SELECT array_agg(ma.attachment_id ORDER BY ma.position)
FROM message_attachments AS ma
WHERE ma.message_id = m.id),
CAST('{}' AS uuid[])
) AS attachment_ids
"""
class AttachmentsNotFound(Exception):
"""Some attachment_ids do not exist or belong to another chat (-> 404)."""
class AttachmentFailed(Exception):
"""A failed attachment cannot be added to a new message (-> 409)."""
async def create_message_idempotent(
db: AsyncSession,
*,
chat_id: UUID,
sender_id: UUID,
text_value: str,
attachment_ids: list[UUID],
idempotency_key: str,
request_hash: str,
ttl_seconds: int,
) -> tuple[dict[str, Any], bool] | None:
"""Insert a message at most once per idempotency key; one transaction.
1. INSERT the key row. Its primary key (user, chat, method, key) lives in
the shared PostgreSQL, so a concurrent request with the same key on
ANY replica blocks on this INSERT until the first transaction ends.
2. If the key was claimed: check that every attachment belongs to this
chat, INSERT the message and its attachment links, link the message
to the key and COMMIT everything together. A failure before COMMIT
(also AttachmentsNotFound) rolls back all of it, so an error is never
stored as the final answer and a retry with the same key runs again.
3. If the key already exists (committed by someone else): return the
stored message, or None when the body differs (-> 409).
A key older than ttl_seconds is reclaimed by the ON CONFLICT ... WHERE.
Returns (message, replayed) or None on a body mismatch.
"""
key_params = {
"user_id": sender_id,
"chat_id": chat_id,
"key": idempotency_key,
}
try:
claimed = await db.execute(
text(
"""
INSERT INTO idempotency_keys AS k
(user_id, chat_id, method, key, request_hash)
VALUES (:user_id, :chat_id, 'POST', :key, :request_hash)
ON CONFLICT (user_id, chat_id, method, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash,
message_id = NULL,
created_at = now()
WHERE k.created_at < now() - make_interval(secs => :ttl)
RETURNING 1
"""
),
{**key_params, "request_hash": request_hash, "ttl": ttl_seconds},
)
if claimed.first() is not None:
if attachment_ids:
# Unknown and foreign IDs look the same (404): the response
# does not reveal that an attachment of another chat exists.
found = await db.execute(
text(
"""
SELECT id, status FROM attachments
WHERE chat_id = :chat_id
AND id = ANY(CAST(:ids AS uuid[]))
FOR SHARE
"""
),
{"chat_id": chat_id, "ids": attachment_ids},
)
attachments = list(found)
if {row.id for row in attachments} != set(attachment_ids):
raise AttachmentsNotFound()
if any(row.status == "failed" for row in attachments):
raise AttachmentFailed()
result = await db.execute(
text(
"""
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES (:id, :chat_id, :sender_id, :text)
RETURNING id, chat_id, sender_id, text, created_at
"""
),
{
"id": uuid4(),
"chat_id": chat_id,
"sender_id": sender_id,
"text": text_value,
},
)
message = dict(result.mappings().one())
if attachment_ids:
await db.execute(
text(
"""
INSERT INTO message_attachments
(message_id, attachment_id, position)
VALUES (:message_id, :attachment_id, :position)
"""
),
[
{
"message_id": message["id"],
"attachment_id": attachment_id,
"position": position,
}
for position, attachment_id in enumerate(attachment_ids)
],
)
await db.execute(
text(
"""
UPDATE idempotency_keys
SET message_id = :message_id
WHERE user_id = :user_id
AND chat_id = :chat_id
AND method = 'POST'
AND key = :key
"""
),
{**key_params, "message_id": message["id"]},
)
await db.commit()
message["attachment_ids"] = list(attachment_ids)
return message, False
stored = await db.execute(
text(
f"""
SELECT k.request_hash,
m.id, m.chat_id, m.sender_id, m.text, m.created_at,
{MESSAGE_ATTACHMENT_IDS}
FROM idempotency_keys AS k
JOIN messages AS m ON m.id = k.message_id
WHERE k.user_id = :user_id
AND k.chat_id = :chat_id
AND k.method = 'POST'
AND k.key = :key
"""
),
key_params,
)
row = stored.mappings().first()
await db.commit()
except Exception:
await db.rollback()
raise
if row is None:
# Not reachable while the key row and the message commit together.
raise RuntimeError("idempotency key without a message")
message = dict(row)
if message.pop("request_hash") != request_hash:
return None
return message, True
async def list_messages_keyset(
db: AsyncSession,
*,
chat_id: UUID,
limit: int,
cursor_created_at: datetime | None,
cursor_id: UUID | None,
) -> list[dict[str, Any]]:
query = f"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
{MESSAGE_ATTACHMENT_IDS}
FROM messages AS m
WHERE m.chat_id = :chat_id
"""
params: dict[str, Any] = {
"chat_id": chat_id,
"limit_plus_one": limit + 1,
}
if cursor_created_at is not None and cursor_id is not None:
query += """
AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id)
"""
params["cursor_created_at"] = cursor_created_at
params["cursor_id"] = cursor_id
query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one"
result = await db.execute(text(query), params)
return [dict(row) for row in result.mappings().all()]
# --- Attachments ------------------------------------------------------------
ATTACHMENT_COLUMNS = """
a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
a.sha256, a.status, a.error_code, a.object_key, a.variants, a.created_at, a.updated_at
"""
async def create_upload_intent(
db: AsyncSession,
*,
object_key: str,
attachment_id: UUID,
chat_id: UUID,
) -> None:
"""Committed BEFORE the S3 PUT: from now on the object can be found and
cleaned up even if this process dies right after the PUT."""
try:
await db.execute(
text(
"""
INSERT INTO upload_intents (object_key, attachment_id, chat_id)
VALUES (:object_key, :attachment_id, :chat_id)
"""
),
{
"object_key": object_key,
"attachment_id": attachment_id,
"chat_id": chat_id,
},
)
await db.commit()
except Exception:
await db.rollback()
raise
async def finalize_attachment(
db: AsyncSession,
*,
attachment_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
object_key: str,
) -> dict[str, Any] | None:
"""Delete the intent, insert queued metadata + outbox in ONE transaction.
Returns None if the intent is already gone: the janitor decided this
upload was abandoned and deleted the object, so the row must not be
created. The DELETE waits on the janitor's row lock, so the two can
never both succeed.
"""
try:
claimed = await db.execute(
text(
"DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1"
),
{"object_key": object_key},
)
if claimed.first() is None:
await db.rollback()
return None
job_id = uuid4()
final_key = f"originals/{attachment_id}/v1/original"
context = carrier()
result = await db.execute(
text(
f"""
INSERT INTO attachments AS a
(id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, width, height, object_key, staging_key, job_id, status,
traceparent, tracestate)
VALUES (:id, :chat_id, :owner_id, :filename, :content_type,
:size_bytes, :sha256, :width, :height, :final_key, :object_key, :job_id, 'queued',
:traceparent, :tracestate)
RETURNING {ATTACHMENT_COLUMNS}
"""
),
{
"id": attachment_id,
"chat_id": chat_id,
"owner_id": owner_id,
"filename": filename,
"content_type": content_type,
"size_bytes": size_bytes,
"sha256": sha256,
"width": width,
"height": height,
"object_key": object_key,
"final_key": final_key,
"job_id": job_id,
"traceparent": context.get("traceparent"),
"tracestate": context.get("tracestate"),
},
)
row = dict(result.mappings().one())
await db.execute(
text("""INSERT INTO outbox (job_id, attachment_id, attempt, traceparent, tracestate)
VALUES (:job_id, :id, 1, :traceparent, :tracestate)"""),
{"job_id": job_id, "id": attachment_id,
"traceparent": context.get("traceparent"), "tracestate": context.get("tracestate")},
)
await db.commit()
return row
except Exception:
await db.rollback()
raise
async def delete_upload_intent(db: AsyncSession, object_key: str) -> None:
try:
await db.execute(
text("DELETE FROM upload_intents WHERE object_key = :object_key"),
{"object_key": object_key},
)
await db.commit()
except Exception:
await db.rollback()
raise
async def get_accessible_attachment(
db: AsyncSession,
*,
attachment_id: UUID,
user_id: UUID,
) -> dict[str, Any] | None:
"""Only members of the attachment's chat see it; others get None (404)."""
result = await db.execute(
text(
f"""
SELECT {ATTACHMENT_COLUMNS}
FROM attachments AS a
JOIN chat_members AS cm
ON cm.chat_id = a.chat_id
AND cm.user_id = :user_id
WHERE a.id = :attachment_id
"""
),
{"attachment_id": attachment_id, "user_id": user_id},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def lock_upload_intent(
db: AsyncSession,
object_key: str,
) -> dict[str, Any] | None:
"""Lock the intent of one upload (waits for a finishing transaction);
the transaction stays open. None: no intent, nothing to compensate."""
result = await db.execute(
text(
"""
SELECT i.object_key,
EXISTS (SELECT 1 FROM attachments AS a
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
FROM upload_intents AS i
WHERE i.object_key = :object_key
FOR UPDATE OF i
"""
),
{"object_key": object_key},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def lock_stale_intent(
db: AsyncSession,
grace_seconds: int,
) -> dict[str, Any] | None:
"""Lock ONE intent older than the grace period; the transaction stays open.
SKIP LOCKED: an upload that is finishing right now holds the row lock in
finalize_attachment and is skipped; several replicas can run the janitor
at once without touching the same intent.
"""
result = await db.execute(
text(
"""
SELECT i.object_key,
EXISTS (SELECT 1 FROM attachments AS a
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
FROM upload_intents AS i
WHERE i.created_at < now() - make_interval(secs => :grace)
ORDER BY i.created_at
LIMIT 1
FOR UPDATE OF i SKIP LOCKED
"""
),
{"grace": grace_seconds},
)
row = result.mappings().first()
return dict(row) if row is not None else None
async def attachment_keys(db: AsyncSession) -> set[str]:
result = await db.execute(text("SELECT object_key FROM attachments"))
return {row.object_key for row in result}
async def intent_keys(db: AsyncSession) -> dict[str, datetime]:
result = await db.execute(
text("SELECT object_key, created_at FROM upload_intents")
)
return {row.object_key: row.created_at for row in result}
+162
View File
@@ -0,0 +1,162 @@
from __future__ import annotations
import base64
import binascii
import hashlib
import json
import secrets
from datetime import datetime, timezone
from uuid import UUID
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerifyMismatchError
password_hasher = PasswordHasher()
def hash_password(password: str) -> str:
return password_hasher.hash(password)
def verify_password(password_hash: str, password: str) -> bool:
try:
return password_hasher.verify(password_hash, password)
except (VerifyMismatchError, InvalidHashError):
return False
def generate_session_token() -> str:
return secrets.token_urlsafe(32)
def hash_session_token(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def parse_basic_authorization(value: str | None) -> tuple[str, str] | None:
if value is None or not value.startswith("Basic "):
return None
try:
encoded_credentials = value[len("Basic "):]
decoded = base64.b64decode(
encoded_credentials,
validate=True,
).decode("utf-8")
username, password = decoded.split(":", 1)
except (
ValueError,
UnicodeDecodeError,
binascii.Error,
):
return None
if not username or not username.isascii():
return None
return username, password
def _encode_cursor_payload(payload: dict[str, str]) -> str:
raw = json.dumps(
payload,
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii")
def _decode_cursor_payload(cursor: str) -> dict[str, str] | None:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
payload = json.loads(raw.decode("utf-8"))
if not isinstance(payload, dict):
return None
return payload
except (
ValueError,
TypeError,
UnicodeDecodeError,
binascii.Error,
):
return None
def _utc_timestamp(value: datetime) -> str:
if value.tzinfo is None:
raise ValueError("Cursor datetime must be timezone-aware.")
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
def encode_chat_cursor(
created_at: datetime,
chat_id: UUID,
) -> str:
return _encode_cursor_payload(
{
"kind": "chat",
"created_at": _utc_timestamp(created_at),
"id": str(chat_id),
}
)
def decode_chat_cursor(
cursor: str,
) -> tuple[datetime, UUID] | None:
payload = _decode_cursor_payload(cursor)
if payload is None or payload.get("kind") != "chat":
return None
try:
created_at = datetime.fromisoformat(
payload["created_at"].replace("Z", "+00:00")
)
chat_id = UUID(payload["id"])
except (KeyError, ValueError, AttributeError):
return None
return created_at, chat_id
def encode_message_cursor(
chat_id: UUID,
created_at: datetime,
message_id: UUID,
) -> str:
return _encode_cursor_payload(
{
"kind": "message",
"chat_id": str(chat_id),
"created_at": _utc_timestamp(created_at),
"id": str(message_id),
}
)
def decode_message_cursor(
cursor: str,
) -> tuple[UUID, datetime, UUID] | None:
payload = _decode_cursor_payload(cursor)
if payload is None or payload.get("kind") != "message":
return None
try:
chat_id = UUID(payload["chat_id"])
created_at = datetime.fromisoformat(
payload["created_at"].replace("Z", "+00:00")
)
message_id = UUID(payload["id"])
except (KeyError, ValueError, AttributeError):
return None
return chat_id, created_at, message_id
+71
View File
@@ -0,0 +1,71 @@
"""Entry point: uvicorn with a drain phase before the graceful shutdown.
Plain uvicorn stops accepting connections immediately on SIGTERM. Behind
HAProxy that turns every stop into a burst of refused connections until the
active health check notices. Instead, on the first signal:
t = 0 readiness -> 503, keep serving (Connection: close);
t = drain (3 s) HAProxy has marked the server DOWN (inter 1 s x fall 2),
uvicorn closes the listener and waits for in-flight
requests up to SHUTDOWN_TIMEOUT_SECONDS (8 s).
stop_grace_period in compose.yaml (15 s) is larger than 3 + 8 s.
A second signal skips the drain phase.
"""
import logging
import threading
import uvicorn
from app.config import settings
from app.lifecycle import draining
logger = logging.getLogger("messenger.server")
class DrainingServer(uvicorn.Server):
def handle_exit(self, sig, frame) -> None:
if draining.is_set():
super().handle_exit(sig, frame)
return
draining.set()
logger.warning(
"signal %s: draining for %.1f s before shutdown",
sig,
settings.shutdown_drain_seconds,
)
# Set should_exit directly instead of super().handle_exit(): uvicorn
# would re-raise the captured SIGTERM after shutdown and the process
# would exit with 143 instead of a clean 0.
timer = threading.Timer(
settings.shutdown_drain_seconds,
self._begin_shutdown,
)
timer.daemon = True
timer.start()
def _begin_shutdown(self) -> None:
logger.warning("drain finished: closing listener")
self.should_exit = True
def main() -> None:
config = uvicorn.Config(
"app.main:app",
host=settings.app_host,
port=settings.app_port,
proxy_headers=False,
# Longer than HAProxy "timeout http-keep-alive 10s": the LB closes an
# idle connection first, so it never reuses one the API just closed.
timeout_keep_alive=30,
timeout_graceful_shutdown=settings.shutdown_timeout_seconds,
log_level="info",
access_log=False, # raw URL can contain filename; safe events are exported instead
)
DrainingServer(config).run()
if __name__ == "__main__":
main()
+432
View File
@@ -0,0 +1,432 @@
"""Business rules for MTUSI Messenger Lab4 (users, chats, messages)."""
from __future__ import annotations
import asyncio
import hashlib
import json
import re
from datetime import datetime, timedelta, timezone
from typing import Any
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app import repositories
from app.cache import REVOKED_SESSION, session_cache
from app.config import settings
from app.errors import ApiError
from app.security import (
decode_chat_cursor,
decode_message_cursor,
encode_chat_cursor,
encode_message_cursor,
generate_session_token,
hash_password,
hash_session_token,
parse_basic_authorization,
verify_password,
)
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _to_user(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": row["id"],
"username": row["username"],
"display_name": row["display_name"],
"created_at": row["created_at"],
}
def _to_chat(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": row["id"],
"title": row["title"],
"member_ids": row["member_ids"],
"created_at": row["created_at"],
}
def _to_message(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": row["id"],
"chat_id": row["chat_id"],
"sender_id": row["sender_id"],
"text": row["text"],
"created_at": row["created_at"],
"attachment_ids": list(row.get("attachment_ids") or []),
}
def _bearer_unauthorized() -> ApiError:
return ApiError(
401,
"unauthorized",
"Authentication is required.",
headers={
"WWW-Authenticate": 'Bearer realm="messenger"',
},
)
def _basic_unauthorized() -> ApiError:
return ApiError(
401,
"unauthorized",
"Invalid credentials.",
headers={
"WWW-Authenticate": (
'Basic realm="messenger", charset="UTF-8"'
),
},
)
# Lab3 contract: ASCII [A-Za-z0-9._:-]{1,128}.
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def _invalid_request(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def _not_found() -> ApiError:
return ApiError(404, "not_found", "Object not found.")
async def register_user(
db: AsyncSession,
*,
username: str,
display_name: str,
password: str,
) -> dict[str, Any]:
# Argon2 is CPU-bound (~0.5-1 s); off the event loop so /health/ready keeps answering.
password_hash = await asyncio.to_thread(hash_password, password)
user = await repositories.create_user(
db,
username=username,
display_name=display_name,
password_hash=password_hash,
)
if user is None:
raise ApiError(
409,
"conflict",
"Username is already taken.",
)
return _to_user(user)
async def create_user_session(
db: AsyncSession,
authorization: str | None,
) -> dict[str, Any]:
credentials = parse_basic_authorization(authorization)
if credentials is None:
raise _basic_unauthorized()
username, password = credentials
user = await repositories.get_user_by_username(db, username)
if user is None or not await asyncio.to_thread(
verify_password,
user["password_hash"],
password,
):
raise _basic_unauthorized()
token = generate_session_token()
expires_at = _utc_now() + timedelta(
seconds=settings.session_ttl_seconds
)
await repositories.create_session(
db,
user_id=user["id"],
token_hash=hash_session_token(token),
expires_at=expires_at,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires_at,
"user": _to_user(user),
}
async def current_user_from_bearer_token(
db: AsyncSession,
token: str,
) -> dict[str, Any]:
token_hash = hash_session_token(token)
cached = await session_cache.get(token_hash)
if cached is REVOKED_SESSION:
raise _bearer_unauthorized()
if cached is not None:
return cached
# Cache miss or cache unavailable: PostgreSQL decides.
session_user = await repositories.get_active_session_user(db, token_hash)
if session_user is None:
raise _bearer_unauthorized()
user = _to_user(session_user)
await session_cache.put(token_hash, user, session_user["expires_at"])
return user
async def logout_current_session(
db: AsyncSession,
token: str,
) -> None:
token_hash = hash_session_token(token)
revoked = await repositories.revoke_active_session(db, token_hash)
if not revoked:
raise _bearer_unauthorized()
# Invalidate after COMMIT in the source of truth.
await session_cache.revoke(token_hash)
async def create_chat(
db: AsyncSession,
*,
title: str,
member_ids: list[UUID],
current_user: dict[str, Any],
) -> dict[str, Any]:
unique_member_ids = set(member_ids)
if len(unique_member_ids) != len(member_ids):
raise _invalid_request(
"member_ids must not contain duplicates."
)
if current_user["id"] not in unique_member_ids:
raise _invalid_request(
"The current user must be included in member_ids."
)
existing_ids = await repositories.user_ids_that_exist(
db,
unique_member_ids,
)
if existing_ids != unique_member_ids:
raise _invalid_request(
"One or more member_ids do not exist."
)
chat = await repositories.create_chat_with_members(
db,
title=title,
member_ids=member_ids,
)
return _to_chat(chat)
async def get_chat(
db: AsyncSession,
*,
chat_id: UUID,
current_user: dict[str, Any],
) -> dict[str, Any]:
chat = await repositories.get_accessible_chat(
db,
chat_id=chat_id,
user_id=current_user["id"],
)
if chat is None:
raise _not_found()
return _to_chat(chat)
async def list_chats(
db: AsyncSession,
*,
limit: int,
cursor: str | None,
current_user: dict[str, Any],
) -> dict[str, Any]:
cursor_created_at: datetime | None = None
cursor_id: UUID | None = None
if cursor is not None:
decoded = decode_chat_cursor(cursor)
if decoded is None:
raise _invalid_request("Invalid cursor.")
cursor_created_at, cursor_id = decoded
rows = await repositories.list_chats_for_user(
db,
user_id=current_user["id"],
limit=limit,
cursor_created_at=cursor_created_at,
cursor_id=cursor_id,
)
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
next_cursor = encode_chat_cursor(
last["created_at"],
last["id"],
)
return {
"items": [_to_chat(row) for row in page],
"next_cursor": next_cursor,
}
def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str:
"""Hash of the validated body: key order and whitespace do not matter,
a missing attachment_ids equals []. Without attachments the canonical
form is the lab3 one, so keys stored by lab3 still replay correctly."""
body: dict[str, Any] = {"text": text_value}
if attachment_ids:
body["attachment_ids"] = [str(item) for item in attachment_ids]
canonical = json.dumps(
body,
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
async def create_message(
db: AsyncSession,
*,
chat_id: UUID,
text_value: str,
attachment_ids: list[UUID],
idempotency_key: str | None,
current_user: dict[str, Any],
) -> tuple[dict[str, Any], bool]:
"""Send a message once per (user, POST, chat, Idempotency-Key).
Returns (message, replayed). A replay re-checks the session (done by
require_user) and chat membership before returning the stored result.
"""
if idempotency_key is None or not IDEMPOTENCY_KEY_PATTERN.fullmatch(
idempotency_key
):
raise _invalid_request(
"Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}."
)
chat = await repositories.get_accessible_chat(
db,
chat_id=chat_id,
user_id=current_user["id"],
)
if chat is None:
raise _not_found()
try:
result = await repositories.create_message_idempotent(
db,
chat_id=chat_id,
sender_id=current_user["id"],
text_value=text_value,
attachment_ids=attachment_ids,
idempotency_key=idempotency_key,
request_hash=_request_fingerprint(text_value, attachment_ids),
ttl_seconds=settings.idempotency_ttl_seconds,
)
except repositories.AttachmentsNotFound:
raise _not_found() from None
except repositories.AttachmentFailed:
raise ApiError(409, "conflict", "Failed attachment cannot be attached.") from None
if result is None:
raise ApiError(
409,
"conflict",
"Idempotency-Key was already used with a different body.",
)
message, replayed = result
return _to_message(message), replayed
async def list_messages(
db: AsyncSession,
*,
chat_id: UUID,
limit: int,
cursor: str | None,
current_user: dict[str, Any],
) -> dict[str, Any]:
chat = await repositories.get_accessible_chat(
db,
chat_id=chat_id,
user_id=current_user["id"],
)
if chat is None:
raise _not_found()
cursor_created_at: datetime | None = None
cursor_id: UUID | None = None
if cursor is not None:
decoded = decode_message_cursor(cursor)
if decoded is None:
raise _invalid_request("Invalid cursor.")
cursor_chat_id, cursor_created_at, cursor_id = decoded
if cursor_chat_id != chat_id:
raise _invalid_request(
"Cursor belongs to another chat."
)
rows = await repositories.list_messages_keyset(
db,
chat_id=chat_id,
limit=limit,
cursor_created_at=cursor_created_at,
cursor_id=cursor_id,
)
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
next_cursor = encode_message_cursor(
chat_id,
last["created_at"],
last["id"],
)
return {
"items": [_to_message(row) for row in page],
"next_cursor": next_cursor,
}
+189
View File
@@ -0,0 +1,189 @@
"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
One long-lived aiobotocore client per API process, path-style addressing,
short connect/read timeouts, at most 2 attempts and an overall deadline
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
fast 503 on the attachment endpoints only. Any client-side or
network error is raised as StorageUnavailable; a missing key is
ObjectMissing. Object keys are generated by the server (see services.py);
the user's filename never becomes part of a key.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import AsyncExitStack
from dataclasses import dataclass
from typing import Any, BinaryIO
from functools import wraps
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from app.config import settings
from app.telemetry import span
def observed(operation):
"""Export operation/outcome only, never S3 URL, key or credentials."""
def decorate(method):
@wraps(method)
async def run(*args, **kwargs):
with span("s3." + operation, {"rpc.system": "aws-api", "rpc.method": operation}):
return await method(*args, **kwargs)
return run
return decorate
logger = logging.getLogger("messenger.storage")
DOWNLOAD_CHUNK = 64 * 1024
class StorageUnavailable(Exception):
"""S3 is unreachable, timed out or refused the request."""
class ObjectMissing(Exception):
"""The key does not exist in the bucket."""
@dataclass
class StoredObject:
key: str
size: int
last_modified: Any
class ObjectStorage:
def __init__(self) -> None:
self._stack: AsyncExitStack | None = None
self._client = None
self.bucket = settings.s3_bucket
async def start(self) -> None:
self._stack = AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(
session.create_client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id,
aws_secret_access_key=settings.s3_secret_access_key,
config=AioConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
retries={"max_attempts": 2, "mode": "standard"},
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
),
)
)
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
self._client = None
@staticmethod
def _deadline(size: int = 0):
return asyncio.timeout(
settings.s3_operation_timeout_seconds
+ size / settings.s3_put_min_bytes_per_second
)
def _require_client(self):
if self._client is None:
raise StorageUnavailable("S3 client is not started")
return self._client
@observed("PutObject")
async def put(
self,
key: str,
body: BinaryIO,
size: int,
content_type: str,
) -> None:
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
itself rejects bytes that were damaged on the way."""
client = self._require_client()
body.seek(0)
try:
async with self._deadline(size):
await client.put_object(
Bucket=self.bucket,
Key=key,
Body=body,
ContentLength=size,
ContentType=content_type,
)
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
@observed("GetObject")
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
"""GET the object; S3 errors surface here, before any byte is sent."""
client = self._require_client()
try:
async with self._deadline():
response = await client.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
raise ObjectMissing(key) from exc
raise StorageUnavailable(type(exc).__name__) from exc
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
body = response["Body"]
async def chunks() -> AsyncIterator[bytes]:
try:
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
yield chunk
finally:
body.close()
return chunks(), int(response["ContentLength"])
@observed("DeleteObject")
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key succeeds in S3."""
client = self._require_client()
try:
async with self._deadline():
await client.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
@observed("HeadObject")
async def exists(self, key: str) -> bool:
client = self._require_client()
try:
async with self._deadline():
await client.head_object(Bucket=self.bucket, Key=key)
return True
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
return False
raise StorageUnavailable(type(exc).__name__) from exc
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
client = self._require_client()
paginator = client.get_paginator("list_objects_v2")
try:
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
for item in page.get("Contents", []):
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
storage = ObjectStorage()
+245
View File
@@ -0,0 +1,245 @@
"""Explicit, bounded OTLP instrumentation; no body/headers/SQL auto-capture.
Only this module's safe event logger is exported. Existing application and
third-party loggers are deliberately not attached to the OTLP handler.
"""
from __future__ import annotations
from contextlib import contextmanager
import json
import logging
import os
import signal
import time
from uuid import uuid4
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.trace import SpanKind, Status, StatusCode
meter = metrics.get_meter("messenger", "1.6.0")
tracer = trace.get_tracer("messenger", "1.6.0")
http_requests = meter.create_counter("messenger.http.requests")
http_duration = meter.create_histogram("messenger.http.duration", unit="s")
jobs_processed = meter.create_counter("messenger.jobs.processed")
jobs_duration = meter.create_histogram("messenger.jobs.duration", unit="s")
export_failures = meter.create_counter("messenger.telemetry.export.failures")
dropped_records = meter.create_counter("messenger.telemetry.dropped.records")
_providers: list = []
_service = "messenger"
_event_logger = logging.getLogger("messenger.telemetry.events")
_event_logger.propagate = False
_event_logger.setLevel(logging.INFO)
_ALLOWED_FIELDS = {
"event", "request_id", "trace_id", "span_id", "method", "route", "status_code",
"outcome", "duration_seconds", "job_type", "job_id", "attachment_id", "attempt",
"exception_type", "operation", "queue", "count",
}
class _NoHttpRetry:
# Verified against pinned OTel 1.28.2. Retry/backpressure is the Collector's
# responsibility; a failing SDK export gets one HTTP attempt, <=2 s socket
# timeout, then discards its bounded batch. No request waits on export.
@staticmethod
def _retryable(_response):
return False
class _SpanExporter(_NoHttpRetry, OTLPSpanExporter):
def export(self, spans):
try:
result = super().export(spans)
except Exception:
result = SpanExportResult.FAILURE
if result != SpanExportResult.SUCCESS:
export_failures.add(1, {"signal": "traces"})
dropped_records.add(len(spans), {"signal": "traces"})
return result
class _LogExporter(_NoHttpRetry, OTLPLogExporter):
def export(self, batch):
try:
result = super().export(batch)
except Exception:
result = LogExportResult.FAILURE
if result != LogExportResult.SUCCESS:
export_failures.add(1, {"signal": "logs"})
dropped_records.add(len(batch), {"signal": "logs"})
return result
class _MetricExporter(_NoHttpRetry, OTLPMetricExporter):
def export(self, metrics_data, timeout_millis=10000, **kwargs):
try:
result = super().export(metrics_data, timeout_millis, **kwargs)
except Exception:
result = MetricExportResult.FAILURE
if result != MetricExportResult.SUCCESS:
export_failures.add(1, {"signal": "metrics"})
return result
class _JsonFormatter(logging.Formatter):
def format(self, record):
value = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)),
"severity": record.levelname, "service.name": _service,
}
value.update({key: getattr(record, key) for key in _ALLOWED_FIELDS
if hasattr(record, key)})
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def configure(service: str, instance_id: str | None = None) -> None:
"""One resource/provider set per process. OTEL_SDK_DISABLED supports off runs."""
global _service
if _providers:
return
_service = service
console = logging.StreamHandler()
console.setFormatter(_JsonFormatter())
_event_logger.addHandler(console)
if os.getenv("OTEL_SDK_DISABLED", "false").lower() == "true":
return
resource = Resource({
"service.name": service,
"service.version": os.getenv("APP_VERSION", "1.6.0"),
"service.instance.id": instance_id or uuid4().hex,
"deployment.environment.name": "lab",
})
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
trace_provider = TracerProvider(resource=resource)
trace_provider.add_span_processor(BatchSpanProcessor(
_SpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
max_queue_size=512, max_export_batch_size=64,
schedule_delay_millis=1000, export_timeout_millis=2000,
))
trace.set_tracer_provider(trace_provider)
log_provider = LoggerProvider(resource=resource)
log_provider.add_log_record_processor(BatchLogRecordProcessor(
_LogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
max_queue_size=512, max_export_batch_size=64,
schedule_delay_millis=1000, export_timeout_millis=2000,
))
set_logger_provider(log_provider)
_event_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
reader = PeriodicExportingMetricReader(
_MetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
export_interval_millis=5000, export_timeout_millis=2500,
)
# Boundaries are seconds. Quantiles are computed over merged replica buckets.
boundaries = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
View(instrument_type=metrics.Histogram,
aggregation=ExplicitBucketHistogramAggregation(boundaries)),
])
metrics.set_meter_provider(metric_provider)
_providers.extend((metric_provider, log_provider, trace_provider))
def shutdown() -> None:
"""Flush bounded background buffers on normal termination (not SIGKILL)."""
for provider in _providers:
provider.shutdown()
_providers.clear()
def install_termination_handler() -> None:
"""Turn worker/publisher SIGTERM into cancellation so their finally runs."""
import asyncio
loop = asyncio.get_running_loop()
task = asyncio.current_task()
if task is not None:
try:
loop.add_signal_handler(signal.SIGTERM, task.cancel)
except NotImplementedError: # Windows host; Compose containers use POSIX.
signal.signal(signal.SIGTERM, lambda *_: loop.call_soon_threadsafe(task.cancel))
def emit(event: str, *, severity: int = logging.INFO, **fields) -> None:
"""Call with a source-defined event and technical fields, never free text."""
current = trace.get_current_span().get_span_context()
attrs = {key: value for key, value in fields.items()
if key in _ALLOWED_FIELDS and value is not None}
attrs.update(event=event, trace_id=f"{current.trace_id:032x}", span_id=f"{current.span_id:016x}")
_event_logger.log(severity, event, extra=attrs)
@contextmanager
def span(name: str, attributes=None, kind=SpanKind.INTERNAL, parent=None):
"""Safe span helper: never export exception messages or stack traces."""
with tracer.start_as_current_span(
name, attributes=attributes, kind=kind, context=parent,
record_exception=False, set_status_on_exception=False,
) as active:
try:
yield active
except BaseException as exc:
active.set_attribute("exception_type", type(exc).__name__)
active.set_status(Status(StatusCode.ERROR))
raise
class HttpTelemetryMiddleware:
"""Outer ASGI wrapper sees final status, streaming duration and 500 responses."""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
from starlette.requests import Request
from app.errors import common_headers, incoming_request_id
request = Request(scope)
request.state.request_id = incoming_request_id(request)
method = scope.get("method", "OTHER")
if method not in {"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}:
method = "OTHER"
status = 500
started = time.perf_counter()
# Only W3C propagation fields are consumed; headers are never exported.
parent = extract(dict(request.headers))
with span("HTTP request", kind=SpanKind.SERVER, parent=parent) as active:
async def observed_send(message):
nonlocal status
if message["type"] == "http.response.start":
status = message["status"]
headers = dict(message.get("headers", []))
for key, value in common_headers(request).items():
headers[key.lower().encode()] = value.encode()
headers[b"x-trace-id"] = f"{active.get_span_context().trace_id:032x}".encode()
message = {**message, "headers": list(headers.items())}
await send(message)
try:
await self.app(scope, receive, observed_send)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
duration = time.perf_counter() - started
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
active.update_name(f"{method} {route}")
active.set_attributes({**attrs, "http.response.status_code": status})
if status >= 500:
active.set_status(Status(StatusCode.ERROR))
http_requests.add(1, attrs)
http_duration.record(duration, attrs)
emit("http.request.complete", severity=logging.ERROR if status >= 500 else logging.INFO,
request_id=request.state.request_id, method=method, route=route,
status_code=status, outcome="error" if status >= 500 else "success",
duration_seconds=duration)
+40
View File
@@ -0,0 +1,40 @@
"""W3C context carried durably alongside a job, never its HTTP credentials.
The upload transaction stores ``carrier()``; publisher and retries restore it
with ``parent_context``. Only a new producer/consumer span changes the parent
span ID. Context-less legacy jobs deliberately start a new trace.
"""
from __future__ import annotations
from collections.abc import Mapping
from opentelemetry.context import Context
from opentelemetry.trace import get_current_span
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
_PROPAGATOR = TraceContextTextMapPropagator()
_FIELDS = ("traceparent", "tracestate")
def carrier() -> dict[str, str]:
"""Return the active span's W3C fields, or {} outside a valid span."""
headers: dict[str, str] = {}
_PROPAGATOR.inject(headers)
return {name: headers[name] for name in _FIELDS if headers.get(name)}
def parent_context(row_or_job: Mapping | None) -> Context:
"""Extract nullable DB/envelope fields without inheriting ambient context.
W3C parsing validates version, nonzero trace/span IDs, flags and tracestate.
Malformed or missing parent IDs become a new root, even if a publisher's
unrelated maintenance span happens to be active.
"""
headers: dict[str, str] = {}
if isinstance(row_or_job, Mapping):
for name in _FIELDS:
value = row_or_job.get(name)
if isinstance(value, str) and value:
headers[name] = value
context = _PROPAGATOR.extract(headers, context=Context())
return context if get_current_span(context).get_span_context().is_valid else Context()
+266
View File
@@ -0,0 +1,266 @@
"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
from pathlib import Path
import sys
import tempfile
import time
from uuid import UUID, uuid4
from sqlalchemy import text
from app.config import settings
from app.db import SessionLocal, engine
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
arm_from_env, broker_channel, crash_if_armed,
event, parse_envelope, retry_or_fail)
from app.storage import ObjectMissing, storage
from app.telemetry import (configure, emit, install_termination_handler, jobs_duration,
jobs_processed, shutdown, span)
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import advanced
from app.trace_context import carrier, parent_context
logger = logging.getLogger("messenger.worker")
class PermanentFailure(Exception):
pass
async def claim(job: dict) -> dict | None:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT * FROM attachments WHERE id=:id FOR UPDATE
"""), {"id": UUID(job["attachment_id"])})
row = result.mappings().first()
if row is None or row["job_id"] != UUID(job["job_id"]) or (
row["pipeline_version"] != job["pipeline_version"] or
row["attempt"] != job["attempt"] or row["status"] != "queued"
):
await db.rollback()
return None
# An early/manual duplicate cannot bypass the durable retry date.
result = await db.execute(text("""
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
traceparent=:traceparent,tracestate=:tracestate,
updated_at=now()
WHERE id=:id AND next_attempt_at<=now() RETURNING *
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS,
"traceparent": carrier().get("traceparent"), "tracestate": carrier().get("tracestate")})
claimed = result.mappings().first()
if claimed is None:
await db.rollback()
return None
claimed = dict(claimed)
await event(db, claimed, "processing")
await db.commit()
return claimed
async def generate(source: Path, directory: Path) -> list[dict]:
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.image_worker", str(source), str(directory),
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
)
try:
try:
async with asyncio.timeout(IMAGE_TIMEOUT):
code = await process.wait()
except TimeoutError:
raise PermanentFailure("image_timeout") from None
if code == 21:
raise PermanentFailure("image_resource_limit")
if code != 0:
raise PermanentFailure("invalid_image")
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
finally:
# Includes cancellation of the entire job. Native image code cannot
# continue using CPU/RAM after its parent timed out.
if process.returncode is None:
process.kill()
await process.wait()
async def build_objects(row: dict) -> list[dict]:
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
directory = Path(temporary)
source = directory / "source"
if not row["staging_key"] or row["staging_deleted"]:
raise PermanentFailure("source_missing")
try:
chunks, declared_length = await storage.open_download(row["staging_key"])
except ObjectMissing:
raise PermanentFailure("source_missing") from None
digest = hashlib.sha256()
size = 0
try:
with source.open("wb") as output:
async for chunk in chunks:
size += len(chunk)
if size > settings.max_upload_bytes:
raise PermanentFailure("source_too_large")
digest.update(chunk)
output.write(chunk)
finally:
await chunks.aclose()
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
raise PermanentFailure("source_checksum_mismatch")
with span("image.generate"):
variants = await generate(source, directory)
# API never publishes directly to this final prefix. Repeating one
# v1 job writes the same bytes to the same four keys, never appends.
with source.open("rb") as body:
await storage.put(row["object_key"], body, size, row["content_type"])
prefix = row["object_key"].rsplit("/", 1)[0]
for variant in variants:
filename = variant.pop("filename")
key = f"{prefix}/{filename}"
with (directory / filename).open("rb") as body:
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
variant["object_key"] = key
await crash_if_armed("crash_after_objects_before_commit")
return variants
async def finish(row: dict, *, variants=None, code: str | None = None,
permanent: bool = False) -> bool:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT * FROM attachments WHERE id=:id AND status='processing'
AND lease_token=:token AND lease_until>now() FOR UPDATE
"""), {"id": row["id"], "token": row["lease_token"]})
current = result.mappings().first()
if current is None:
# Lease recovery or explicit operator action already owns a
# newer attempt. Never overwrite its state with an old worker.
await db.rollback()
return False
if code is not None:
await retry_or_fail(db, current, code, permanent=permanent)
else:
await db.execute(text("""
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
WHERE id=:id
"""), {"id": row["id"], "variants": json.dumps(variants)})
await event(db, current, "ready")
await db.commit()
if code is None:
advanced.observe_ready(current)
return True
async def _handle(message) -> str:
try:
job = parse_envelope(message.body)
except (ValueError, TypeError, KeyError, AttributeError):
logger.warning("invalid envelope sent to durable DLQ")
await message.reject(requeue=False)
return "invalid"
row = await claim(job)
if row is None:
# A ready/failed/stale attempt is a durable no-op. A duplicate of
# active processing is covered by the DB lease recovery, so it too
# can be acknowledged without running a second concurrent decoder.
await message.ack()
return "duplicate"
outcome = "error"
try:
async with asyncio.timeout(TOTAL_TIMEOUT):
variants = await build_objects(row)
except PermanentFailure as exc:
await finish(row, code=str(exc), permanent=True)
except Exception as exc:
logger.warning("temporary processing failure attachment=%s type=%s",
row["id"], type(exc).__name__)
await finish(row, code="processing_failed")
else:
if await finish(row, variants=variants):
await crash_if_armed("crash_after_commit_before_ack")
logger.info("ready attachment=%s", row["id"])
outcome = "success"
else:
outcome = "duplicate"
# If finish failed, the exception propagates and the connection closes:
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
await message.ack()
return outcome
async def handle(message) -> None:
# Technical IDs in logs/spans only; metric dimensions stay low-cardinality.
job = {}
try:
job = parse_envelope(message.body)
except (ValueError, TypeError, KeyError, AttributeError):
pass
started = time.perf_counter()
outcome = "error"
with span("attachments process", kind=SpanKind.CONSUMER, parent=parent_context(job), attributes={
"messaging.system": "rabbitmq", "messaging.destination.name": "attachments",
"messaging.operation": "process", "job_type": "attachment.process.v1",
}) as active:
try:
advanced.busy = 1
outcome = await _handle(message)
finally:
advanced.busy = 0
duration = time.perf_counter() - started
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
jobs_processed.add(1, attrs)
if outcome in {"success", "error"}:
jobs_duration.record(duration, attrs)
active.set_attribute("outcome", outcome)
if outcome in {"error", "invalid"}:
active.set_status(Status(StatusCode.ERROR))
for name in ("job_id", "attachment_id", "attempt"):
if name in job:
active.set_attribute(name, job[name])
emit("job.complete", severity=logging.WARNING if outcome == "error" else logging.INFO,
**attrs, job_id=job.get("job_id"), attachment_id=job.get("attachment_id"),
attempt=job.get("attempt"), duration_seconds=duration)
async def main() -> None:
configure("messenger-worker")
install_termination_handler()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
for name in ("aio_pika", "aiormq"):
logging.getLogger(name).setLevel(logging.CRITICAL)
await storage.start()
connection = None
try:
await arm_from_env("WORKER_FAULT")
while True:
try:
connection, _, queue = await broker_channel()
async with queue.iterator() as deliveries:
async for message in deliveries:
await handle(message)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning("worker reconnect: %s", type(exc).__name__)
if connection:
await connection.close()
connection = None
await asyncio.sleep(1)
finally:
if connection:
await connection.close()
await storage.close()
await engine.dispose()
await asyncio.to_thread(shutdown)
if __name__ == "__main__":
try:
asyncio.run(main())
except asyncio.CancelledError:
pass # SIGTERM has run finally/SDK flush; exit cleanly.
+455
View File
@@ -0,0 +1,455 @@
#file: noinspection ComposeUnknownValues
# Lab6: API x2 -> durable S3 staging + PostgreSQL outbox -> publisher
# -> RabbitMQ -> worker -> final private S3 objects + PostgreSQL metadata.
# Start: see REPORT.md, "Запуск из чистого clone".
name: messenger-lab6
x-api-env: &api-env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
SESSION_STORE: valkey
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_CACHE_TTL_SECONDS: "60"
CACHE_TIMEOUT_SECONDS: "0.25"
CACHE_MAX_CONNECTIONS: "20"
CACHE_RETRY_AFTER_SECONDS: "5"
DB_POOL_SIZE: "10"
DB_MAX_OVERFLOW: "5"
DB_POOL_TIMEOUT_SECONDS: "3"
DB_CONNECT_TIMEOUT_SECONDS: "3"
DB_STATEMENT_TIMEOUT_MS: "5000"
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
SHUTDOWN_DRAIN_SECONDS: "3"
SHUTDOWN_TIMEOUT_SECONDS: "8"
# Lab4: S3 with the API's own restricted credentials (not admin).
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_OBJECT_PREFIX: originals/
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: "2"
S3_READ_TIMEOUT_SECONDS: "5"
S3_OPERATION_TIMEOUT_SECONDS: "6"
S3_PUT_MIN_BYTES_PER_SECOND: "1048576"
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
UPLOAD_SPOOL_MEMORY_BYTES: "1048576"
UPLOAD_TIMEOUT_SECONDS: "60"
MAX_CONCURRENT_UPLOADS: "8"
UPLOAD_SLOT_WAIT_SECONDS: "10"
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}"
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-30}"
JOB_TOTAL_TIMEOUT_SECONDS: "${JOB_TOTAL_TIMEOUT_SECONDS:-90}"
WORKER_MEMORY_MB: "${WORKER_MEMORY_MB:-768}"
JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-120}"
RETRY_BASE_SECONDS: "${RETRY_BASE_SECONDS:-1}"
STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-604800}"
PUBLISHER_FAULT: "${PUBLISHER_FAULT:-}"
WORKER_FAULT: "${WORKER_FAULT:-}"
# API fault-injection is disabled during normal operation.
FAULT_INJECTION: "${FAULT_INJECTION:-}"
OTEL_SDK_DISABLED: "${OTEL_SDK_DISABLED:-false}"
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: service.version=1.6.0,deployment.environment.name=lab
OTEL_METRIC_EXPORT_INTERVAL: "5000"
services:
db:
image: postgres:16.4-alpine
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 3s
retries: 20
start_period: 5s
networks: [backend]
stop_grace_period: 30s
restart: unless-stopped
deploy:
resources:
limits: { cpus: "1.0", memory: 512M }
# Session cache only (lab2). PostgreSQL stays the source of truth, so the
# cache is intentionally not persisted. It is shared by all API replicas:
# a logout on one replica writes the "revoked" tombstone seen by all.
valkey:
image: valkey/valkey:8.0.1-alpine
command:
- sh
- -c
- >-
exec valkey-server
--requirepass "$$(cat /run/secrets/valkey_password)"
--save ""
--appendonly no
--maxmemory 64mb
--maxmemory-policy allkeys-lru
secrets: [valkey_password]
healthcheck:
test:
- CMD-SHELL
- valkey-cli -a "$$(cat /run/secrets/valkey_password)" --no-auth-warning ping | grep -q PONG
interval: 2s
timeout: 3s
retries: 20
networks: [backend]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
# S3: SeaweedFS all-in-one. No "ports": reachable only from the internal
# backend network; its admin web UIs are bound to 127.0.0.1 inside the
# container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume.
s3:
image: chrislusf/seaweedfs:3.97
entrypoint: ["/bin/sh", "/opt/lab/start.sh"]
environment:
# Soft limit of the Go heap: without it the GC lets SeaweedFS grow to
# the container limit under parallel 10 MiB PUTs (measured, REPORT.md).
GOMEMLIMIT: 600MiB
volumes:
- s3_data:/data
- ./infra/seaweedfs:/opt/lab:ro
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_access_key_id
- s3_secret_access_key
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"]
interval: 2s
timeout: 3s
retries: 30
start_period: 60s
networks: [backend]
stop_grace_period: 30s
restart: unless-stopped
deploy:
resources:
# 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed
# SeaweedFS to the limit and the kernel killed it (REPORT.md).
limits: { cpus: "1.0", memory: 1G }
# Idempotent bucket bootstrap + self-checks (private bucket, least
# privilege of the API key). The only container with the S3 admin key.
s3-init:
build: .
image: messenger-lab6-api:${APP_VERSION:-dev}
command: ["python", "scripts/s3_bootstrap.py"]
environment:
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_OBJECT_PREFIX: originals/
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_access_key_id
- s3_secret_access_key
depends_on:
s3: { condition: service_healthy }
networks: [backend]
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128M }
# Runs the migrations once, before any API replica starts (no race of
# replicas). A repeated run only prints SKIP.
migrate:
build: .
image: messenger-lab6-api:${APP_VERSION:-dev}
command: ["python", "scripts/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
db: { condition: service_healthy }
networks: [backend]
restart: "no"
deploy:
resources:
limits: { cpus: "0.5", memory: 128M }
api:
build: .
image: messenger-lab6-api:${APP_VERSION:-dev}
environment: *api-env
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
# No "ports": replicas are reachable only through the LB.
# No "container_name": the service is scaled by deploy.replicas / --scale.
networks: [edge, backend]
depends_on:
db: { condition: service_healthy }
valkey: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
healthcheck:
test:
- CMD
- python
- -c
- "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"
interval: 2s
timeout: 3s
retries: 2
start_period: 5s
stop_signal: SIGTERM
# > SHUTDOWN_DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (3 + 8 s).
stop_grace_period: 15s
restart: unless-stopped
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "0.5", memory: 512M }
# A stable nodename keeps durable queues in the same RabbitMQ database
# even after the container is re-created. No AMQP/management port is public.
broker:
image: rabbitmq:3.13.7-alpine
hostname: broker
entrypoint: ["/bin/sh", "-ec"]
command:
- 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server'
environment:
RABBITMQ_DEFAULT_USER: messenger
RABBITMQ_DEFAULT_VHOST: /
# Bound Erlang schedulers to this small container, rather than the
# host's core count; health-check CLI VMs need only one scheduler.
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
secrets: [broker_password]
volumes:
- broker_data:/var/lib/rabbitmq
networks: [backend]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 3s
timeout: 5s
retries: 20
start_period: 15s
stop_grace_period: 20s
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
publisher:
build: .
image: messenger-lab6-api:${APP_VERSION:-dev}
command: ["python", "-m", "app.jobs"]
environment:
<<: *api-env
OTEL_SERVICE_NAME: messenger-publisher
BROKER_URL_FILE: /run/secrets/broker_url
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
networks: [backend]
depends_on:
broker: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
worker:
build: .
image: messenger-lab6-api:${APP_VERSION:-dev}
command: ["python", "-m", "app.worker"]
environment:
<<: *api-env
OTEL_SERVICE_NAME: messenger-worker
BROKER_URL_FILE: /run/secrets/broker_url
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
networks: [backend]
depends_on:
broker: { condition: service_healthy }
migrate: { condition: service_completed_successfully }
s3-init: { condition: service_completed_successfully }
restart: unless-stopped
stop_grace_period: 10s
# One active job and one reserved delivery; image subprocess has its own
# address-space/time limits in addition to this container memory limit.
deploy:
resources:
limits: { cpus: "1.0", memory: 1G }
lb:
image: haproxy:3.0.11-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
# Backend state page for the operator, only on the host loopback.
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
networks: [edge]
depends_on:
api: { condition: service_healthy }
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
interval: 5s
timeout: 2s
retries: 3
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 128M }
# Telemetry is deliberately absent from API depends_on/readiness.
# No OTLP ports on the host: application traffic stays in backend.
otel-collector:
image: otel/opentelemetry-collector-contrib:0.111.0
command: ["--config=/etc/otelcol/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
networks: [backend]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
prometheus:
image: prom/prometheus:v2.55.0
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=512MB
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheus_data:/prometheus
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
networks: [backend, observability]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
loki:
image: grafana/loki:3.2.1
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"]
networks: [backend, observability]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 384M }
tempo:
image: grafana/tempo:2.6.1
environment:
# Bound the Go heap while searching many retained blocks on this small stand.
GOMEMLIMIT: 256MiB
# A new named volume needs write access for the local evaluation backend.
user: "0:0"
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"]
networks: [backend, observability]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "1.0", memory: 512M }
grafana:
image: grafana/grafana:11.2.2
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
networks: [backend, observability]
restart: unless-stopped
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
# Runs the public and own checks in Linux (needed on Windows: the hooks are
# POSIX .sh scripts that call the docker CLI). Profile "tools": not started
# by a plain "up". The Docker socket grants control over the Docker host —
# use it only on the lab stand.
tester:
profiles: [tools]
build:
context: .
dockerfile: student-tests/Dockerfile.tester
working_dir: /work
environment:
BASE_URL: http://lb:8080
COMPOSE_PROJECT: messenger-lab6
S3_ENDPOINT: http://s3:9000
volumes:
- ./:/work
- /var/run/docker.sock:/var/run/docker.sock
# backend: only for the "private bucket from outside the API" check.
networks: [edge, backend]
networks:
observability: {}
edge: {}
# PostgreSQL, Valkey and S3: no route outside, the LB is not attached.
backend:
internal: true
volumes:
postgres_data: {}
s3_data: {}
broker_data: {}
prometheus_data: {}
loki_data: {}
tempo_data: {}
grafana_data: {}
secrets:
postgres_password:
file: ./infra/secrets/postgres_password.txt
database_url:
file: ./infra/secrets/database_url.txt
valkey_password:
file: ./infra/secrets/valkey_password.txt
cache_url:
file: ./infra/secrets/cache_url.txt
s3_admin_access_key_id:
file: ./infra/secrets/s3_admin_access_key_id.txt
s3_admin_secret_access_key:
file: ./infra/secrets/s3_admin_secret_access_key.txt
s3_access_key_id:
file: ./infra/secrets/s3_access_key_id.txt
s3_secret_access_key:
file: ./infra/secrets/s3_secret_access_key.txt
broker_password:
file: ./infra/secrets/broker_password.txt
broker_url:
file: ./infra/secrets/broker_url.txt
+191
View File
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.14954320399965582",
"event": "http.request.complete",
"flags": "1",
"method": "POST",
"observed_timestamp": "1795252636588490258",
"outcome": "success",
"request_id": "observe-60a6c2da00814bf08cb3a801b8388ef4",
"route": "/api/v1/chats/{chat_id}/attachments",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "b33ff621673d",
"service_name": "messenger-api",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "7f43d8d88f960fff",
"status_code": "202",
"trace_id": "115550cd6d2c36ddf2bea96f48e89c6b"
},
"values": [
[
"1795252636588292864",
"http.request.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3571455,
"linesProcessedPerSecond": 18129,
"totalBytesProcessed": 18124,
"totalLinesProcessed": 92,
"execTime": 0.005075,
"queueTime": 0.000262,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 0,
"shards": 1,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 16192
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 1,
"totalChunksMatched": 1,
"totalBatches": 2,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 18124,
"headChunkLines": 92,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 16192,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 81728,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
+191
View File
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"attachment_id": "eb62cd3e-75a1-4dbe-8965-9e012b312282",
"attempt": "1",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.42506639099974564",
"event": "job.complete",
"flags": "1",
"job_id": "e1b6a557-f9fa-462c-ac79-1ab71f0b35bf",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795252637174010973",
"outcome": "success",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "4d9015b23ab54bde9d37384ddc4bf89b",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "c71f9d5977e9bd59",
"trace_id": "115550cd6d2c36ddf2bea96f48e89c6b"
},
"values": [
[
"1795252637173831168",
"job.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 79386,
"linesProcessedPerSecond": 422,
"totalBytesProcessed": 376,
"totalLinesProcessed": 2,
"execTime": 0.004736,
"queueTime": 0.000244,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 0,
"shards": 1,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 352
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 1,
"totalChunksMatched": 1,
"totalBatches": 2,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 376,
"headChunkLines": 2,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 352,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 138740,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
+6
View File
@@ -0,0 +1,6 @@
UTC: 2026-11-21T09:16:46Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
RESULT: PASS (exit 0) at 2026-11-21T09:16:46Z
+5
View File
@@ -0,0 +1,5 @@
UTC: 2026-11-21T09:16:45Z
Command: docker compose config --quiet
RESULT: PASS (exit 0) at 2026-11-21T09:16:46Z
+220
View File
@@ -0,0 +1,220 @@
Before public persistence — 2026-11-21T09:14:39Z
UTC: 2026-11-21T09:14:39Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
UTC: 2026-11-21T09:14:40Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:10:22.045070464Z finished=2026-11-21T09:10:21.647750535Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:10:18.318585259Z finished=2026-11-21T09:10:17.940277751Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:10:24.158928332Z finished=2026-11-21T09:10:23.907977242Z
/messenger-lab6-db-1 status=running started=2026-11-21T08:59:12.528963564Z finished=2026-11-21T08:58:31.826616128Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T08:51:52.621809468Z finished=0001-01-01T00:00:00Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:10:55.825987305Z finished=2026-11-21T09:10:55.547390235Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:09.569875679Z finished=2026-11-21T09:13:44.880718518Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T08:45:28.390651046Z finished=2026-11-21T08:45:27.109421727Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:13:45.19691868Z finished=2026-11-21T09:13:41.499885247Z
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
After public persistence — 2026-11-21T09:15:30Z
UTC: 2026-11-21T09:15:30Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
Before public failover — 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
After public failover, before API restoration — 2026-11-21T09:15:52Z
UTC: 2026-11-21T09:15:52Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
UTC: 2026-11-21T09:15:53Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=exited started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:15:40.679293151Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
After API restoration — 2026-11-21T09:16:00Z
UTC: 2026-11-21T09:16:00Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
UTC: 2026-11-21T09:16:01Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:54.07308443Z finished=2026-11-21T09:15:40.679293151Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
+187
View File
@@ -0,0 +1,187 @@
{
"captured_at": "2026-11-21T09:17:47.509294Z",
"live_worker_capacity": 1.0,
"panels": 24,
"promql_checks": [
{
"panel": "Business API requests / second",
"query": "sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m]))",
"series": 1
},
{
"panel": "Business API 5xx / all responses",
"query": "(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "API mean duration (seconds)",
"query": "sum(rate(messenger_http_duration_seconds_sum{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_duration_seconds_count{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "Worker attempt outcomes / second",
"query": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
"series": 3
},
{
"panel": "Worker mean attempt duration (seconds)",
"query": "sum(rate(messenger_jobs_duration_seconds_sum{service_name=\"messenger-worker\"}[1m])) / clamp_min(sum(rate(messenger_jobs_duration_seconds_count{service_name=\"messenger-worker\"}[1m])), 1e-9)",
"series": 1
},
{
"panel": "Worker completed attempts (counter)",
"query": "sum by (outcome) (messenger_jobs_processed_total{service_name=\"messenger-worker\"})",
"series": 3
},
{
"panel": "Structured application events (expand for trace_id / span_id / request_id)",
"query": "{service_name=~\"messenger-api|messenger-worker|messenger-publisher\"}",
"series": 1343
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "API duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Worker duration p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Durable unfinished jobs (count)",
"query": "max(messenger_queue_depth{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "Oldest unfinished job / observer sample age (seconds)",
"query": "max(messenger_queue_oldest_age_seconds{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "Oldest unfinished job / observer sample age (seconds)",
"query": "max(messenger_queue_sample_age_seconds{service_name=\"messenger-publisher\"})",
"series": 1
},
{
"panel": "DB pool used / capacity (connections)",
"query": "messenger_db_pool_used",
"series": 13
},
{
"panel": "DB pool used / capacity (connections)",
"query": "messenger_db_pool_capacity",
"series": 13
},
{
"panel": "DB acquisitions in progress (requests)",
"query": "messenger_db_pool_waiting",
"series": 13
},
{
"panel": "Worker saturation (busy / capacity)",
"query": "(sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)) / (sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0))",
"series": 1
},
{
"panel": "Worker busy / capacity (jobs)",
"query": "sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
"series": 1
},
{
"panel": "Worker busy / capacity (jobs)",
"query": "sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
"series": 1
},
{
"panel": "Application CPU (cores)",
"query": "rate(messenger_process_cpu_seconds_total[1m])",
"series": 13
},
{
"panel": "Application resident RAM (bytes)",
"query": "messenger_process_memory_bytes",
"series": 13
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"series": 1
},
{
"panel": "API availability SLI / SLO (5 minutes)",
"query": "1 - (sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[5m]))",
"series": 1
},
{
"panel": "API availability SLI / SLO (5 minutes)",
"query": "vector(0.99)",
"series": 1
},
{
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\",le=\"60\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count{service_name=\"messenger-worker\"}[5m]))",
"series": 1
},
{
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
"query": "vector(0.95)",
"series": 1
},
{
"panel": "Local alert state: pending / firing",
"query": "ALERTS{alertname=~\"MessengerHighErrorRate|MessengerQueueLag\"}",
"series": 0
},
{
"panel": "Collector queue occupancy (batches)",
"query": "otelcol_exporter_queue_size{exporter=~\"otlphttp/.*\"}",
"series": 2
},
{
"panel": "Collector queue occupancy (batches)",
"query": "otelcol_exporter_queue_capacity{exporter=~\"otlphttp/.*\"}",
"series": 2
},
{
"panel": "Collector failed export attempts / second",
"query": "rate(otelcol_exporter_send_failed_spans[1m])",
"series": 1
},
{
"panel": "Collector failed export attempts / second",
"query": "rate(otelcol_exporter_send_failed_log_records[1m])",
"series": 1
}
]
}
+19
View File
@@ -0,0 +1,19 @@
UTC: 2026-11-21T09:15:31Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode failover --action-script scripts/stop-one.sh
Container messenger-lab6-tester-run-0af389d5dbc3 Creating
Container messenger-lab6-tester-run-0af389d5dbc3 Created
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.189}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
RESULT: PASS (exit 0) at 2026-11-21T09:15:52Z
UTC: 2026-11-21T09:15:53Z
Command: docker compose --profile tools run --rm -T --no-deps tester sh scripts/start-api.sh
Container messenger-lab6-tester-run-b9e9496603b9 Creating
Container messenger-lab6-tester-run-b9e9496603b9 Created
started api container 81ebb23d65ec
running api replicas: 2
RESULT: PASS (exit 0) at 2026-11-21T09:16:00Z
+19
View File
@@ -0,0 +1,19 @@
UTC: 2026-11-21T09:09:53Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/image_validation_unit.py
Container messenger-lab6-tester-run-56217a03ad55 Creating
Container messenger-lab6-tester-run-56217a03ad55 Created
test_adam7_pass_filter_boundaries (__main__.ImageValidationTests.test_adam7_pass_filter_boundaries) ... ok
test_crc_compressed_tail_and_pixel_bomb (__main__.ImageValidationTests.test_crc_compressed_tail_and_pixel_bomb) ... ok
test_critical_headers_and_chunk_order (__main__.ImageValidationTests.test_critical_headers_and_chunk_order) ... ok
test_filter_values_and_row_boundaries (__main__.ImageValidationTests.test_filter_values_and_row_boundaries) ... ok
test_jpeg_and_truncated_progressive_above_old_bypass_limit (__main__.ImageValidationTests.test_jpeg_and_truncated_progressive_above_old_bypass_limit) ... ok
test_palette_shape_and_transparency (__main__.ImageValidationTests.test_palette_shape_and_transparency) ... ok
test_valid_png_and_trailing_padding (__main__.ImageValidationTests.test_valid_png_and_trailing_padding) ... ok
----------------------------------------------------------------------
Ran 7 tests in 0.462s
OK
RESULT: PASS (exit 0) at 2026-11-21T09:09:55Z
+14
View File
@@ -0,0 +1,14 @@
UTC: 2026-11-21T09:09:55Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_images.py
Container messenger-lab6-tester-run-d0860115a1da Creating
Container messenger-lab6-tester-run-d0860115a1da Created
PASS centered crop, three real variants, checksums, original bytes and ACL
PASS no upscaling and floor rounding for thumbnail/optimized
PASS transparent PNG composited onto white JPEG
PASS EXIF orientation applied to variants and metadata removed
PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202
PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400
PASS: lab5 grade4 image checks
RESULT: PASS (exit 0) at 2026-11-21T09:10:05Z
+191
View File
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "error",
"duration_seconds": "3.3880072909998944",
"event": "http.request.complete",
"flags": "1",
"method": "POST",
"observed_timestamp": "1795251516034833239",
"outcome": "error",
"request_id": "f44f1b74-f3f1-4cbe-8b44-510cfbe21ebd",
"route": "/api/v1/users",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "0d13c5814dab",
"service_name": "messenger-api",
"service_version": "1.6.0",
"severity_number": "17",
"severity_text": "ERROR",
"span_id": "6389f79f96d8c840",
"status_code": "503",
"trace_id": "819eab187881e315742feb7f2ebb70ff"
},
"values": [
[
"1795251516034664192",
"http.request.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 4455963,
"linesProcessedPerSecond": 22619,
"totalBytesProcessed": 70329,
"totalLinesProcessed": 357,
"execTime": 0.015783,
"queueTime": 0.002886,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 2,
"shards": 2,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 62832
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 2,
"totalChunksMatched": 2,
"totalBatches": 3,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 70329,
"headChunkLines": 357,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 62832,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 584920,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
+107
View File
@@ -0,0 +1,107 @@
{
"batches": [
{
"resource": {
"attributes": [
{
"key": "service.version",
"value": {
"stringValue": "1.6.0"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "0d13c5814dab"
}
},
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-api"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger",
"version": "1.6.0"
},
"spans": [
{
"traceId": "gZ6rGHiB4xV0L+t/Lrtw/w==",
"spanId": "z5kgvNQvytc=",
"parentSpanId": "Y4n3n5bYyEA=",
"name": "db.acquire",
"kind": "SPAN_KIND_INTERNAL",
"startTimeUnixNano": "1795251513033234050",
"endTimeUnixNano": "1795251516033386648",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
},
{
"key": "exception_type",
"value": {
"stringValue": "TimeoutError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "gZ6rGHiB4xV0L+t/Lrtw/w==",
"spanId": "Y4n3n5bYyEA=",
"name": "POST /api/v1/users",
"kind": "SPAN_KIND_SERVER",
"startTimeUnixNano": "1795251512647658833",
"endTimeUnixNano": "1795251516034910448",
"attributes": [
{
"key": "method",
"value": {
"stringValue": "POST"
}
},
{
"key": "route",
"value": {
"stringValue": "/api/v1/users"
}
},
{
"key": "status_class",
"value": {
"stringValue": "5xx"
}
},
{
"key": "http.response.status_code",
"value": {
"intValue": "503"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
}
]
}
]
}
]
}
+443
View File
@@ -0,0 +1,443 @@
UTC: 2026-11-21T09:09:39Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/test_lab5.py
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools ps --status running --services
api
broker
db
grafana
lb
loki
otel-collector
prometheus
publisher
s3
tempo
valkey
worker
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 34 minutes ago Up 24 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 34 minutes ago Up 10 minutes (healthy) 5432/tcp
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 19 minutes ago Up 18 minutes 127.0.0.1:3000->3000/tcp
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 17 minutes ago Up 17 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 19 minutes ago Up 8 minutes 127.0.0.1:3100->3100/tcp
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate 18 minutes ago Exited (0) 16 minutes ago
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 20 minutes ago Up 20 minutes 4317/tcp, 55678-55679/tcp
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 18 minutes 127.0.0.1:9090->9090/tcp
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher 18 minutes ago Up 17 minutes 8080/tcp
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 34 minutes ago Up 13 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init 18 minutes ago Exited (0) 16 minutes ago
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes 127.0.0.1:3200->3200/tcp
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 34 minutes ago Up 24 minutes (healthy) 6379/tcp
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker 18 minutes ago Up 5 minutes 8080/tcp
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools build tester
Image messenger-lab6-tester Building
#1 [internal] load local bake definitions
#1 reading from stdin 587B 0.0s done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile.tester
#2 transferring dockerfile: 405B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.12.11-alpine3.22
#3 DONE 0.2s
#4 [internal] load metadata for docker.io/library/docker:28.5.1-cli
#4 DONE 0.2s
#5 [internal] load .dockerignore
#5 transferring context: 286B done
#5 DONE 0.0s
#6 [stage-1 1/5] FROM docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
#6 resolve docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 0.0s done
#6 DONE 0.1s
#7 [docker-cli 1/1] FROM docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad
#7 resolve docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad 0.0s done
#7 DONE 0.1s
#8 [stage-1 2/5] RUN apk add --no-cache make
#8 CACHED
#9 [stage-1 3/5] RUN pip install --no-cache-dir Pillow==11.0.0
#9 CACHED
#10 [stage-1 4/5] COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
#10 CACHED
#11 [stage-1 5/5] WORKDIR /work
#11 CACHED
#12 exporting to image
#12 exporting layers done
#12 exporting manifest sha256:1c93bba11e9ded8b819bbcadafd9217f8b6b4fc5301deeca1e720c49a45f1b2e done
#12 exporting config sha256:9ad2daad56b9cfccc6c03e7be0468faf51994e1604e86b502077a03f28844325 done
#12 exporting attestation manifest sha256:96fa2816d4a538ed8e6e3ea5c702033990470ee27e842b91c5f2979f0304e95a 0.1s done
#12 exporting manifest list sha256:3e586fe0bf4aeb3814ad182a780c6173a9e252da1c33e7e61f20187228c8b23f 0.0s done
#12 naming to docker.io/library/messenger-lab6-tester:latest
#12 naming to docker.io/library/messenger-lab6-tester:latest done
#12 unpacking to docker.io/library/messenger-lab6-tester:latest 0.0s done
#12 DONE 0.2s
#13 resolving provenance for metadata file
#13 DONE 0.0s
Image messenger-lab6-tester Built
RESULT: PASS (exit 0) at 2026-11-21T09:09:42Z
UTC: 2026-11-21T09:09:42Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u scripts/check_contract.py
Container messenger-lab6-tester-run-e5ea96fea8b0 Creating
Container messenger-lab6-tester-run-e5ea96fea8b0 Created
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
RESULT: PASS (exit 0) at 2026-11-21T09:09:46Z
UTC: 2026-11-21T09:09:46Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/smoke.py
Container messenger-lab6-tester-run-42a477b98467 Creating
Container messenger-lab6-tester-run-42a477b98467 Created
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
RESULT: PASS (exit 0) at 2026-11-21T09:09:53Z
UTC: 2026-11-21T09:09:53Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/image_validation_unit.py
Container messenger-lab6-tester-run-56217a03ad55 Creating
Container messenger-lab6-tester-run-56217a03ad55 Created
test_adam7_pass_filter_boundaries (__main__.ImageValidationTests.test_adam7_pass_filter_boundaries) ... ok
test_crc_compressed_tail_and_pixel_bomb (__main__.ImageValidationTests.test_crc_compressed_tail_and_pixel_bomb) ... ok
test_critical_headers_and_chunk_order (__main__.ImageValidationTests.test_critical_headers_and_chunk_order) ... ok
test_filter_values_and_row_boundaries (__main__.ImageValidationTests.test_filter_values_and_row_boundaries) ... ok
test_jpeg_and_truncated_progressive_above_old_bypass_limit (__main__.ImageValidationTests.test_jpeg_and_truncated_progressive_above_old_bypass_limit) ... ok
test_palette_shape_and_transparency (__main__.ImageValidationTests.test_palette_shape_and_transparency) ... ok
test_valid_png_and_trailing_padding (__main__.ImageValidationTests.test_valid_png_and_trailing_padding) ... ok
----------------------------------------------------------------------
Ran 7 tests in 0.462s
OK
RESULT: PASS (exit 0) at 2026-11-21T09:09:55Z
UTC: 2026-11-21T09:09:55Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_images.py
Container messenger-lab6-tester-run-d0860115a1da Creating
Container messenger-lab6-tester-run-d0860115a1da Created
PASS centered crop, three real variants, checksums, original bytes and ACL
PASS no upscaling and floor rounding for thumbnail/optimized
PASS transparent PNG composited onto white JPEG
PASS EXIF orientation applied to variants and metadata removed
PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202
PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400
PASS: lab5 grade4 image checks
RESULT: PASS (exit 0) at 2026-11-21T09:10:05Z
UTC: 2026-11-21T09:10:05Z
Command: docker compose --profile tools exec -T worker python -u - < student-tests/lab5_limits.py
PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB
PASS timeout: decoder killed and reaped; parent continues
PASS: lab5 decoder resource limits
RESULT: PASS (exit 0) at 2026-11-21T09:10:08Z
UTC: 2026-11-21T09:10:08Z
Command: docker compose --profile tools run --rm -T --no-deps tester chmod +x scripts/restart.sh scripts/stop-one.sh
Container messenger-lab6-tester-run-100a83886eb9 Creating
Container messenger-lab6-tester-run-100a83886eb9 Created
RESULT: PASS (exit 0) at 2026-11-21T09:10:09Z
UTC: 2026-11-21T09:10:09Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_recovery.py
Container messenger-lab6-tester-run-6606f20c462f Creating
Container messenger-lab6-tester-run-6606f20c462f Created
PASS persistence: attachment=9d7de7ac-016a-49d7-a0f1-9b880cee3c83 same session/ID after API+broker restart; volumes preserved; queued content409; messages201; ready
PASS crash_after_commit_before_publish: attachment=af40fc00-4da4-4fe6-80b3-27a8879d7661 hits=1 recovered_seconds=4.0; same original; three variants
PASS crash_after_objects_before_commit: attachment=30ee3a05-086c-45fd-bbf1-78df42514413 hits=1 recovered_seconds=118.6; same original; three variants
PASS crash_after_commit_before_ack: attachment=4aa3f19d-638f-492a-b17e-ccb103f1cae9 hits=1 recovered_seconds=0.4; same original; three variants
PASS failed/replay: attachment=00b66546-aa21-4115-aca1-13fbc7f47f2c; permanent error fails once; failed registry; content/link409; audited replay preserves job_id; ready duplicate no-op
PASS transient S3 outage: attachment=242a36c2-40f8-440f-899b-228164e5ddc4; attempts=5; durable retries, retry_exhausted, explicit replay after recovery
PASS: lab5 grade4 recovery checks (all)
RESULT: PASS (exit 0) at 2026-11-21T09:14:39Z
UTC: 2026-11-21T09:14:39Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
UTC: 2026-11-21T09:14:40Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:10:22.045070464Z finished=2026-11-21T09:10:21.647750535Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:10:18.318585259Z finished=2026-11-21T09:10:17.940277751Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:10:24.158928332Z finished=2026-11-21T09:10:23.907977242Z
/messenger-lab6-db-1 status=running started=2026-11-21T08:59:12.528963564Z finished=2026-11-21T08:58:31.826616128Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T08:51:52.621809468Z finished=0001-01-01T00:00:00Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:10:55.825987305Z finished=2026-11-21T09:10:55.547390235Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:09.569875679Z finished=2026-11-21T09:13:44.880718518Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T08:45:28.390651046Z finished=2026-11-21T08:45:27.109421727Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:13:45.19691868Z finished=2026-11-21T09:13:41.499885247Z
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
UTC: 2026-11-21T09:14:40Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
Container messenger-lab6-tester-run-e3bdaf9250f1 Creating
Container messenger-lab6-tester-run-e3bdaf9250f1 Created
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
RESULT: PASS (exit 0) at 2026-11-21T09:15:30Z
UTC: 2026-11-21T09:15:30Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
UTC: 2026-11-21T09:15:31Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode failover --action-script scripts/stop-one.sh
Container messenger-lab6-tester-run-0af389d5dbc3 Creating
Container messenger-lab6-tester-run-0af389d5dbc3 Created
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.189}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
RESULT: PASS (exit 0) at 2026-11-21T09:15:52Z
UTC: 2026-11-21T09:15:52Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
UTC: 2026-11-21T09:15:53Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=exited started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:15:40.679293151Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
UTC: 2026-11-21T09:15:53Z
Command: docker compose --profile tools run --rm -T --no-deps tester sh scripts/start-api.sh
Container messenger-lab6-tester-run-b9e9496603b9 Creating
Container messenger-lab6-tester-run-b9e9496603b9 Created
started api container 81ebb23d65ec
running api replicas: 2
RESULT: PASS (exit 0) at 2026-11-21T09:16:00Z
UTC: 2026-11-21T09:16:00Z
Command: docker compose --profile tools ps -a -q
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
UTC: 2026-11-21T09:16:01Z
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:54.07308443Z finished=2026-11-21T09:15:40.679293151Z
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
UTC: 2026-11-21T09:16:01Z
Command: docker compose --profile tools ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab6-api-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 7 seconds (healthy) 8080/tcp
messenger-lab6-api-2 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 43 seconds (healthy) 8080/tcp
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 40 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 40 minutes ago Up About a minute (healthy) 5432/tcp
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 25 minutes ago Up 24 minutes 127.0.0.1:3000->3000/tcp
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 24 minutes ago Up 36 seconds (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 25 minutes ago Up 14 minutes 127.0.0.1:3100->3100/tcp
messenger-lab6-migrate-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/migr…" migrate 24 minutes ago Exited (0) 23 minutes ago
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 27 minutes ago Up 26 minutes 4317/tcp, 55678-55679/tcp
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 25 minutes ago Up 24 minutes 127.0.0.1:9090->9090/tcp
messenger-lab6-publisher-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.jobs" publisher 24 minutes ago Up 44 seconds 8080/tcp
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 40 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
messenger-lab6-s3-init-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/s3_b…" s3-init 24 minutes ago Exited (0) 22 minutes ago
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 9 minutes ago Up 9 minutes 127.0.0.1:3200->3200/tcp
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 40 minutes ago Up About a minute (healthy) 6379/tcp
messenger-lab6-worker-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.worker" worker 24 minutes ago Up 44 seconds 8080/tcp
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
PASS: lab5 full checks. UTF-8 logs: C:\Users\masha\WebstormProjects\mtusi2026lab6\evidence
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
+8
View File
@@ -0,0 +1,8 @@
UTC: 2026-11-21T09:10:05Z
Command: docker compose --profile tools exec -T worker python -u - < student-tests/lab5_limits.py
PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB
PASS timeout: decoder killed and reaped; parent continues
PASS: lab5 decoder resource limits
RESULT: PASS (exit 0) at 2026-11-21T09:10:08Z
+562
View File
@@ -0,0 +1,562 @@
{
"start": "2026-11-21T08:51:58Z",
"end": "2026-11-21T08:52:21Z",
"queries": {
"rss_bytes": "messenger_process_memory_bytes",
"cpu_cores": "rate(messenger_process_cpu_seconds_total[15s])"
},
"results": {
"rss_bytes": {
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "0d13c5814dab",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "0d13c5814dab",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"120827904"
],
[
1795251123,
"121270272"
],
[
1795251128,
"122515456"
],
[
1795251133,
"125259776"
],
[
1795251138,
"127827968"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "0fa6e7066e3145a993d3264494361be3",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "0fa6e7066e3145a993d3264494361be3",
"service_name": "messenger-worker",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"89513984"
],
[
1795251123,
"89513984"
],
[
1795251128,
"89513984"
],
[
1795251133,
"89513984"
],
[
1795251138,
"89513984"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "551e5017a02c4426af0773950c83c10e",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "551e5017a02c4426af0773950c83c10e",
"service_name": "messenger-worker",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"88707072"
],
[
1795251123,
"88707072"
],
[
1795251128,
"88707072"
],
[
1795251133,
"88707072"
],
[
1795251138,
"88707072"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "6cacaff48df9",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "6cacaff48df9",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"122683392"
],
[
1795251123,
"122683392"
],
[
1795251128,
"122683392"
],
[
1795251133,
"122683392"
],
[
1795251138,
"122683392"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "81ebb23d65ec",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "81ebb23d65ec",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"120791040"
],
[
1795251123,
"121085952"
],
[
1795251128,
"122593280"
],
[
1795251133,
"125325312"
],
[
1795251138,
"128040960"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "a068b747445b44439e7e1b7bc72675c2",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "a068b747445b44439e7e1b7bc72675c2",
"service_name": "messenger-publisher",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"88813568"
],
[
1795251123,
"88813568"
],
[
1795251128,
"88813568"
],
[
1795251133,
"88813568"
],
[
1795251138,
"88813568"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "da2c8073a2144ecc9965ea4553926bf3",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "da2c8073a2144ecc9965ea4553926bf3",
"service_name": "messenger-publisher",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"89305088"
],
[
1795251123,
"89305088"
],
[
1795251128,
"89305088"
],
[
1795251133,
"89305088"
],
[
1795251138,
"89305088"
]
]
},
{
"metric": {
"__name__": "messenger_process_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "e16c50966c56",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "e16c50966c56",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"123514880"
],
[
1795251123,
"123514880"
],
[
1795251128,
"123514880"
],
[
1795251133,
"123514880"
],
[
1795251138,
"123514880"
]
]
}
]
}
},
"cpu_cores": {
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "0d13c5814dab",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "0d13c5814dab",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251123,
"0.022415557507320008"
],
[
1795251128,
"0.17908626842684267"
],
[
1795251133,
"0.3572547659765976"
],
[
1795251138,
"0.3890083826"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "0fa6e7066e3145a993d3264494361be3",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "0fa6e7066e3145a993d3264494361be3",
"service_name": "messenger-worker",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"0"
],
[
1795251123,
"0"
],
[
1795251128,
"0"
],
[
1795251133,
"0"
],
[
1795251138,
"0"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "551e5017a02c4426af0773950c83c10e",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "551e5017a02c4426af0773950c83c10e",
"service_name": "messenger-worker",
"service_version": "1.6.0"
},
"values": [
[
1795251123,
"0.0005782936565199904"
],
[
1795251128,
"0.0007851155115511435"
],
[
1795251133,
"0.0005319559955995547"
],
[
1795251138,
"0.0004686850999999992"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "6cacaff48df9",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "6cacaff48df9",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"0"
],
[
1795251123,
"0"
],
[
1795251128,
"0"
],
[
1795251133,
"0"
],
[
1795251138,
"0"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "81ebb23d65ec",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "81ebb23d65ec",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251123,
"0.038017601540733366"
],
[
1795251128,
"0.17548991699169916"
],
[
1795251133,
"0.34144971107110705"
],
[
1795251138,
"0.3803775972000001"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "a068b747445b44439e7e1b7bc72675c2",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "a068b747445b44439e7e1b7bc72675c2",
"service_name": "messenger-publisher",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"0"
],
[
1795251123,
"0"
],
[
1795251128,
"0"
],
[
1795251133,
"0"
],
[
1795251138,
"0"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "da2c8073a2144ecc9965ea4553926bf3",
"exported_job": "messenger-publisher",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "da2c8073a2144ecc9965ea4553926bf3",
"service_name": "messenger-publisher",
"service_version": "1.6.0"
},
"values": [
[
1795251123,
"0.01116402495005333"
],
[
1795251128,
"0.020074461846184624"
],
[
1795251133,
"0.021264563956395647"
],
[
1795251138,
"0.021109743999999986"
]
]
},
{
"metric": {
"deployment_environment_name": "lab",
"exported_instance": "e16c50966c56",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "messenger",
"service_instance_id": "e16c50966c56",
"service_name": "messenger-api",
"service_version": "1.6.0"
},
"values": [
[
1795251118,
"0"
],
[
1795251123,
"0"
],
[
1795251128,
"0"
],
[
1795251133,
"0"
],
[
1795251138,
"0"
]
]
}
]
}
}
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 20.0,
"elapsed_seconds": 20.27,
"concurrency": 4,
"successful_requests": 1645,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 81.154,
"successful_latency_ms": {
"p50": 38.796,
"p95": 114.354,
"p99": 194.742
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
UTC: 2026-11-21T08:51:58Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 tests/load.py --base-url http://localhost:8086 --duration 20 --concurrency 4 --output evidence/load-summary.json
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 20.0,
"elapsed_seconds": 20.27,
"concurrency": 4,
"successful_requests": 1645,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 81.154,
"successful_latency_ms": {
"p50": 38.796,
"p95": 114.354,
"p99": 194.742
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
RESULT: PASS (exit 0) at 2026-11-21T08:52:21Z
+11
View File
@@ -0,0 +1,11 @@
UTC: 2026-11-21T09:16:46Z
Command: docker compose --profile tools run --rm -T --no-deps tester make check test
Container messenger-lab6-tester-run-f02cd878c6ba Creating
Container messenger-lab6-tester-run-f02cd878c6ba Created
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://lb:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
RESULT: PASS (exit 0) at 2026-11-21T09:16:55Z
File diff suppressed because it is too large Load Diff
+6
View File
@@ -0,0 +1,6 @@
UTC: 2026-11-21T09:16:58Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --baseline
PASS: real API/worker logs, traces and metrics; evidence/observability-baseline.json
RESULT: PASS (exit 0) at 2026-11-21T09:17:18Z
File diff suppressed because it is too large Load Diff
+8
View File
@@ -0,0 +1,8 @@
UTC: 2026-11-21T08:58:20Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --incidents
PASS: worker stopped
PASS: PostgreSQL unavailable
PASS: Tempo and Loki unavailable
RESULT: PASS (exit 0) at 2026-11-21T09:02:41Z
+26
View File
@@ -0,0 +1,26 @@
{
"scenario": "S3 unavailable during first attempt, then restored",
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"request_id": "observe-927e3831c3d241ac80d3d43391ad1ff4",
"api_trace_id": "94af84b9ec1663416ac05526995fbd91",
"accepted_at": "2026-11-21T08:56:07.085875Z",
"worker_trace_id": "94af84b9ec1663416ac05526995fbd91",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"worker_attempts": [
"1",
"2",
"2",
"3"
],
"same_trace": true,
"correlated_at": "2026-11-21T08:58:20.029926Z",
"content": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"original_bytes": 111817,
"thumbnail_size": [
256,
160
]
}
}
+6
View File
@@ -0,0 +1,6 @@
UTC: 2026-11-21T08:56:05Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --chain-with-retry
PASS: propagated upload/publish/consume/S3/DB trace including retry
RESULT: PASS (exit 0) at 2026-11-21T08:58:20Z
+15
View File
@@ -0,0 +1,15 @@
UTC: 2026-11-21T09:16:55Z
Command: docker compose exec -T api python - < student-tests/observability_unit.py
test_http_route_template_and_request_trace_log_correlation_without_secrets (__main__.ObservabilityTests.test_http_route_template_and_request_trace_log_correlation_without_secrets) ... ok
test_missing_or_invalid_context_starts_new_root_even_inside_span (__main__.ObservabilityTests.test_missing_or_invalid_context_starts_new_root_even_inside_span) ... ok
test_unknown_route_and_exception_message_do_not_escape_into_telemetry (__main__.ObservabilityTests.test_unknown_route_and_exception_message_do_not_escape_into_telemetry) ... ok
test_unsampled_trace_flags_and_tracestate_survive (__main__.ObservabilityTests.test_unsampled_trace_flags_and_tracestate_survive) ... ok
test_w3c_parent_child_and_durable_retry_preserve_trace (__main__.ObservabilityTests.test_w3c_parent_child_and_durable_retry_preserve_trace) ... ok
----------------------------------------------------------------------
Ran 5 tests in 0.694s
OK
RESULT: PASS (exit 0) at 2026-11-21T09:16:58Z
+9
View File
@@ -0,0 +1,9 @@
UTC: 2026-11-21T09:14:40Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
Container messenger-lab6-tester-run-e3bdaf9250f1 Creating
Container messenger-lab6-tester-run-e3bdaf9250f1 Created
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
RESULT: PASS (exit 0) at 2026-11-21T09:15:30Z
+14
View File
@@ -0,0 +1,14 @@
UTC: 2026-11-21T09:10:09Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_recovery.py
Container messenger-lab6-tester-run-6606f20c462f Creating
Container messenger-lab6-tester-run-6606f20c462f Created
PASS persistence: attachment=9d7de7ac-016a-49d7-a0f1-9b880cee3c83 same session/ID after API+broker restart; volumes preserved; queued content409; messages201; ready
PASS crash_after_commit_before_publish: attachment=af40fc00-4da4-4fe6-80b3-27a8879d7661 hits=1 recovered_seconds=4.0; same original; three variants
PASS crash_after_objects_before_commit: attachment=30ee3a05-086c-45fd-bbf1-78df42514413 hits=1 recovered_seconds=118.6; same original; three variants
PASS crash_after_commit_before_ack: attachment=4aa3f19d-638f-492a-b17e-ccb103f1cae9 hits=1 recovered_seconds=0.4; same original; three variants
PASS failed/replay: attachment=00b66546-aa21-4115-aca1-13fbc7f47f2c; permanent error fails once; failed registry; content/link409; audited replay preserves job_id; ready duplicate no-op
PASS transient S3 outage: attachment=242a36c2-40f8-440f-899b-228164e5ddc4; attempts=5; durable retries, retry_exhausted, explicit replay after recovery
PASS: lab5 grade4 recovery checks (all)
RESULT: PASS (exit 0) at 2026-11-21T09:14:39Z
+191
View File
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.06001993599988964",
"event": "http.request.complete",
"flags": "1",
"method": "POST",
"observed_timestamp": "1795251367084401557",
"outcome": "success",
"request_id": "observe-927e3831c3d241ac80d3d43391ad1ff4",
"route": "/api/v1/chats/{chat_id}/attachments",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "81ebb23d65ec",
"service_name": "messenger-api",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "10167513c3917aba",
"status_code": "202",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251367084238848",
"http.request.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 6425057,
"linesProcessedPerSecond": 32614,
"totalBytesProcessed": 86680,
"totalLinesProcessed": 440,
"execTime": 0.013491,
"queueTime": 0.001181,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 0,
"shards": 1,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 77440
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 1,
"totalChunksMatched": 1,
"totalBatches": 2,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 86680,
"headChunkLines": 440,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 77440,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 190898,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
+318
View File
@@ -0,0 +1,318 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"attempt": "1",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "warn",
"duration_seconds": "1.1696733750000021",
"event": "job.complete",
"flags": "1",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795251373761214222",
"outcome": "error",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "13",
"severity_text": "WARN",
"span_id": "604b1094f4b8ea5a",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251373761101312",
"job.complete"
]
]
},
{
"stream": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"attempt": "2",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.04622686900006556",
"event": "job.complete",
"flags": "1",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795251405312162771",
"outcome": "duplicate",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "860a8bda1f144a65878ee73824659b78",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "ba1e0fd901467175",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251405311928576",
"job.complete"
]
]
},
{
"stream": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"attempt": "2",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"event": "job.retry",
"flags": "1",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"observed_timestamp": "1795251373753026344",
"outcome": "retry",
"queue": "attachments",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "604b1094f4b8ea5a",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251373752916480",
"job.retry"
]
]
},
{
"stream": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"attempt": "2",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "warn",
"duration_seconds": "1.4581017409998367",
"event": "job.complete",
"flags": "1",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795251375868803562",
"outcome": "error",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "13",
"severity_text": "WARN",
"span_id": "83adf2393f95e484",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251375868673792",
"job.complete"
]
]
},
{
"stream": {
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
"attempt": "3",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.36952172699989205",
"event": "job.complete",
"flags": "1",
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795251497430982971",
"outcome": "success",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "860a8bda1f144a65878ee73824659b78",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "ba826640e32b3505",
"trace_id": "94af84b9ec1663416ac05526995fbd91"
},
"values": [
[
"1795251497430907392",
"job.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 142743,
"linesProcessedPerSecond": 766,
"totalBytesProcessed": 1117,
"totalLinesProcessed": 6,
"execTime": 0.007825,
"queueTime": 0.000908,
"subqueries": 0,
"totalEntriesReturned": 5,
"splits": 0,
"shards": 1,
"totalPostFilterLines": 5,
"totalStructuredMetadataBytesProcessed": 1048
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 1,
"totalChunksMatched": 1,
"totalBatches": 2,
"totalLinesSent": 5,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 1117,
"headChunkLines": 6,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 5,
"headChunkStructuredMetadataBytes": 1048,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 103240,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
+73
View File
@@ -0,0 +1,73 @@
{
"captured_at": "2026-11-21T09:17:48.990874Z",
"host": "Windows, Intel Core Ultra 5 125U",
"docker": {
"ServerVersion": "29.3.0",
"NCPU": 14,
"MemTotal": 7965700096,
"OperatingSystem": "Docker Desktop"
},
"revision_before_final_commit": "d0a062e2d2fbeda54c2a24f518eb3687650f7c9e",
"working_tree_grade4": true,
"api_url": "http://localhost:8086",
"code_sha256": {
".env.example": "f4d81ed6a4559e9f26c0c5a3d8accc495adb330c107c4f47cc34d47321a81dfe",
".github/workflows/application.yml": "b486fe8a8299d363cf4e403dc2987de70c35a3ac6cc7e39f7656b6b77e6763ea",
".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8",
"app/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"app/advanced.py": "27762f863185cb6b51a64ae7698e60350b5aba0f924992e7eecaf58511d6e5cc",
"app/attachments.py": "fae87bae0835fe6cafe8e85b685d1a1a0039d67516d52a633bac66e26c7fb8c1",
"app/cache.py": "c7d5a210c4b85624ec089c2626cb69a84154790d325a6308cc8b0f578db3178b",
"app/config.py": "0ee78d89095878150de4c195e28ffc17c44ab40f5e6b40fc6cb9ef23672d10aa",
"app/db.py": "28eade9a5dda08887f9efeb786e52040832dd28975619a0c4b23988be2ba1ca3",
"app/deps.py": "40b9353387289b7b3ff964a216446f13bbde66ee5a26bf5c847e39fe1948b062",
"app/errors.py": "b06a0001aa2f1410cdcd4c385719ad3b90a0dacae62784649129c1e1c6dd933c",
"app/image_worker.py": "66821f54709707fb8ad7d336eeceaa97cb69d6e6b1ae369147b59241a1e099f5",
"app/images.py": "9e0cd83ef5d673356bde672d77ec106f8f3bb86812aaf46fbe3c932db2f7df54",
"app/jobs.py": "b1b6dd46529042b72a088a61a3e153f6c8861d41550dbe713056be301dc5c3dc",
"app/lifecycle.py": "097fcd74d178654fa91b3704d45c5215cfdec2f0ab3bb21b432e028160c9aa5e",
"app/main.py": "fabcf21af59875defa0c130b03bf14df8db3ac9352df654dc3da4fcf38b84f50",
"app/models.py": "69eb6bd83683c7ab1329e3d0d331577ce40612c156dc17c8ae1bdd231a921ecc",
"app/operator.py": "d9046d7c757bc9c40721f11569f4ca1a7c9e52586ad05a04f0b6179b31b551b6",
"app/pipeline.py": "3bcdd1484d4633c901a7bcfcd9f800db9f5e53db6d1c610d82d556d1337fbd4c",
"app/repositories.py": "86b2774b89e2f7b875283cd684c7b346eeba1da2dd76d836661f6b45812ebcd8",
"app/security.py": "b5e0fa73943a046501d1015d9b0288272b93e6a8458a47651c1bd9fa3b03a680",
"app/server.py": "092af34b790f3103eb715f956c686655e18309cf6bab7073a015222f709d61b4",
"app/services.py": "128a6668ad74697c05612fd9ad13107d14ba0ed0c9074338fb2d14437b54c785",
"app/storage.py": "8573d43b3993c9f1e0e5ecbf82a9f65f58041e6bd50d4a315875d24b67042bd1",
"app/telemetry.py": "232fde35c8445e3a80a067f41a1ac264fd41c662b3ae1ebbc3c7e661200fe9bc",
"app/trace_context.py": "4ad1e98c346c6542df0bba808f0449922127ced410289b05ca2a3d94354b9740",
"app/worker.py": "8eb05a2c37325a6d1da77102a683a19cc25fc7cb8adeeaa2b1c525a4029c7bbc",
"compose.yaml": "a29a6af5228d9c2a1a529818071787690d3108d82cb83d7edc668ea0656fd969",
"Dockerfile": "a0029707f843a15501a7d05142c06662b2705e16eb0292954148674cfe739b87",
"infra/grafana/dashboards/messenger.json": "826c301adef62987d040a4f0d764d7f3dc89275de0c488979a03737ad73dbce9",
"infra/grafana/provisioning/dashboards/dashboards.yaml": "e764b58e795886455758da0c93b1199e75d220bdf9e0c087f859a4ce67f744ef",
"infra/grafana/provisioning/datasources/datasources.yaml": "8234d0c4b273127f3f5aad084f942dcc4dc498640f796bcc34feaae3a1eae1cb",
"infra/loki/loki.yaml": "4614b904e6297165bebdafb4004e2c4ba93ea6bb278b4f53f196566621740dbd",
"infra/otel/collector.yaml": "52fd1266cb78fd1e73bdc2a36057ec913abba6f50e7ce93bb3a31db69e25b8ba",
"infra/prometheus/alerts.yaml": "252499c676f9fa9564c6466e7f268a9975c9044df0bdd82de78565f90548bbbe",
"infra/prometheus/prometheus.yaml": "362d7f375ef6bb85fbd2e2e1e9c09c1a8507e94a3b115e13b0869ffcbc777d47",
"infra/tempo/tempo.yaml": "a87b8a536ee4a42d8f83716c753b0380498ea17893082c1b7d1d21aafc21fc51",
"migrations/001_initial_schema.sql": "e5c956278405d4f65be50b2e0b92c37f0870589e9c0d972c70d6733f1ae439e2",
"migrations/002_indexes.sql": "6b0a375c09505f5220eb3add3ff3e4115b8426bbe1e1d97556effb8f64e4c7ff",
"migrations/003_idempotency_keys.sql": "9554fd815d4b0e3085d2fe725c0e0e8b2a36e212fe3a486345e28afaf38e8b69",
"migrations/004_attachments.sql": "2f421cbfb0cbf50d80d4bb8812378cdbc2619a5b8fa7aa5116238e287a8c6c6b",
"migrations/005_async_pipeline.sql": "ce928ea8304c95a8bf2ad0ca92e6ffce81070362020de1a3373e381fd3dab247",
"migrations/006_trace_context.sql": "a452982ba83fb0c270c54a3796d5c257bd86f74bdd8627474dcb3b022e33ad33",
"requirements.txt": "4abdbc52e41c0ac0cfe5d91718558e5e9ee3951e27efe99d8c31a83c736f2924",
"scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671",
"scripts/migrate.py": "9f095a2181ed2a84584b09fe8cdfbf896c5c26306c7c5c5c262b4bbbb3f28d88",
"scripts/observe_lab6.py": "bbfa84f0c360c538db68396afd97d228e272bd9ca68456c72a7a89eb77b91025",
"scripts/s3_bootstrap.py": "f7bddb150df0e94e97a21efa2cad22c426e00fc935e710c16e11070f15344ef2",
"scripts/test_lab5.py": "957546f3939f1d120d119d61a6df672034c0d234a32d014c413152490bec59ef",
"scripts/test_lab6.py": "fd2ee01770b601c3fb0c89ea8e79c50d18a859e20a06c19f873ae38b1b6d06d0",
"scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54",
"student-tests/image_validation_unit.py": "531a2de31f90c92e094f7e5ef8bea663f2d70ecee41464a461e0e54d9800f795",
"student-tests/lab5_helpers.py": "68a8a0cc9fc644d883342cb82642b2a679a32ed2db252f2e5a5b845d9ad2cec1",
"student-tests/lab5_images.py": "048cc6dbb674869608c13ae290f20d57a8356da79a510f4517242bf57f5a505d",
"student-tests/lab5_limits.py": "c2095797ce002e78a68e2d00afb2b10aa46a85787e8f1a43e6fb2b5a70045336",
"student-tests/lab5_recovery.py": "8d7aa7de8dc09e18f206e32cac3001f4a916e9448e3887d0de7154aa832553d9",
"student-tests/observability_unit.py": "9759b4ca7e312d656a2a0fb6f886a6dcb19ca50983a3f1733371142b24579d09"
},
"notes": "Collected before third commit; source hashes normalize CRLF to LF. Synthetic data only."
}
+83
View File
@@ -0,0 +1,83 @@
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools ps --status running --services
api
broker
db
grafana
lb
loki
otel-collector
prometheus
publisher
s3
tempo
valkey
worker
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools build tester
Image messenger-lab6-tester Building
#1 [internal] load local bake definitions
#1 reading from stdin 587B 0.0s done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile.tester
#2 transferring dockerfile: 405B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.12.11-alpine3.22
#3 DONE 0.2s
#4 [internal] load metadata for docker.io/library/docker:28.5.1-cli
#4 DONE 0.2s
#5 [internal] load .dockerignore
#5 transferring context: 286B done
#5 DONE 0.0s
#6 [stage-1 1/5] FROM docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
#6 resolve docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 0.0s done
#6 DONE 0.1s
#7 [docker-cli 1/1] FROM docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad
#7 resolve docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad 0.0s done
#7 DONE 0.1s
#8 [stage-1 2/5] RUN apk add --no-cache make
#8 CACHED
#9 [stage-1 3/5] RUN pip install --no-cache-dir Pillow==11.0.0
#9 CACHED
#10 [stage-1 4/5] COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
#10 CACHED
#11 [stage-1 5/5] WORKDIR /work
#11 CACHED
#12 exporting to image
#12 exporting layers done
#12 exporting manifest sha256:1c93bba11e9ded8b819bbcadafd9217f8b6b4fc5301deeca1e720c49a45f1b2e done
#12 exporting config sha256:9ad2daad56b9cfccc6c03e7be0468faf51994e1604e86b502077a03f28844325 done
#12 exporting attestation manifest sha256:96fa2816d4a538ed8e6e3ea5c702033990470ee27e842b91c5f2979f0304e95a 0.1s done
#12 exporting manifest list sha256:3e586fe0bf4aeb3814ad182a780c6173a9e252da1c33e7e61f20187228c8b23f 0.0s done
#12 naming to docker.io/library/messenger-lab6-tester:latest
#12 naming to docker.io/library/messenger-lab6-tester:latest done
#12 unpacking to docker.io/library/messenger-lab6-tester:latest 0.0s done
#12 DONE 0.2s
#13 resolving provenance for metadata file
#13 DONE 0.0s
Image messenger-lab6-tester Built
RESULT: PASS (exit 0) at 2026-11-21T09:09:42Z
UTC: 2026-11-21T09:10:08Z
Command: docker compose --profile tools run --rm -T --no-deps tester chmod +x scripts/restart.sh scripts/stop-one.sh
Container messenger-lab6-tester-run-100a83886eb9 Creating
Container messenger-lab6-tester-run-100a83886eb9 Created
RESULT: PASS (exit 0) at 2026-11-21T09:10:09Z
+8
View File
@@ -0,0 +1,8 @@
UTC: 2026-11-21T09:09:46Z
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/smoke.py
Container messenger-lab6-tester-run-42a477b98467 Creating
Container messenger-lab6-tester-run-42a477b98467 Created
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
RESULT: PASS (exit 0) at 2026-11-21T09:09:53Z
+17
View File
@@ -0,0 +1,17 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api About a minute ago Up About a minute (healthy) 8080/tcp
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api About a minute ago Up About a minute (healthy) 8080/tcp
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 42 minutes ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 42 minutes ago Up 2 minutes (healthy) 5432/tcp
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 27 minutes ago Up 26 minutes 127.0.0.1:3000->3000/tcp
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 26 minutes ago Up 2 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 27 minutes ago Up 16 minutes 127.0.0.1:3100->3100/tcp
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate About a minute ago Exited (0) About a minute ago
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 29 minutes ago Up 28 minutes 4317/tcp, 55678-55679/tcp
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 27 minutes ago Up 26 minutes 127.0.0.1:9090->9090/tcp
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher About a minute ago Up About a minute 8080/tcp
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 42 minutes ago Up 2 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init About a minute ago Exited (0) About a minute ago
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo About a minute ago Up About a minute 127.0.0.1:3200->3200/tcp
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 42 minutes ago Up 2 minutes (healthy) 6379/tcp
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker About a minute ago Up About a minute 8080/tcp
+44
View File
@@ -0,0 +1,44 @@
UTC: 2026-11-21T09:09:40Z
Command: docker compose --profile tools ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 34 minutes ago Up 24 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 34 minutes ago Up 10 minutes (healthy) 5432/tcp
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 19 minutes ago Up 18 minutes 127.0.0.1:3000->3000/tcp
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 17 minutes ago Up 17 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 19 minutes ago Up 8 minutes 127.0.0.1:3100->3100/tcp
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate 18 minutes ago Exited (0) 16 minutes ago
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 20 minutes ago Up 20 minutes 4317/tcp, 55678-55679/tcp
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 18 minutes 127.0.0.1:9090->9090/tcp
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher 18 minutes ago Up 17 minutes 8080/tcp
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 34 minutes ago Up 13 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init 18 minutes ago Exited (0) 16 minutes ago
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes 127.0.0.1:3200->3200/tcp
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 34 minutes ago Up 24 minutes (healthy) 6379/tcp
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker 18 minutes ago Up 5 minutes 8080/tcp
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
UTC: 2026-11-21T09:16:01Z
Command: docker compose --profile tools ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab6-api-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 7 seconds (healthy) 8080/tcp
messenger-lab6-api-2 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 43 seconds (healthy) 8080/tcp
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 40 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 40 minutes ago Up About a minute (healthy) 5432/tcp
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 25 minutes ago Up 24 minutes 127.0.0.1:3000->3000/tcp
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 24 minutes ago Up 36 seconds (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 25 minutes ago Up 14 minutes 127.0.0.1:3100->3100/tcp
messenger-lab6-migrate-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/migr…" migrate 24 minutes ago Exited (0) 23 minutes ago
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 27 minutes ago Up 26 minutes 4317/tcp, 55678-55679/tcp
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 25 minutes ago Up 24 minutes 127.0.0.1:9090->9090/tcp
messenger-lab6-publisher-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.jobs" publisher 24 minutes ago Up 44 seconds 8080/tcp
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 40 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
messenger-lab6-s3-init-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/s3_b…" s3-init 24 minutes ago Exited (0) 22 minutes ago
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 9 minutes ago Up 9 minutes 127.0.0.1:3200->3200/tcp
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 40 minutes ago Up About a minute (healthy) 6379/tcp
messenger-lab6-worker-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.worker" worker 24 minutes ago Up 44 seconds 8080/tcp
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
File diff suppressed because it is too large Load Diff
+191
View File
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.05064601900016896",
"event": "http.request.complete",
"flags": "1",
"method": "POST",
"observed_timestamp": "1795251728469179436",
"outcome": "success",
"request_id": "observe-81adf6fa6db94a2caf9dbdf96d62b259",
"route": "/api/v1/chats/{chat_id}/attachments",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "0d13c5814dab",
"service_name": "messenger-api",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "0ff19193d60d0a11",
"status_code": "202",
"trace_id": "167d5623ef64fb0ed646ec1e2fe01ad8"
},
"values": [
[
"1795251728468912896",
"http.request.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 87014288,
"linesProcessedPerSecond": 188878,
"totalBytesProcessed": 1367324,
"totalLinesProcessed": 2968,
"execTime": 0.015714,
"queueTime": 0.000907,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 2,
"shards": 2,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 1251756
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 2,
"totalChunksMatched": 2,
"totalBatches": 3,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 60282,
"headChunkLines": 306,
"decompressedBytes": 1307042,
"decompressedLines": 2662,
"compressedBytes": 58630,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 53856,
"decompressedStructuredMetadataBytes": 1197900
},
"chunkRefsFetchTime": 386158,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,191 @@
{
"status": "success",
"data": {
"resultType": "streams",
"result": [
{
"stream": {
"attachment_id": "567d9b19-91b2-468c-95bb-8317c6ef90f2",
"attempt": "1",
"code_filepath": "/app/app/telemetry.py",
"code_function": "emit",
"code_lineno": "180",
"deployment_environment_name": "lab",
"detected_level": "info",
"duration_seconds": "0.4046185140000489",
"event": "job.complete",
"flags": "1",
"job_id": "2c0f93ca-74a0-4191-9b63-85fd07008e86",
"job_type": "attachment.process.v1",
"observed_timestamp": "1795251729011236193",
"outcome": "success",
"scope_name": "messenger.telemetry.events",
"service_instance_id": "b5902353dfd648c39ac966e0780e90a3",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"severity_number": "9",
"severity_text": "INFO",
"span_id": "59e8500a32f934ab",
"trace_id": "167d5623ef64fb0ed646ec1e2fe01ad8"
},
"values": [
[
"1795251729011139328",
"job.complete"
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 197625,
"linesProcessedPerSecond": 1051,
"totalBytesProcessed": 1128,
"totalLinesProcessed": 6,
"execTime": 0.005708,
"queueTime": 0.001056,
"subqueries": 0,
"totalEntriesReturned": 1,
"splits": 2,
"shards": 2,
"totalPostFilterLines": 1,
"totalStructuredMetadataBytesProcessed": 1056
},
"querier": {
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": false,
"chunk": {
"headChunkBytes": 0,
"headChunkLines": 0,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 0,
"headChunkStructuredMetadataBytes": 0,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 0,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"ingester": {
"totalReached": 2,
"totalChunksMatched": 2,
"totalBatches": 3,
"totalLinesSent": 1,
"store": {
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunksDownloadTime": 0,
"queryReferencedStructuredMetadata": true,
"chunk": {
"headChunkBytes": 1128,
"headChunkLines": 6,
"decompressedBytes": 0,
"decompressedLines": 0,
"compressedBytes": 0,
"totalDuplicates": 0,
"postFilterLines": 1,
"headChunkStructuredMetadataBytes": 1056,
"decompressedStructuredMetadataBytes": 0
},
"chunkRefsFetchTime": 227539,
"congestionControlLatency": 0,
"pipelineWrapperFilteredLines": 0
}
},
"cache": {
"chunk": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"index": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"volumeResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"seriesResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"labelResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
},
"instantMetricResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0,
"queryLengthServed": 0
}
},
"index": {
"totalChunks": 0,
"postFilterChunks": 0,
"shardsDuration": 0
}
}
}
}
File diff suppressed because it is too large Load Diff
+1
View File
@@ -0,0 +1 @@
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+20
View File
@@ -0,0 +1,20 @@
UTC: 2026-11-21T09:03:11Z
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 -c "import json
import scripts.observe_lab6 as m
client,stand=m.Business(),m.Stand()
try:
m.wait_backends();client.bootstrap()
result=m.lag_incident(client,stand,'MessengerQueueLag')
path=m.EVIDENCE/'observability-incidents.json'
data=json.loads(path.read_text(encoding='utf-8'))
data['incidents'][0]=result
data['captured_at']=m.now()
m.save('observability-incidents.json',data)
print('PASS: clean queue and inactive alert -> newly firing -> resolved; original bytes verified')
finally:
stand.restore()
"
PASS: clean queue and inactive alert -> newly firing -> resolved; original bytes verified
RESULT: PASS (exit 0) at 2026-11-21T09:04:11Z
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,42 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: '^trace_id$'
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: false
filterBySpanID: false
customQuery: true
query: '{$${__tags}} | trace_id="$${__span.traceId}"'
tags: [{ key: service.name, value: service_name }]
nodeGraph:
enabled: true
+102
View File
@@ -0,0 +1,102 @@
# HAProxy 3.0 — единственная публичная точка входа (порт 8080).
# Backend-адреса берутся из встроенного DNS Docker: имя сервиса `api` возвращает A-запись
# каждого работающего контейнера-реплики. Реплики не публикуют порты наружу.
global
log stdout format raw local0 info
maxconn 2048
# Встроенный DNS Docker. hold valid 2s — как часто перечитывается список реплик
# (новые контейнеры при scale/rolling update появляются в пуле за ≤ 2–3 s).
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold other 2s
hold refused 2s
hold nx 2s
hold timeout 2s
hold obsolete 2s
accepted_payload_size 8192
defaults
mode http
log global
option httplog
option dontlognull
# Таймауты согласованы с приложением:
# connect 1s — недоступная реплика быстро даёт ошибку соединения -> retry на другую;
# server 10s — больше statement_timeout БД (5 s) + ожидания пула (3 s);
# http-keep-alive 10s < timeout_keep_alive uvicorn (30 s): LB закрывает простой первым.
timeout connect 1s
timeout client 30s
timeout server 10s
timeout http-request 10s
timeout http-keep-alive 10s
timeout queue 5s
retries 2
option redispatch 1
frontend http_in
bind :8080
# X-Request-Id создаётся на входе (если клиент не прислал корректный), передаётся в API
# и попадает в лог LB — один ID в логах LB и реплики.
unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
acl has_valid_rid req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$
http-request set-header X-Request-Id %[unique-id] unless has_valid_rid
http-request set-var(txn.rid) req.hdr(X-Request-Id)
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.rid)] \"%HM %HPO\""
default_backend api
# Ошибки, созданные самим LB (нет живых реплик / timeout), — в формате контракта.
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Реплика API не ответила вовремя\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
backend api
# Обычный round-robin без sticky session: корректность обеспечивает общее состояние в PostgreSQL.
balance roundrobin
# Активная проверка готовности каждые 1 s; 2 неудачи подряд -> DOWN, 2 успеха -> UP.
# Максимальное время обнаружения отказа «процесс жив, но не готов»:
# inter × fall + timeout check = 1 s × 2 + 1 s ≈ 3 s.
# Упавший процесс (connection refused) обнаруживается быстрее: observe layer4 + error-limit 1
# переводит сервер в DOWN по первой реальной ошибке соединения, а запрос уходит на другую реплику.
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker init-addr none resolve-prefer ipv4 check
timeout check 1s
# Повторы на другую реплику:
# - ошибка установления соединения повторяется всегда (запрос не дошёл до приложения);
# - L7-повторы (пустой ответ / 503 от дренирующейся реплики) разрешены только для
# безопасных GET и для POST сообщения С Idempotency-Key — его повтор не создаст дубль.
# POST /users, /chats, /auth/sessions после отправки не повторяются (нет ключа идемпотентности).
retry-on conn-failure empty-response 503
acl is_message_post path_reg ^/api/v1/chats/[^/]+/messages$
acl has_idem_key req.hdr(Idempotency-Key) -m found
http-request disable-l7-retry if METH_POST !is_message_post
http-request disable-l7-retry if METH_POST !has_idem_key
http-request disable-l7-retry if METH_DELETE
# Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор
# только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся.
http-request disable-l7-retry if { path_beg /api/v1/attachments/ }
# Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном
# 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало.
http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ }
# До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес.
server-template api 8 api:8080
# Служебная страница состояния backend-ов (UP/DOWN, счётчики ошибок/повторов).
# В compose.yaml публикуется только на 127.0.0.1 хоста — не часть публичного API.
frontend stats
bind :8404
http-request return status 200 content-type text/plain string "ok" if { path /lb-health }
stats enable
stats uri /stats
stats refresh 2s
stats show-legends
+39
View File
@@ -0,0 +1,39 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
# Bound cardinality: request IDs / trace IDs remain structured metadata.
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+16 -4
View File
@@ -1,7 +1,19 @@
# Конфигурация наблюдаемости — реализовать
# Наблюдаемость lab6
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
`API / publisher / worker → OTLP HTTP → Collector → Prometheus / Loki / Tempo → Grafana`.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
* [Collector](collector.yaml): OTLP receivers, memory limiter, bounded batch,
sending queues и retry. Collector доступен только во внутренней сети Compose.
* [Prometheus](../prometheus/prometheus.yaml) хранит метрики приложения и самого
Collector; [alerts](../prometheus/alerts.yaml) оцениваются каждые 5 секунд.
* [Loki](../loki/loki.yaml) принимает native OTLP logs; технические ID —
structured metadata, не индексные labels.
* [Tempo](../tempo/tempo.yaml) хранит реальные API/producer/consumer spans.
* [Grafana](../grafana/provisioning/datasources/datasources.yaml) подключает
три datasource; [dashboard](../grafana/dashboards/messenger.json) импортируется
provisioning автоматически. Backend ports и Grafana опубликованы на loopback.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Сценарии диагностики и точные SLI/SLO: [RUNBOOK](RUNBOOK.md).
Запуск, семантика метрик, результаты и ограничения: [отчёт](../../REPORT.md).
Воспроизводимая проверка с evidence: `python -X utf8 scripts/test_lab6.py --incidents`.
Подробные сценарии находятся в `scripts/observe_lab6.py`.
+99
View File
@@ -0,0 +1,99 @@
# Диагностика стенда lab6
Правила предназначены для локального учебного стенда и синтетической нагрузки.
Пороги 15 секунд и `for: 10s` сокращены для демонстрации, а не выведены из
производственной статистики. Alert оценивается Prometheus каждые 5 секунд.
Firing и последующее отсутствие активного alert сохраняет сценарий проверки;
внешняя доставка уведомлений не требуется.
## SLI и SLO
- Доступность API: доля бизнес-ответов с кодом ниже 500 среди всех
бизнес-ответов. `/health/*` исключается, 4xx считаются доступным сервисом.
Учебный SLO: не менее 99% в скользящем окне 5 минут.
- Готовность вложения: доля завершений `ready` с временем не более 60 секунд
от устойчивого принятия до COMMIT `ready`. SLO: не менее 95% в окне 5 минут.
Доля вычисляется по `messenger_attachment_time_to_ready_seconds_bucket{le="60"}`
и `_count`; незавершённые и failed задачи отдельно видны в backlog/errors.
Поэтому этот SLI условен по успешным завершениям и один не доказывает,
что 95% всех принятых загрузок достигают ready. При остановленном worker
отсутствие завершений не интерпретируется как выполненный SLO.
- Percentile считается `histogram_quantile` после `sum by (le)` bucket rate
всех нужных реплик. Средние p95 отдельных реплик не усредняются.
## API errors
Ссылка alert: `MessengerHighErrorRate`.
1. Открыть Grafana `http://localhost:3000`, dashboard **Messenger lab6**.
Проверить интервал инцидента, RPS и 5xx ratio. Ошибки входа 401 и
недоступность PostgreSQL 503 отвечают на разные вопросы.
2. Выбрать ошибочный API trace из Tempo и связанный структурный лог по
`trace_id`; `request_id` связывает лог с `X-Request-Id` ответа.
3. Найти DB span: ошибка соединения или большая длительность DB при
коротком handler указывает на зависимость. Сравнить pool used/waiting
и DB operation duration. В spans нет текста SQL и параметров.
Если DB spans быстрые, искать время между ними/в handler, S3 или
ожидание свободного ресурса; одно значение p95 причины не устанавливает.
4. Проверить `docker compose ps db api`. В контролируемом сценарии
`docker compose stop db` ожидаются отказы синхронного пути и readiness.
HAProxy может вернуть собственный 503, не попадающий в счётчик API:
сценарий диагностики должен сохранять также прямой ответ API.
5. Восстановить `docker compose start db`, дождаться API ready, повторить
запросы. В Prometheus `http://localhost:9090/alerts` alert сначала
становится inactive после выхода ошибок из окна 1 минуты.
Сохранять ответ без токена, trace JSON, лог, запросы метрик и состояние alert
до воздействия, во время и после. Не считать текст ошибки DB в stdout
единственным доказательством причины: нужны сохранённые сигналы приложения.
## Queue lag
Ссылка alert: `MessengerQueueLag`.
1. Сравнить queue depth, oldest age, свежесть наблюдения, busy/capacity worker,
успешные/ошибочные jobs и длительность попыток. Backlog измерен по
устойчивым незавершённым заданиям PostgreSQL, включая outbox и retry;
это не только число сообщений ready в RabbitMQ.
2. Если depth > 0 и возраст растёт при отсутствии завершений и busy=0,
проверить `docker compose ps worker publisher broker`. Остановка worker
не останавливает publisher-наблюдатель.
3. Если busy=capacity, искать медленные DB/S3/image spans в worker trace.
Если растут retry/errors, найти соответствующий consumer span и событие
retry. По traceparent пройти producer → consumer → DB/S3, включая попытки.
4. Контролируемое воздействие: `docker compose stop worker`, затем несколько
upload стандартного fixture. Ответ 202 означает устойчивое принятие;
рост oldest age и firing ожидаются через 15 секунд плюс `for` и интервалы
сбора/оценки. Счётчик duration не заменяет возраст ожидающей задачи.
5. Восстановить `docker compose start worker`. Дождаться ready вложений,
depth=0 и inactive alert. Скачать original и thumbnail, проверить байты
и размеры. Уменьшение backlog само по себе ещё не доказывает сохранность.
Sample age защищает от интерпретации устаревшей очереди как свежей. Если DB
недоступна и sample age растёт, сначала расследовать DB; автоматически
приписывать это остановленному worker нельзя.
## Недоступен backend telemetry
1. Перед воздействием сохранить Collector exporter queue/failed counters,
память API/worker/Collector и контрольное сообщение/вложение.
2. `docker compose stop tempo loki` отключает хранилища traces/logs.
Prometheus остаётся включённым и сохраняет self metrics Collector:
очередь, ошибки отправки и память. Отдельно проверить недоступность
Collector, если нужно исследовать именно переполнение SDK.
3. Продолжать бизнес-запросы и загрузки. Проверить 201 сообщений, 202 → ready
вложений и чтение данных. Telemetry отсутствует в readiness; зависимость
от неё не должна отбирать все API из балансировки.
4. Collector: memory limiter 192 MiB, spike 48 MiB, batch максимум 512,
sending queue максимум 256 batches на exporter, 2 consumers; timeout 3s,
retry не более 30s на batch. После исчерпания retries данные отбрасываются.
SDK также имеет конечные очереди и timeout; экспорт выполняется фоново.
Ограничения RAM контейнеров — дополнительный предел, а не замена очередям.
5. `docker compose start tempo loki`; дождаться backend ready и повторить
контрольный запрос. Новые trace/log должны появиться. Часть старых
сигналов может потеряться: сохранённый рост failed/dropped counters и
границы измерения описываются в отчёте. Нельзя обещать полное восстановление
telemetry после перезапуска Collector: его очереди находятся в памяти.
Команды `docker compose down` сохраняют volumes. `down -v` удаляет учебные
данные и историю, для восстановления после инцидента эта команда не нужна.
+69
View File
@@ -0,0 +1,69 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:8889
metric_expiration: 5m
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 3s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlphttp/loki:
# otlphttp appends /v1/logs; Loki 3.2 accepts native OTLP here.
endpoint: http://loki:3100/otlp
timeout: 3s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
telemetry:
logs:
level: warn
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
+32
View File
@@ -0,0 +1,32 @@
groups:
- name: messenger-lab6
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: |
(
sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*",status_class="5xx"}[1m]))
/
clamp_min(sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*"}[1m])), 1e-9)
) > 0.05
for: 10s
labels:
severity: warning
annotations:
summary: "Business API 5xx ratio exceeds 5%"
description: "5xx / all business HTTP responses over 1 minute, continuously above 5% for 10 seconds. Health and client 4xx are not failures in this SLI."
runbook: "infra/otel/RUNBOOK.md#api-errors"
- alert: MessengerQueueLag
expr: |
max(messenger_queue_oldest_age_seconds{service_name="messenger-publisher"}) > 15
and
max(messenger_queue_depth{service_name="messenger-publisher"}) > 0
and
max(messenger_queue_sample_age_seconds{service_name="messenger-publisher"}) < 10
for: 10s
labels:
severity: warning
annotations:
summary: "Oldest unfinished attachment job exceeds 15 seconds"
description: "Durable PostgreSQL backlog is nonempty and oldest job age is above 15 seconds for 10 seconds; the observer sample is less than 10 seconds old. This is queue age, not worker processing duration."
runbook: "infra/otel/RUNBOOK.md#queue-lag"
+17
View File
@@ -0,0 +1,17 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: messenger
static_configs:
- targets: [otel-collector:8889]
- job_name: collector
static_configs:
- targets: [otel-collector:8888]
- job_name: prometheus
static_configs:
- targets: [localhost:9090]
+22
View File
@@ -0,0 +1,22 @@
{
"identities": [
{
"name": "admin",
"credentials": [
{ "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" }
],
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
},
{
"name": "messenger-api",
"credentials": [
{ "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" }
],
"actions": [
"Read:messenger-attachments",
"Write:messenger-attachments",
"List:messenger-attachments"
]
}
]
}
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand.
#
# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the
# container only (-ip.bind): their HTTP API has no authentication and must
# not be reachable from the Docker network. Only the S3 gateway (9000)
# listens on all interfaces, and it requires SigV4 credentials.
# - Identities and rights come from s3.json.template; the secret values are
# substituted at start from Docker secrets into a file readable by root
# only (never stored in Git or in the image).
set -eu
umask 077
s() { cat "/run/secrets/$1"; }
sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \
-e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \
-e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \
-e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \
/opt/lab/s3.json.template > /tmp/s3.json
exec weed -logtostderr=true server \
-dir=/data \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
-master.volumeSizeLimitMB=256 -volume.max=0 \
-s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json
+25
View File
@@ -0,0 +1,25 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
trace_idle_period: 5s
max_block_duration: 1m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+48
View File
@@ -0,0 +1,48 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE,
display_name VARCHAR(100) NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT users_username_format_check
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_nonblank_check
CHECK (char_length(btrim(display_name)) >= 1)
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title VARCHAR(255) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT chats_title_nonblank_check
CHECK (char_length(btrim(title)) >= 1)
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text VARCHAR(4000) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT messages_text_nonblank_check
CHECK (char_length(btrim(text)) >= 1)
);
CREATE TABLE IF NOT EXISTS sessions (
id UUID PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
token_hash CHAR(64) NOT NULL UNIQUE,
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
+12
View File
@@ -0,0 +1,12 @@
CREATE INDEX IF NOT EXISTS messages_keyset_idx
ON messages (chat_id, created_at ASC, id ASC);
CREATE INDEX IF NOT EXISTS chats_keyset_idx
ON chats (created_at ASC, id ASC);
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
ON chat_members (user_id, chat_id);
CREATE INDEX IF NOT EXISTS sessions_active_token_idx
ON sessions (token_hash, expires_at)
WHERE revoked_at IS NULL;
+18
View File
@@ -0,0 +1,18 @@
-- Lab3: shared idempotency of POST /chats/{chat_id}/messages.
-- Scope: user + method + chat path + key. The key row and the message are
-- written in one transaction, so either both exist or neither does.
-- Expand-only change: the running lab2/lab3 API keeps working with it.
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
method VARCHAR(8) NOT NULL,
key VARCHAR(128) NOT NULL,
request_hash CHAR(64) NOT NULL,
message_id UUID REFERENCES messages(id) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, method, key)
);
-- For a periodic cleanup of keys older than IDEMPOTENCY_TTL_SECONDS.
CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx
ON idempotency_keys (created_at);
+68
View File
@@ -0,0 +1,68 @@
-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private
-- S3 bucket under a server-generated object_key. Expand-only migration: the
-- lab3 API keeps working on this schema.
-- Text may be empty when the message has attachments; "not blank without
-- attachments" is checked by the API (one message + its links are inserted
-- in one transaction).
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check;
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT,
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
-- Display name only; never used as a path or an S3 key.
filename VARCHAR(255) NOT NULL,
content_type VARCHAR(32) NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 CHAR(64) NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
-- Lab4 has only "ready": the row is inserted after the object is stored.
status VARCHAR(16) NOT NULL DEFAULT 'ready',
error_code VARCHAR(64),
object_key TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT attachments_content_type_check
CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_check
CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_check
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_status_check
CHECK (status IN ('ready'))
);
CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id);
-- A message references 0-10 attachments of its own chat; "position" keeps
-- the order of attachment_ids. One attachment may be referenced by several
-- messages of the chat.
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
position SMALLINT NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position)
);
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
ON message_attachments (attachment_id);
-- Upload intent: committed BEFORE the S3 PUT and deleted in the same
-- transaction that inserts the attachment row. An intent older than the
-- grace period therefore means "the object may exist in S3, but the request
-- never committed" (crash, lost DB, client gone) -> the janitor deletes the
-- object and then the intent.
CREATE TABLE IF NOT EXISTS upload_intents (
object_key TEXT PRIMARY KEY,
attachment_id UUID NOT NULL,
chat_id UUID NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS upload_intents_created_idx
ON upload_intents (created_at);
+49
View File
@@ -0,0 +1,49 @@
-- Lab5: the database owns the work intent; RabbitMQ carries only its IDs.
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
ALTER TABLE attachments
ADD COLUMN staging_key TEXT,
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
ADD COLUMN job_id UUID UNIQUE,
ADD COLUMN attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
ADD COLUMN attempts_run INTEGER NOT NULL DEFAULT 0,
ADD COLUMN lease_token UUID,
ADD COLUMN lease_until TIMESTAMPTZ,
ADD COLUMN next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
ADD COLUMN staging_deleted BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'::jsonb;
CREATE TABLE outbox (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL,
attachment_id UUID NOT NULL REFERENCES attachments(id),
attempt INTEGER NOT NULL DEFAULT 1,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX outbox_pending_idx ON outbox(available_at, id) WHERE published_at IS NULL;
CREATE INDEX attachments_lease_idx ON attachments(lease_until) WHERE status = 'processing';
CREATE INDEX attachments_staging_idx ON attachments(updated_at)
WHERE staging_key IS NOT NULL AND NOT staging_deleted;
-- Failed registry = attachments WHERE status='failed'. This append-only
-- journal explains retry, lease recovery and explicit operator replay.
CREATE TABLE job_events (
id BIGSERIAL PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id),
event TEXT NOT NULL,
attempt INTEGER NOT NULL,
detail TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX job_events_attachment_idx ON job_events(attachment_id, id);
-- Internal one-shot crash injection, never exposed through public HTTP.
CREATE TABLE fault_injections (
point TEXT PRIMARY KEY,
remaining INTEGER NOT NULL DEFAULT 0 CHECK (remaining >= 0),
hits INTEGER NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
+4
View File
@@ -0,0 +1,4 @@
-- W3C context is part of the durable work intent, not an in-memory cache.
-- Retry attempts retain the latest causal span; business payload is unchanged.
ALTER TABLE attachments ADD COLUMN traceparent TEXT, ADD COLUMN tracestate TEXT;
ALTER TABLE outbox ADD COLUMN traceparent TEXT, ADD COLUMN tracestate TEXT;
+20
View File
@@ -0,0 +1,20 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
pydantic==2.9.2
SQLAlchemy==2.0.36
asyncpg==0.30.0
argon2-cffi==23.1.0
redis==5.0.8
# Lab4: S3 client (async) and image header/JPEG decoding
aiobotocore==2.15.2
botocore==1.35.36
aiohttp==3.10.10
Pillow==11.0.0
# Lab5: durable RabbitMQ messages and publisher confirmations.
aio-pika==9.4.3
# Explicit OTLP/HTTP SDK; semantic conventions bundled as 0.49b2.
opentelemetry-api==1.28.2
opentelemetry-sdk==1.28.2
opentelemetry-exporter-otlp-proto-http==1.28.2
+32
View File
@@ -0,0 +1,32 @@
# Creates the same local secret files as init-secrets.sh on Windows.
# Existing values are preserved and never printed.
$ErrorActionPreference = 'Stop'
$secretDirectory = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../infra/secrets'))
[IO.Directory]::CreateDirectory($secretDirectory) | Out-Null
function New-RandomHex {
$bytes = New-Object byte[] 24
$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create()
try { $randomGenerator.GetBytes($bytes) } finally { $randomGenerator.Dispose() }
return ([BitConverter]::ToString($bytes)).Replace('-', '').ToLowerInvariant()
}
function Write-NewSecret([string]$Name, [string]$Value) {
$secretPath = Join-Path $secretDirectory $Name
if (Test-Path -LiteralPath $secretPath) { Write-Output "keep infra/secrets/$Name" }
else {
[IO.File]::WriteAllText($secretPath, $Value, (New-Object Text.UTF8Encoding $false))
Write-Output "create infra/secrets/$Name"
}
}
Write-NewSecret 'postgres_password.txt' (New-RandomHex)
$databasePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'postgres_password.txt')).Trim()
Write-NewSecret 'database_url.txt' "postgresql://messenger:${databasePassword}@db:5432/messenger"
Write-NewSecret 'valkey_password.txt' (New-RandomHex)
$cachePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'valkey_password.txt')).Trim()
Write-NewSecret 'cache_url.txt' "redis://:${cachePassword}@valkey:6379/0"
Write-NewSecret 's3_admin_access_key_id.txt' ('admin' + (New-RandomHex).Substring(0,15))
Write-NewSecret 's3_admin_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
Write-NewSecret 's3_access_key_id.txt' ('api' + (New-RandomHex).Substring(0,17))
Write-NewSecret 's3_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
Write-NewSecret 'broker_password.txt' (New-RandomHex)
$queuePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'broker_password.txt')).Trim()
Write-NewSecret 'broker_url.txt' "amqp://messenger:${queuePassword}@broker:5672/"
+51
View File
@@ -0,0 +1,51 @@
#!/bin/sh
# Creates local Docker secrets for a synthetic lab stand.
# Existing files are never overwritten; values are never printed.
set -eu
dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets"
mkdir -p "$dir"
# Host access is limited by the directory mode; the files themselves must stay
# readable by the non-root users inside containers (bind-mounted secrets).
chmod 700 "$dir" 2>/dev/null || true
random() {
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
}
write_new() {
if [ -f "$dir/$1" ]; then
echo "keep infra/secrets/$1"
else
printf '%s' "$2" > "$dir/$1"
chmod 644 "$dir/$1" 2>/dev/null || true
echo "create infra/secrets/$1"
fi
}
if [ ! -f "$dir/postgres_password.txt" ]; then
write_new postgres_password.txt "$(random)"
fi
pg_password="$(cat "$dir/postgres_password.txt")"
write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger"
if [ ! -f "$dir/valkey_password.txt" ]; then
write_new valkey_password.txt "$(random)"
fi
valkey_password="$(cat "$dir/valkey_password.txt")"
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
# bootstrap; the API gets its own key pair restricted to one bucket.
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
write_new s3_secret_access_key.txt "$(random)$(random)"
# Lab5: RabbitMQ uses a separate local password. The URL is a Docker secret;
# no credentials appear in compose.yaml or the report.
if [ ! -f "$dir/broker_password.txt" ]; then
write_new broker_password.txt "$(random)"
fi
broker_password="$(cat "$dir/broker_password.txt")"
write_new broker_url.txt "amqp://messenger:${broker_password}@broker:5672/"
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown.
# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh.
set -eu
: "${TARGET_INSTANCE_ID:?target API instance alias required}"
. "$(dirname "$0")/lib.sh"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
esac
for id in $(service_ids api); do
case "$id" in "$TARGET_INSTANCE_ID"*)
docker kill --signal KILL "$id" >/dev/null
echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)"
exit 0 ;;
esac
done
echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2
exit 3
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
PROJECT="${COMPOSE_PROJECT:-messenger-lab6}"
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
service_ids() {
# $1 — сервис, $2 — необязательный флаг -a
docker ps ${2:-} -q --no-trunc \
--filter "label=com.docker.compose.project=$PROJECT" \
--filter "label=com.docker.compose.service=$1"
}
# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом).
# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть
# переменные вызывающего цикла.
wait_healthy() {
_wh_timeout="$1"; shift
_wh_deadline=$(( $(date +%s) + _wh_timeout ))
for _wh_id in "$@"; do
while :; do
_wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing)
case "$_wh_state" in healthy|running) break ;; esac
if [ "$(date +%s)" -ge "$_wh_deadline" ]; then
echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2
return 1
fi
sleep 1
done
done
}
short() { echo "$1" | cut -c1-12; }

Some files were not shown because too many files have changed in this diff Show More