WIP: ЛР 6 - БВТ2303 - Максимова— на 4 #70
@@ -18,3 +18,9 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
infra/secrets
|
||||
Защита.md
|
||||
ЗАЩИТА.md
|
||||
|
||||
защита.md
|
||||
|
||||
+16
-4
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -19,12 +19,24 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
# The current consumer is sequential; this documents capacity, not a tuning knob.
|
||||
WORKER_CONCURRENCY=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
JOB_TIMEOUT_SECONDS=30
|
||||
JOB_TOTAL_TIMEOUT_SECONDS=90
|
||||
JOB_LEASE_SECONDS=120
|
||||
STAGING_RETENTION_SECONDS=604800
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
|
||||
# Actual Compose overrides (all other application settings have defaults).
|
||||
LB_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
GRAFANA_PORT=3000
|
||||
PROMETHEUS_PORT=9090
|
||||
LOKI_PORT=3100
|
||||
TEMPO_PORT=3200
|
||||
# If another lab uses 8080/8404, choose e.g. 8086/8406 locally.
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.yaml text eol=lf
|
||||
Dockerfile* text eol=lf
|
||||
@@ -0,0 +1,20 @@
|
||||
name: Application smoke
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: sh scripts/init-secrets.sh
|
||||
- run: docker compose up -d --build --wait --wait-timeout 240
|
||||
- run: docker compose --profile tools run --rm -T --no-deps tester make check test
|
||||
- run: docker compose exec -T api python - < student-tests/observability_unit.py
|
||||
- run: python scripts/observe_lab6.py --baseline
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
@@ -30,3 +30,8 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
# Local defense preparation
|
||||
/защита.md
|
||||
/Защита.md
|
||||
/ЗАЩИТА.md
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
# Messenger lab5: one image for API, publisher, worker, migrations and bootstrap.
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/
|
||||
COPY migrations ./migrations
|
||||
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Exec form: python is PID 1 and receives SIGTERM from "docker stop" itself.
|
||||
# (lab2 used "sh -c uvicorn ...": sh stayed PID 1, uvicorn never got SIGTERM
|
||||
# and was killed together with the container instead of draining.)
|
||||
CMD ["python", "-m", "app.server"]
|
||||
@@ -1,37 +1,303 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: Максимова Мария, БВТ2303. Целевая оценка: 4.
|
||||
|
||||
## Перенос lab5
|
||||
|
||||
Исходники перенесены из соседнего репозитория `mtusi2026lab5`, версия
|
||||
`19cbacd`. Перенесены приложение, миграции 001–005, Dockerfile, Compose,
|
||||
операторские скрипты, собственные проверки. Контракт и публичные тесты lab6
|
||||
сохранены. Доказательства старой лабораторной не выдаются за результаты lab6.
|
||||
|
||||
Архитектура: HAProxy → две реплики FastAPI → PostgreSQL/Valkey и приватное
|
||||
S3 (SeaweedFS). До 202 сохраняются S3 staging и транзакционный outbox в БД.
|
||||
Publisher доставляет задания в RabbitMQ, worker публикует original,
|
||||
thumbnail JPEG 256×256, crop JPEG 128×128 и optimized WebP 1280×1280.
|
||||
Применяются at least once, publisher confirms, ack после commit,
|
||||
lease с fencing token, ограниченные retry с jitter и журнал ошибок.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine/Compose v2 (Linux containers), доступ к registry/PyPI,
|
||||
около 8 GiB RAM у Docker и Python 3.9+ для stdlib HTTP-проверок.
|
||||
Образ приложения: Python 3.12.7; PostgreSQL 16.4, Valkey 8.0.1,
|
||||
SeaweedFS 3.97, RabbitMQ 3.13.7, HAProxy 3.0.11. Python-зависимости
|
||||
закреплены в `requirements.txt`. `Dockerfile` общий для API/publisher/worker.
|
||||
|
||||
## Архитектура и решения
|
||||
Windows PowerShell:
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File scripts/init-secrets.ps1
|
||||
Copy-Item .env.example .env
|
||||
# Если заняты 8080/8404, изменить LB_PORT/LB_STATS_PORT в .env.
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait --wait-timeout 240
|
||||
docker compose --profile tools build tester
|
||||
docker compose --profile tools run --rm -T --no-deps tester make check test
|
||||
$env:BASE_URL = 'http://localhost:8080' # либо выбранный LB_PORT
|
||||
python -X utf8 scripts/observe_lab6.py --baseline
|
||||
python -X utf8 scripts/observe_lab6.py --chain-with-retry
|
||||
python -X utf8 scripts/observe_lab6.py --incidents
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
Linux/macOS: `sh scripts/init-secrets.sh`, `cp .env.example .env`, те же
|
||||
команды Compose; `BASE_URL=http://localhost:8080 python3 scripts/observe_lab6.py ...`.
|
||||
Unit: `docker compose exec -T api python - < student-tests/observability_unit.py`
|
||||
(на PowerShell: `Get-Content -Raw student-tests/observability_unit.py | docker compose exec -T api python -`).
|
||||
Для полной схемы: `python -m venv .venv`, установить `requirements-dev.txt`
|
||||
в venv, затем `python -X utf8 scripts/validate_openapi.py` из этого venv.
|
||||
Собственные проверки изображений/восстановления: `python -X utf8 scripts/test_lab5.py`
|
||||
(без `--quick` выполняются намеренные отказы). Не запускать сценарии отказов параллельно.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Init-скрипты создают случайные локальные credentials в `infra/secrets/`,
|
||||
существующие сохраняют; файлы исключены из Git и Docker build context.
|
||||
`migrate` до запуска API последовательно применяет миграции 001–006 под
|
||||
блокировкой и журналирует их; повторный запуск пропускает уже применённые.
|
||||
`s3-init` создаёт приватный bucket и проверяет права. Ручной seed не нужен.
|
||||
`--wait` проверяет health основных компонентов; baseline дополнительно
|
||||
ожидает поступление реальных данных в backend.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Безопасная остановка: `docker compose down`. Данные сохраняются в volumes.
|
||||
**Только для намеренного сброса учебных данных:** `docker compose down -v`.
|
||||
Эта команда удалит и бизнес-данные, и сохранённую телеметрию.
|
||||
В текущем прогоне используются локальные порты API **8086**, LB stats **8406**,
|
||||
Grafana **3000**, Prometheus **9090**, Loki **3100**, Tempo **3200**.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Наблюдаемость: уровень 3
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
API, worker и publisher используют OpenTelemetry Python SDK 1.28.2,
|
||||
OTLP HTTP/protobuf → Collector contrib 0.111.0 → Prometheus 2.55.0,
|
||||
Loki 3.2.1 (native OTLP), Tempo 2.6.1 → Grafana 11.2.2.
|
||||
Полные конфигурации и импортируемый dashboard находятся в `infra/`.
|
||||
Данные backend и состояние Grafana сохраняются в именованных volumes.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Метрики SDK `messenger.http.requests` и `messenger.jobs.processed` — counters;
|
||||
`messenger.http.duration` и `messenger.jobs.duration` — histograms в секундах.
|
||||
Prometheus добавляет `_total` к counters и `_seconds_bucket/sum/count` к
|
||||
histograms. Labels HTTP: method, route template, status_class; jobs:
|
||||
job_type и outcome. Resource attributes: service.name, service.version=1.6.0,
|
||||
service.instance.id, deployment.environment.name=lab. Сервисы называются
|
||||
messenger-api, messenger-worker и messenger-publisher.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
API log `http.request.complete` связывается с ответом по X-Request-Id и
|
||||
X-Trace-Id. Worker log `job.complete` связывается с реальным CONSUMER span
|
||||
по trace_id/span_id. В логах есть timestamp/severity/event и service resource.
|
||||
Технические job/attachment ID разрешены только в traces/logs синтетического
|
||||
стенда; метрики не используют идентификаторы объектов.
|
||||
SDK экспортирует только явные allowlist-события, без текстов сообщений,
|
||||
паролей, токенов, SQL и exception text. Uvicorn access log выключен.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Grafana доступна только через loopback http://localhost:3000 (учебный
|
||||
anonymous Viewer); dashboard **Messenger — lab6**, три datasource заданы
|
||||
provisioning автоматически. Prometheus/Loki/Tempo на loopback
|
||||
9090/3100/3200 нужны также для воспроизводимых проверок.
|
||||
OTLP порты доступны только внутри сети Compose.
|
||||
|
||||
Collector применяет memory_limiter (192 MiB), batch (не больше 512 records),
|
||||
очереди exporters по 256 batches и retry до 30 секунд. У SDK фоновые очереди
|
||||
traces/logs по 512 records, batches по 64, HTTP timeout 2 секунды, период
|
||||
метрик 5 секунд. Ни API request, ни job не ожидают сетевой экспорт.
|
||||
Зависимости telemetry не входят в readiness приложения.
|
||||
|
||||
Источники: [OTel SDK](https://opentelemetry-python.readthedocs.io/en/stable/sdk/),
|
||||
[Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
|
||||
[Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
|
||||
Реализация и документация подготовлены с помощью Codex.
|
||||
|
||||
## Уровень 4: причинная связь и насыщение
|
||||
|
||||
Миграция 006 добавляет nullable W3C carrier в attachments/outbox без изменения
|
||||
публичных схем. API сохраняет carrier в той же транзакции, что и metadata/outbox;
|
||||
publisher извлекает его, создаёт PRODUCER span, записывает новый carrier в
|
||||
envelope. Worker создаёт CONSUMER span с этим parent, сохраняет consumer
|
||||
context при claim для lease recovery. Следующая попытка получает новый span ID
|
||||
в той же трассе. Operator replay/redeliver сохраняет причинный контекст.
|
||||
Транзакционная граница outbox и семантика at least once остаются прежними.
|
||||
|
||||
Внутренние spans: `db.acquire`, `db.query`, `db.commit`, `db.health`,
|
||||
`s3.PutObject/GetObject/DeleteObject/HeadObject`, `image.generate`.
|
||||
SQL span содержит только тип операции, exception — только имя класса.
|
||||
GetObject измеряет получение headers; чтение streaming body продолжается
|
||||
внутри вызывающей операции, поэтому этот span не является полной download latency.
|
||||
|
||||
| Prometheus metric | Смысл и единица |
|
||||
| --- | --- |
|
||||
| `messenger_queue_depth` | queued + processing в PostgreSQL, jobs; включает outbox и delayed retry |
|
||||
| `messenger_queue_oldest_age_seconds` | возраст старейшей незавершённой job, секунды |
|
||||
| `messenger_queue_sample_age_seconds` | возраст последнего успешного наблюдения, секунды |
|
||||
| `messenger_db_pool_used` / `waiting` / `capacity` | занятые соединения / acquisition requests in progress / лимит соединений |
|
||||
| `messenger_worker_busy` / `capacity` | занятая обработка / доступная ёмкость (один consumer, capacity=1) |
|
||||
| `messenger_process_cpu_seconds_total` | CPU time родительского процесса, секунды |
|
||||
| `messenger_process_memory_bytes` | текущая RSS родительского процесса, байты |
|
||||
| `messenger_attachment_time_to_ready_seconds_*` | histogram от created_at принятого attachment до успешного COMMIT ready |
|
||||
|
||||
Waiting включает создание/проверку соединения, а не только очередь при
|
||||
исчерпанном pool. Метрики CPU/RSS не включают подпроцесс декодера; контейнер
|
||||
дополнительно ограничен Docker. Backlog измеряет publisher независимо от worker,
|
||||
sample age позволяет обнаружить устаревшее наблюдение при отказе БД.
|
||||
`WORKER_CONCURRENCY` из логических рекомендаций skeleton не используется:
|
||||
реальный worker последователен, prefetch=1; scale worker потребует отдельной проверки.
|
||||
Dashboard фильтрует worker resources по heartbeat не старше 15 секунд,
|
||||
затем суммирует busy/capacity. Так старые process resources после restart
|
||||
не завышают ёмкость и не показывают мёртвый busy worker. Collector удаляет
|
||||
не обновляемые серии через 5 минут; heartbeat делает эту задержку явной.
|
||||
|
||||
Пример p95 API в секундах (для p50/p99 меняется только первый аргумент):
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95,
|
||||
sum by (le) (rate(messenger_http_duration_seconds_bucket{
|
||||
service_name="messenger-api",route!~"/health/.*"}[5m])))
|
||||
```
|
||||
|
||||
Сначала объединяются buckets реплик с одинаковыми границами, потом вычисляется
|
||||
percentile. Buckets SDK: 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5,
|
||||
5, 10, 30, 60, 120, +Inf секунд. Это приближённая оценка, не среднее p95.
|
||||
Все запросы, легенды и единицы находятся в импортируемом dashboard JSON.
|
||||
|
||||
SLI доступности = 1 − (5xx / все бизнес-ответы), health исключены, 4xx не
|
||||
считаются отказом сервера. Учебный SLO: ≥99% за скользящие 5 минут.
|
||||
Дополнительный SLO: ≥95% успешных ready-завершений за ≤60 секунд в окне 5 минут
|
||||
по histogram; он условен по завершениям, pending/failed отдельно контролируются
|
||||
backlog/errors. При отсутствии трафика или завершений выполнение SLO не доказано.
|
||||
Alert high error rate: >5% за 1 минуту непрерывно 10 секунд. Queue lag:
|
||||
age >15 секунд при depth >0 и свежем sample <10 секунд непрерывно 10 секунд.
|
||||
Оценка каждые 5 секунд; короткие пороги выбраны для защиты, а не production SLA.
|
||||
Правила — `infra/prometheus/alerts.yaml`, диагностика —
|
||||
[RUNBOOK](infra/otel/RUNBOOK.md). Доставка уведомлений наружу не используется.
|
||||
|
||||
## Конфигурация и границы устойчивости
|
||||
|
||||
`.env.example` содержит логические рекомендации skeleton. Реально Compose
|
||||
передаёт DATABASE_URL_FILE, CACHE_URL_FILE, BROKER_URL_FILE и S3 *_FILE через
|
||||
secrets; сессии кэшируются в Valkey, источник истины PostgreSQL. DB pool
|
||||
10+5 соединений на процесс, pool/connect timeout 3 секунды, statement timeout
|
||||
5 секунд. Upload: ≤10 MiB и ≤40M pixels; spool RAM 1 MiB на upload,
|
||||
не более 8 concurrent uploads на API. Worker: image timeout 30 секунд,
|
||||
total job timeout 90, lease 120, максимум 5 попыток, exponential backoff+jitter.
|
||||
Thumbnail JPEG quality=85, crop quality=85, optimized WebP quality=80,
|
||||
metadata EXIF удаляется после применения orientation; original сохраняется побайтно.
|
||||
|
||||
Порты, API_REPLICAS, SESSION_TTL_SECONDS, IDEMPOTENCY_TTL_SECONDS,
|
||||
JOB_* и RETRY_BASE_SECONDS можно переопределить через `.env` по `compose.yaml`.
|
||||
Для локальных OTLP-ресурсов выбраны имена messenger-api/worker/publisher;
|
||||
endpoint и resource заданы явно в Compose. `OTEL_SDK_DISABLED=true` отключает SDK.
|
||||
API readiness зависит от PostgreSQL и drain; S3 влияет на вложения, а broker,
|
||||
worker и telemetry не выключают весь синхронный путь сообщений.
|
||||
|
||||
Две API-реплики выдерживают остановку одной. Postgres/S3/broker volumes
|
||||
переживают перезапуск контейнеров; outbox, publisher confirms, ack после commit,
|
||||
идемпотентная обработка и fencing token защищают от окон потери/дублей.
|
||||
Один Docker-хост, один LB, DB, S3, broker и Collector — точки отказа.
|
||||
Это не кластер с независимыми узлами и не backup; отказ диска/хоста здесь
|
||||
не покрыт. HTTP разрешён только для локального синтетического стенда.
|
||||
|
||||
Telemetry экспортируется фоново с ограниченной памятью. При недоступности
|
||||
backend Collector повторяет до 30 секунд, затем отбрасывает batch; при заполнении
|
||||
очереди возможны дополнительные потери. SDK counters export_failures и
|
||||
dropped_records отражают неуспешные exports; overflow SDK queue отдельно
|
||||
не подсчитывается. Collector send_failed — число неуспешных попыток с records,
|
||||
не число уникально потерянных spans. Отказ наблюдаемости не должен останавливать
|
||||
бизнес-записи. Семидневный retention задан для всех трёх сигналов;
|
||||
Prometheus дополнительно ограничен 512 MB, остальные backend ограничены
|
||||
контейнерами и свободным местом учебного Docker-диска.
|
||||
|
||||
## Методика и результаты проверки
|
||||
|
||||
Проверки выполнены 21.11.2026 на Windows/Docker Desktop 29.3.0, Linux
|
||||
containers, Intel Core Ultra 5 125U. Docker видит 14 CPU и 7 965 700 096 bytes
|
||||
RAM. Лимиты ресурсов каждого сервиса закреплены в Compose; это один хост.
|
||||
Телеметрия включена, sampling полный; данные синтетические.
|
||||
Результаты сняты из рабочего дерева уровня 4 после коммита `d0a062e` и
|
||||
включены в третий коммит. История: `git log -4 --oneline`.
|
||||
|
||||
Профиль нагрузки: публичный `tests/load.py`, closed loop POST messages,
|
||||
concurrency=4, длительность 20 секунд, один прогон, без автоматических retry.
|
||||
Отдельный длительный прогрев не выполнялся; до нагрузки стенд работал
|
||||
несколько минут и прошёл smoke. Создано **1645 сообщений**, ошибок **0**,
|
||||
успешные RPS **81.154**, p50 **38.796 ms**, p95 **114.354 ms**, p99 **194.742 ms**.
|
||||
Это client-side latency успешных записей; ошибки учитываются отдельно,
|
||||
closed loop имеет coordinated omission. Этот прогон не доказывает максимальную
|
||||
пропускную способность или выполнение SLO за длительный период.
|
||||
Данные: [load-summary.json](evidence/load-summary.json),
|
||||
[команда/stdout](evidence/load.txt).
|
||||
|
||||
За тот же интервал Prometheus зафиксировал максимальную RSS одного API
|
||||
128 040 960 bytes, worker 89 513 984, publisher 89 305 088. Максимальный
|
||||
15-секундный rate CPU одного API 0.389 CPU cores, worker 0.001,
|
||||
publisher 0.021. Это дискретные samples родительских процессов, не суммарные
|
||||
CPU/RAM всех контейнеров; image worker подпроцесс не включён.
|
||||
[Сырые запросы и range samples](evidence/load-resources.json).
|
||||
|
||||
| Критерий | Команда / сценарий | Доказательство |
|
||||
| --- | --- | --- |
|
||||
| Контракт и публичный smoke с telemetry | `python -X utf8 scripts/test_lab6.py` | [make-check-test.txt](evidence/make-check-test.txt) |
|
||||
| Полная OpenAPI / JSON Schema | venv Python, `scripts/validate_openapi.py` | [validate.txt](evidence/validate.txt) |
|
||||
| SDK, корреляция, W3C, отсутствие утечек | `student-tests/observability_unit.py` в API image | [5 tests](evidence/observability-unit.txt) |
|
||||
| Метрики HTTP / jobs, реальные API и worker logs/traces | `scripts/observe_lab6.py --baseline` | [baseline](evidence/observability-baseline.json), [trace](evidence/baseline-api-trace.json), [API logs](evidence/baseline-api-logs.json), [worker logs](evidence/baseline-worker-logs.json) |
|
||||
| Устойчивый upload, изображения и лимиты lab5 | `scripts/test_lab5.py --quick` | [images](evidence/images.txt), [decoder limits](evidence/limits.txt) |
|
||||
| Сохранность после restart | public persistence с `scripts/restart.sh` | [persistence.txt](evidence/persistence.txt) |
|
||||
| Смена API-реплики без потери сессии/данных | public failover с `scripts/stop-one.sh` | [failover.txt](evidence/failover.txt) |
|
||||
| Outbox gap, crash до commit / до ack, retry exhaustion, replay и duplicate no-op | `scripts/test_lab5.py` (полный прогон с telemetry) | [recovery.txt](evidence/recovery.txt) |
|
||||
|
||||
Метрики/логи/трассы из evidence запрошены у настоящих backend; это не
|
||||
статические фиктивные примеры. JSON dashboard и provisioning находятся в Git,
|
||||
а не только в скриншоте. Содержимое вывода и JSON проверяется на текущие
|
||||
синтетические secret markers и запрещённые telemetry keys/metric labels.
|
||||
Собственная проверка oversized upload отправляет заведомо чрезмерный
|
||||
Content-Length и проверяет ранний 413 до передачи body: urllib может потерять
|
||||
ранний ответ, продолжая писать уже отклонённые 10 MiB. Публичные тесты не менялись.
|
||||
|
||||
## Контролируемые отказы и диагностика
|
||||
|
||||
Общие данные: [observability-incidents.json](evidence/observability-incidents.json).
|
||||
Сценарии восстанавливают остановленные сервисы в `finally`; worker запускается
|
||||
через точные Docker container IDs, чтобы Compose не поднял одновременно
|
||||
искусственно остановленную зависимость. Перед проверкой alerts ожидаются
|
||||
пустой backlog и inactive alert, чтобы не принять старое срабатывание за новое.
|
||||
|
||||
| Воздействие | Наблюдение и причина | Восстановление |
|
||||
| --- | --- | --- |
|
||||
| Worker остановлен, приняты четыре upload (09:03:12–09:04:10 UTC) | Растут depth/oldest age при свежем publisher sample. `MessengerQueueLag` activeAt 09:03:35, затем firing. API продолжает выдавать 202; причина — отсутствие consumer, не длительность HTTP | После старта все четыре ready, оригиналы побайтно и SHA-256 совпали, thumbnail 256×160. Alert resolved в 09:04:10; полный сценарий 58 секунд |
|
||||
| PostgreSQL остановлен (08:58:31–09:00:26 UTC) | 18 прямых бизнес-запросов в две реальные API-реплики дали 503 unavailable. `MessengerHighErrorRate` activeAt 08:58:55, firing. DB error span и связанный log указывают на зависимость | БД запущена с тем же volume, readiness восстановился, сообщение записано. Alert resolved 09:00:25 после выхода ошибок из минутного окна |
|
||||
| Tempo и Loki остановлены на 65 секунд | 339 успешных записей сообщений; exporter send_failed растёт, очереди не превышают capacity=256, максимальная Collector RSS 204 MiB < контейнерного лимита 256 MiB | После старта и опустошения очередей новый upload снова имеет связанный API/worker trace и logs. Проверены все 339 контрольных trace ID и SERVER spans: отсутствующих 0; отсутствующих контрольных API logs 5 |
|
||||
|
||||
Для первого инцидента отдельный чистый прогон:
|
||||
[worker-lag-isolated.txt](evidence/worker-lag-isolated.txt).
|
||||
Для БД: [trace](evidence/incident-db-trace.json) и
|
||||
[logs](evidence/incident-db-logs.json). Запросы идут прямо в API-контейнеры,
|
||||
потому что после отрицательного readiness HAProxy сам возвращает 503;
|
||||
такой ответ LB не следует выдавать за ошибку, измеренную SDK API.
|
||||
Значение alert `activeAt` — начало pending, не точное время перехода firing;
|
||||
само состояние firing сохранено в JSON.
|
||||
|
||||
Потери измерены по известным request/trace IDs контрольных бизнес-записей
|
||||
после восстановления и опустошения очередей, с дополнительной проверкой
|
||||
наличия SERVER span. Это количество отсутствующих контрольных событий,
|
||||
а не всех внутренних spans системы. Отсутствующий log при сохранённой
|
||||
бизнес-записи показывает допустимую потерю наблюдений при ограниченном retry.
|
||||
[Сырые контрольные logs](evidence/telemetry-loss-control-logs.json) и список IDs
|
||||
с результатами поиска находятся в evidence. Нулевая потеря этих traces
|
||||
не означает гарантию сохранности telemetry при любом отказе.
|
||||
|
||||
Отдельно воспроизведены S3 outage, retry и lease recovery:
|
||||
[summary](evidence/observability-retry.json),
|
||||
[полная трасса](evidence/retry-api-trace.json),
|
||||
[worker logs](evidence/retry-worker-logs.json).
|
||||
Upload `8da6897d-7ad0-4a47-92a1-63e8820c6fb3` имеет общий trace
|
||||
`94af84b9ec1663416ac05526995fbd91`; видны error попытки 1/2,
|
||||
duplicate redelivery и success попытки 3. Проверен путь parent каждого
|
||||
consumer через producer до исходного SERVER span, а также DB/S3 children.
|
||||
Оригинал 111817 bytes сохранился, thumbnail имеет размер 256×160.
|
||||
Lease 120 секунд объясняет длительное восстановление после прерывания
|
||||
worker: бюджет smoke 60 секунд относится к свободному исправному worker,
|
||||
не к восстановлению аварийной попытки.
|
||||
|
||||
После массового чтения накопленных трасс настройки Tempo уточнены:
|
||||
GOMEMLIMIT=256 MiB, контейнер 512 MiB / 1 CPU. Итоговая конфигурация сохраняет
|
||||
ограниченную память и выдержала повторный поиск всех контрольных SERVER spans.
|
||||
Нагрузочный прогон и первоначальный backend outage сняты до этого изменения;
|
||||
бизнес-код и настройки Collector при этом не менялись.
|
||||
|
||||
Трек оценки 5 не заявлен. Нет сравнительного исследования sampling,
|
||||
доказанной production ёмкости или отказоустойчивого кластера хранилищ.
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""Low-cardinality saturation/backlog signals and safe dependency spans (lab6/4)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
import resource
|
||||
import time
|
||||
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.metrics import Observation
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.telemetry import span
|
||||
|
||||
meter = metrics.get_meter("messenger")
|
||||
waiting = 0
|
||||
busy = 0
|
||||
snapshot: dict[str, float] = {}
|
||||
ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
|
||||
retries = meter.create_counter("messenger.jobs.retries", unit="{retry}")
|
||||
|
||||
|
||||
class ObservedSession(AsyncSession):
|
||||
async def execute(self, statement, params=None, **kwargs):
|
||||
global waiting
|
||||
# Number of acquisition requests in progress (includes new connects).
|
||||
# Acquisition is separate from query execution to diagnose pool waits.
|
||||
waiting += 1
|
||||
try:
|
||||
with span("db.acquire", {"db.system": "postgresql"}):
|
||||
await self.connection()
|
||||
finally:
|
||||
waiting -= 1
|
||||
operation = str(statement).lstrip().split(None, 1)[0].upper()
|
||||
if operation not in {"SELECT", "INSERT", "UPDATE", "DELETE", "SET"}:
|
||||
operation = "OTHER"
|
||||
with span("db.query", {"db.system": "postgresql", "db.operation": operation}):
|
||||
return await super().execute(statement, params, **kwargs)
|
||||
|
||||
async def commit(self):
|
||||
with span("db.commit", {"db.system": "postgresql"}):
|
||||
return await super().commit()
|
||||
|
||||
|
||||
def install_gauges(engine) -> None:
|
||||
pool = engine.sync_engine.pool
|
||||
meter.create_observable_gauge("messenger.db.pool.used",
|
||||
callbacks=[lambda _: [Observation(pool.checkedout())]])
|
||||
meter.create_observable_gauge("messenger.db.pool.waiting",
|
||||
callbacks=[lambda _: [Observation(waiting)]])
|
||||
meter.create_observable_gauge("messenger.db.pool.capacity",
|
||||
callbacks=[lambda _: [Observation(pool.size() + pool._max_overflow)]])
|
||||
meter.create_observable_gauge("messenger.worker.busy",
|
||||
callbacks=[lambda _: [Observation(busy)]])
|
||||
capacity = 1 if os.getenv("OTEL_SERVICE_NAME") == "messenger-worker" else 0
|
||||
meter.create_observable_gauge("messenger.worker.capacity",
|
||||
callbacks=[lambda _: [Observation(capacity)]])
|
||||
if capacity:
|
||||
meter.create_observable_gauge("messenger.worker.heartbeat", unit="s",
|
||||
callbacks=[lambda _: [Observation(time.time())]])
|
||||
meter.create_observable_counter("messenger.process.cpu", unit="s",
|
||||
callbacks=[lambda _: [Observation(time.process_time())]])
|
||||
|
||||
def rss(_):
|
||||
# Linux /proc resident pages, current RSS (not ru_maxrss high watermark).
|
||||
try:
|
||||
with open("/proc/self/statm", encoding="ascii") as statm:
|
||||
value = int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
|
||||
except OSError:
|
||||
value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
|
||||
return [Observation(value)]
|
||||
|
||||
meter.create_observable_gauge("messenger.process.memory", unit="By", callbacks=[rss])
|
||||
if os.getenv("OTEL_SERVICE_NAME") == "messenger-publisher":
|
||||
for name, unit, key in (("depth", "", "depth"),
|
||||
("oldest_age", "s", "oldest_age"),
|
||||
("sample_age", "s", "sample_age")):
|
||||
def observe(_, key=key):
|
||||
if "sampled_at" not in snapshot:
|
||||
return []
|
||||
value = (time.monotonic() - snapshot["sampled_at"]
|
||||
if key == "sample_age" else snapshot[key])
|
||||
return [Observation(value, {"queue": "attachments"})]
|
||||
meter.create_observable_gauge("messenger.queue." + name, unit=unit,
|
||||
callbacks=[observe])
|
||||
|
||||
|
||||
async def backlog_loop(session_factory) -> None:
|
||||
# Authoritative logical queue = queued + processing, including durable
|
||||
# outbox and delayed retries. Remains observable when worker is stopped.
|
||||
while True:
|
||||
try:
|
||||
async with session_factory() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT count(*) AS depth,
|
||||
COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS oldest_age
|
||||
FROM attachments WHERE status IN ('queued','processing')
|
||||
"""))
|
||||
row = result.mappings().one()
|
||||
snapshot.update(depth=float(row["depth"]), oldest_age=float(row["oldest_age"]),
|
||||
sampled_at=time.monotonic())
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
# Retain last sample; sample_age makes stale observations explicit.
|
||||
pass
|
||||
await asyncio.sleep(2)
|
||||
|
||||
|
||||
def observe_ready(row) -> None:
|
||||
elapsed = (datetime.now(timezone.utc) - row["created_at"]).total_seconds()
|
||||
ready_duration.record(max(0, elapsed), {"job_type": "attachment.process.v1"})
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Attachments: durable upload to staging, queued metadata + outbox (lab5).
|
||||
|
||||
Upload, in this order (all checks before the bytes are stored):
|
||||
|
||||
1. filename (400) -> chat membership (404) -> Content-Type header (415)
|
||||
-> Content-Length (413 without reading the body);
|
||||
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
|
||||
3. the body is read in chunks into a SpooledTemporaryFile (only
|
||||
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
|
||||
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
|
||||
4. content validation in a thread (app.images): format by magic bytes,
|
||||
pixel limit from the header (413), integrity (415);
|
||||
5. upload intent COMMIT -> staging S3 PUT -> one transaction "delete intent +
|
||||
insert queued attachment + outbox" -> 202. Worker produces all variants.
|
||||
|
||||
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
|
||||
in the background):
|
||||
- PUT failed or timed out: the outcome is unknown - S3 may still finish
|
||||
writing the object AFTER our DELETE. So the object is deleted best
|
||||
effort, but the intent is KEPT; the janitor deletes the object again
|
||||
after UPLOAD_GRACE_SECONDS and only then the intent;
|
||||
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
|
||||
and only the answer was lost. Under the intent's row lock: no intent ->
|
||||
the row is committed, keep the object; else delete object, then intent;
|
||||
- the process died after PUT: the intent stays; the janitor deletes the
|
||||
object and the intent after UPLOAD_GRACE_SECONDS.
|
||||
So an object without a row exists only while its intent exists, and an
|
||||
object of a committed row is never deleted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from app import repositories
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.errors import (
|
||||
ApiError,
|
||||
Conflict,
|
||||
InvalidRequest,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
RateLimited,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
from app.images import InvalidImage, TooManyPixels, validate_image
|
||||
from app.storage import ObjectMissing, StorageUnavailable, storage
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.attachments")
|
||||
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
# Characters that can turn a display name into a path or hide its real
|
||||
# extension: separators, NUL/control characters, bidi overrides etc.
|
||||
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
|
||||
|
||||
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
|
||||
# Full integrity checks are CPU work too. In particular, progressive JPEG
|
||||
# retains decoder coefficients even with draft(); never run eight at once.
|
||||
validation_slot = asyncio.Semaphore(1)
|
||||
# Strong references to background compensation tasks (asyncio keeps only
|
||||
# weak ones); if the process dies first, the intent stays for the janitor.
|
||||
_compensations: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [
|
||||
{key: value for key, value in variant.items() if key != "object_key"}
|
||||
for variant in _variants(row)
|
||||
],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def _variants(row: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
value = row.get("variants") or []
|
||||
return json.loads(value) if isinstance(value, str) else value
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> str:
|
||||
if not 1 <= len(filename) <= 255 or not filename.strip():
|
||||
raise InvalidRequest("filename must be 1-255 characters, not blank.")
|
||||
|
||||
if "/" in filename or "\\" in filename:
|
||||
raise InvalidRequest("filename must not contain path separators.")
|
||||
|
||||
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
|
||||
raise InvalidRequest("filename must not contain control characters.")
|
||||
|
||||
if filename.strip(". ") == "":
|
||||
raise InvalidRequest("filename must not be a relative path.")
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def declared_content_type(request: Request) -> str:
|
||||
raw = request.headers.get("Content-Type", "")
|
||||
media_type = raw.split(";", 1)[0].strip().lower()
|
||||
|
||||
if media_type not in ALLOWED_CONTENT_TYPES:
|
||||
raise UnsupportedMediaType(
|
||||
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
|
||||
)
|
||||
|
||||
return media_type
|
||||
|
||||
|
||||
def _too_large(message: str) -> PayloadTooLarge:
|
||||
error = PayloadTooLarge(message)
|
||||
# The rest of the body is not read: the connection cannot be reused.
|
||||
error.headers["Connection"] = "close"
|
||||
return error
|
||||
|
||||
|
||||
def check_declared_length(request: Request) -> None:
|
||||
raw = request.headers.get("Content-Length")
|
||||
if raw is None:
|
||||
return # chunked: the limit is enforced while reading
|
||||
|
||||
if not raw.isdigit():
|
||||
raise InvalidRequest("Invalid Content-Length.")
|
||||
|
||||
if int(raw) > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
|
||||
|
||||
async def _receive_body(request: Request, spool) -> tuple[int, str]:
|
||||
"""Stream the body into the spool; return (size, sha256)."""
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_timeout_seconds):
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise ApiError(
|
||||
400,
|
||||
"invalid_request",
|
||||
"Upload was not completed in time.",
|
||||
{"Connection": "close"},
|
||||
) from None
|
||||
except ClientDisconnect:
|
||||
# Nobody will read this answer; nothing was stored anywhere.
|
||||
logger.info("upload interrupted by the client after %d bytes", size)
|
||||
raise InvalidRequest("Upload was interrupted.") from None
|
||||
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
async def _acquire_slot() -> None:
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_slot_wait_seconds):
|
||||
await upload_slots.acquire()
|
||||
except TimeoutError:
|
||||
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
|
||||
|
||||
|
||||
async def _discard_after_failed_put(object_key: str) -> None:
|
||||
"""Best effort only; the intent stays for the janitor (late PUT)."""
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
|
||||
|
||||
async def _compensate_failed_commit(object_key: str) -> None:
|
||||
"""Delete the object only if its upload really did not commit."""
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_upload_intent(db, object_key)
|
||||
if row is None:
|
||||
# COMMIT succeeded after all (or the janitor finished it).
|
||||
await db.rollback()
|
||||
return
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
return
|
||||
await repositories.delete_upload_intent(db, object_key)
|
||||
except Exception as exc:
|
||||
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
|
||||
|
||||
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
filename: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
validate_filename(filename)
|
||||
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db, chat_id=chat_id, user_id=current_user["id"]
|
||||
)
|
||||
if chat is None:
|
||||
raise NotFound()
|
||||
# Do not keep a pooled connection idle in a transaction while the
|
||||
# client uploads up to 10 MiB.
|
||||
await db.rollback()
|
||||
|
||||
content_type = declared_content_type(request)
|
||||
check_declared_length(request)
|
||||
|
||||
await _acquire_slot()
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(
|
||||
max_size=settings.upload_spool_memory_bytes
|
||||
) as spool:
|
||||
size, sha256 = await _receive_body(request, spool)
|
||||
if size == 0:
|
||||
raise InvalidRequest("Request body with image bytes is required.")
|
||||
|
||||
try:
|
||||
async with validation_slot:
|
||||
validation = asyncio.create_task(asyncio.to_thread(
|
||||
validate_image, spool, content_type, settings.max_image_pixels
|
||||
))
|
||||
try:
|
||||
info = await asyncio.shield(validation)
|
||||
except asyncio.CancelledError:
|
||||
# A thread cannot be killed: keep its spool and the
|
||||
# validation slot alive until it has really stopped.
|
||||
await asyncio.gather(validation, return_exceptions=True)
|
||||
raise
|
||||
except TooManyPixels:
|
||||
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
|
||||
except InvalidImage as exc:
|
||||
raise UnsupportedMediaType(str(exc)) from None
|
||||
|
||||
return await _store(
|
||||
db,
|
||||
spool=spool,
|
||||
chat_id=chat_id,
|
||||
owner_id=current_user["id"],
|
||||
filename=filename,
|
||||
content_type=info.content_type,
|
||||
size=size,
|
||||
sha256=sha256,
|
||||
width=info.width,
|
||||
height=info.height,
|
||||
)
|
||||
finally:
|
||||
upload_slots.release()
|
||||
|
||||
|
||||
async def _store(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
spool,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> dict[str, Any]:
|
||||
attachment_id = uuid4()
|
||||
# Server-generated random key, unrelated to the filename and to the
|
||||
# public attachment ID.
|
||||
object_key = f"staging/{uuid4().hex}"
|
||||
|
||||
await repositories.create_upload_intent(
|
||||
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
|
||||
)
|
||||
|
||||
try:
|
||||
await storage.put(object_key, spool, size, content_type)
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 PUT failed: %s", exc)
|
||||
_in_background(_discard_after_failed_put(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if settings.fault_injection == "crash_after_s3_put":
|
||||
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
|
||||
os._exit(1)
|
||||
|
||||
try:
|
||||
if settings.fault_injection == "db_error_after_s3_put":
|
||||
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
|
||||
|
||||
row = await repositories.finalize_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=size,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
object_key=object_key,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
|
||||
_in_background(_compensate_failed_commit(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if row is None:
|
||||
# The janitor already claimed this intent (upload slower than the
|
||||
# grace period): the object is (being) deleted, report a failure.
|
||||
logger.warning("upload intent expired before COMMIT")
|
||||
raise Unavailable()
|
||||
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def get_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def open_attachment_content(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
variant: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
|
||||
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
await db.rollback()
|
||||
|
||||
if row["status"] != "ready":
|
||||
raise Conflict("Attachment content is not ready.")
|
||||
|
||||
content = row
|
||||
if variant != "original":
|
||||
content = next((item for item in _variants(row) if item["name"] == variant), None)
|
||||
if content is None:
|
||||
raise NotFound()
|
||||
|
||||
try:
|
||||
chunks, length = await storage.open_download(content["object_key"])
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 GET failed: %s", exc)
|
||||
raise Unavailable() from None
|
||||
except ObjectMissing:
|
||||
# Metadata says "ready" but the bytes are gone: an inconsistency,
|
||||
# not a user error (logged as ERROR).
|
||||
logger.error("object of attachment %s is missing in S3", attachment_id)
|
||||
raise
|
||||
|
||||
if length != content["size_bytes"]:
|
||||
logger.error("object of attachment %s has a wrong size", attachment_id)
|
||||
raise ObjectMissing(content["object_key"])
|
||||
|
||||
headers = {
|
||||
"Content-Type": content["content_type"],
|
||||
"Content-Length": str(length),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"ETag": f'"{content["sha256"]}"',
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
}
|
||||
return chunks, headers
|
||||
|
||||
|
||||
def _in_background(compensation) -> None:
|
||||
"""Do not keep the client waiting for compensation: with S3 down each
|
||||
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
|
||||
anyway."""
|
||||
task = asyncio.create_task(compensation)
|
||||
_compensations.add(task)
|
||||
task.add_done_callback(_compensations.discard)
|
||||
|
||||
|
||||
# --- Janitor ----------------------------------------------------------------
|
||||
|
||||
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
|
||||
"""Delete objects of upload intents older than the grace period.
|
||||
|
||||
One intent per transaction: lock it (SKIP LOCKED), delete the object in
|
||||
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
|
||||
stop: the intent stays and the next run retries.
|
||||
"""
|
||||
cleaned = 0
|
||||
for _ in range(limit):
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_stale_intent(db, grace_seconds)
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
break
|
||||
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(row["object_key"])
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("janitor: S3 unavailable, will retry")
|
||||
break
|
||||
|
||||
await repositories.delete_upload_intent(db, row["object_key"])
|
||||
cleaned += 1
|
||||
logger.warning(
|
||||
"janitor: removed abandoned upload %s (object %s)",
|
||||
row["object_key"],
|
||||
"kept, row committed" if row["committed"] else "deleted",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
async def janitor_loop() -> None:
|
||||
while True:
|
||||
# Jitter: replicas do not all wake up at the same moment.
|
||||
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
|
||||
try:
|
||||
await cleanup_stale_uploads(settings.upload_grace_seconds)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("janitor run failed: %s", type(exc).__name__)
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
"""Valkey session cache.
|
||||
|
||||
PostgreSQL is the source of truth for sessions. Valkey only keeps a short
|
||||
lived copy of "token hash -> user" to avoid a SQL lookup on every request:
|
||||
|
||||
- key: messenger:session:<sha256(token)>, the raw token is never stored;
|
||||
- value TTL: min(SESSION_CACHE_TTL_SECONDS, remaining session lifetime),
|
||||
and expires_at is checked again on every read;
|
||||
- logout writes a "revoked" tombstone (SET ... EX, overwrite), while cache
|
||||
population uses SET ... NX, so a concurrent request cannot resurrect a
|
||||
revoked session in the cache;
|
||||
- any Valkey error or timeout is a cache miss: requests go to PostgreSQL and
|
||||
the cache is skipped for CACHE_RETRY_AFTER_SECONDS. After such an error,
|
||||
and on every API process start, the cache is flushed before its next use,
|
||||
because an invalidation may have been lost while Valkey was unreachable
|
||||
(also by a previous API process).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from redis.asyncio import ConnectionPool, Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.cache")
|
||||
|
||||
KEY_PREFIX = "messenger:session:"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class Revoked:
|
||||
"""Marker returned when the cache knows the session was revoked."""
|
||||
|
||||
|
||||
REVOKED_SESSION = Revoked()
|
||||
|
||||
|
||||
def _key(token_hash: str) -> str:
|
||||
return KEY_PREFIX + token_hash
|
||||
|
||||
|
||||
def _serialize(user: dict[str, Any], expires_at: datetime) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": str(user["id"]),
|
||||
"username": user["username"],
|
||||
"display_name": user["display_name"],
|
||||
"created_at": user["created_at"].isoformat(),
|
||||
"expires_at": expires_at.isoformat(),
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _deserialize(raw: str) -> tuple[dict[str, Any], datetime]:
|
||||
payload = json.loads(raw)
|
||||
user = {
|
||||
"id": UUID(payload["id"]),
|
||||
"username": payload["username"],
|
||||
"display_name": payload["display_name"],
|
||||
"created_at": datetime.fromisoformat(payload["created_at"]),
|
||||
}
|
||||
return user, datetime.fromisoformat(payload["expires_at"])
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str | None) -> None:
|
||||
self._client: Redis | None = None
|
||||
self._unavailable_until = 0.0
|
||||
# A process never trusts a cache it has not flushed itself: an
|
||||
# earlier process may have lost an invalidation (logout while
|
||||
# Valkey was unreachable) and that knowledge died with it.
|
||||
# Cleared only after a successful FLUSHDB.
|
||||
self._needs_flush = True
|
||||
|
||||
if url:
|
||||
pool = ConnectionPool.from_url(
|
||||
url,
|
||||
max_connections=settings.cache_max_connections,
|
||||
socket_timeout=settings.cache_timeout_seconds,
|
||||
socket_connect_timeout=settings.cache_timeout_seconds,
|
||||
decode_responses=True,
|
||||
)
|
||||
self._client = Redis(connection_pool=pool)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._client is not None
|
||||
|
||||
async def _run(self, operation):
|
||||
"""Run one cache operation; return (ok, result)."""
|
||||
if self._client is None:
|
||||
return False, None
|
||||
|
||||
if time.monotonic() < self._unavailable_until:
|
||||
return False, None
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(settings.cache_timeout_seconds * 2):
|
||||
if self._needs_flush:
|
||||
await self._client.flushdb()
|
||||
self._needs_flush = False
|
||||
logger.warning("session cache flushed before use")
|
||||
return True, await operation(self._client)
|
||||
except (RedisError, OSError, TimeoutError) as exc:
|
||||
self._unavailable_until = (
|
||||
time.monotonic() + settings.cache_retry_after_seconds
|
||||
)
|
||||
self._needs_flush = True
|
||||
logger.warning(
|
||||
"session cache unavailable, falling back to PostgreSQL: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False, None
|
||||
|
||||
async def get(
|
||||
self,
|
||||
token_hash: str,
|
||||
) -> dict[str, Any] | Revoked | None:
|
||||
ok, raw = await self._run(lambda client: client.get(_key(token_hash)))
|
||||
|
||||
if not ok or raw is None:
|
||||
return None
|
||||
|
||||
if raw == REVOKED:
|
||||
return REVOKED_SESSION
|
||||
|
||||
try:
|
||||
user, expires_at = _deserialize(raw)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
if expires_at <= datetime.now(timezone.utc):
|
||||
return None
|
||||
|
||||
return user
|
||||
|
||||
async def put(
|
||||
self,
|
||||
token_hash: str,
|
||||
user: dict[str, Any],
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
remaining = (expires_at - datetime.now(timezone.utc)).total_seconds()
|
||||
ttl = min(settings.session_cache_ttl_seconds, math.floor(remaining))
|
||||
|
||||
if ttl < 1:
|
||||
return
|
||||
|
||||
value = _serialize(user, expires_at)
|
||||
await self._run(
|
||||
lambda client: client.set(_key(token_hash), value, ex=ttl, nx=True)
|
||||
)
|
||||
|
||||
async def revoke(self, token_hash: str) -> None:
|
||||
await self._run(
|
||||
lambda client: client.set(
|
||||
_key(token_hash),
|
||||
REVOKED,
|
||||
ex=settings.session_cache_ttl_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
session_cache = SessionCache(settings.cache_url)
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def read_value(name: str, default: str | None = None) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
|
||||
value = os.getenv(name, default)
|
||||
|
||||
if value is None:
|
||||
raise RuntimeError(f"{name} is required.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def read_positive_int(name: str, default: str) -> int:
|
||||
value = int(os.getenv(name, default))
|
||||
|
||||
if value < 1:
|
||||
raise RuntimeError(f"{name} must be positive.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def read_positive_float(name: str, default: str) -> float:
|
||||
value = float(os.getenv(name, default))
|
||||
|
||||
if value <= 0:
|
||||
raise RuntimeError(f"{name} must be positive.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
app_host: str
|
||||
app_port: int
|
||||
database_url: str
|
||||
session_ttl_seconds: int
|
||||
# PostgreSQL connection pool and timeouts.
|
||||
db_pool_size: int
|
||||
db_max_overflow: int
|
||||
db_pool_timeout_seconds: float
|
||||
db_connect_timeout_seconds: float
|
||||
db_statement_timeout_ms: int
|
||||
# Valkey is a session cache; PostgreSQL stays the source of truth.
|
||||
cache_url: str | None
|
||||
session_cache_ttl_seconds: int
|
||||
cache_timeout_seconds: float
|
||||
cache_max_connections: int
|
||||
cache_retry_after_seconds: float
|
||||
# Lab3: X-Instance-Id is the container hostname (short Docker ID), stable
|
||||
# for the process lifetime; scripts/stop-one.sh maps it to a container.
|
||||
instance_id: str
|
||||
idempotency_ttl_seconds: int
|
||||
# Graceful shutdown: readiness answers 503 for shutdown_drain_seconds so
|
||||
# the LB drops this replica, then uvicorn closes the listener and waits
|
||||
# for in-flight requests up to shutdown_timeout_seconds.
|
||||
shutdown_drain_seconds: float
|
||||
shutdown_timeout_seconds: int
|
||||
# Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private
|
||||
# S3 bucket; the API uses its own restricted credentials (not root).
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
s3_object_prefix: str
|
||||
s3_connect_timeout_seconds: float
|
||||
s3_read_timeout_seconds: float
|
||||
# Deadline of one whole S3 call (DNS + retries included), below the LB
|
||||
# "timeout server" (10 s): a stopped S3 container makes Docker DNS hang
|
||||
# ~8 s per lookup, which connect_timeout does not cover.
|
||||
s3_operation_timeout_seconds: float
|
||||
# PUT gets operation timeout + size / this rate: 10 MiB under load is
|
||||
# slower than a HEAD. HAProxy allows 40 s for an upload POST.
|
||||
s3_put_min_bytes_per_second: int
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
# One upload may not take unlimited RAM/time: the body is spooled to a
|
||||
# temporary file (only upload_spool_memory_bytes stay in RAM), receiving
|
||||
# it is limited by upload_timeout_seconds, and at most
|
||||
# max_concurrent_uploads uploads per replica run at once.
|
||||
upload_spool_memory_bytes: int
|
||||
upload_timeout_seconds: float
|
||||
max_concurrent_uploads: int
|
||||
upload_slot_wait_seconds: float
|
||||
# Janitor: an upload intent older than upload_grace_seconds means the
|
||||
# request died between S3 PUT and COMMIT; its object is deleted.
|
||||
upload_grace_seconds: int
|
||||
janitor_interval_seconds: float
|
||||
# Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put
|
||||
fault_injection: str
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
session_store = os.getenv("SESSION_STORE", "postgres").lower()
|
||||
|
||||
if session_store not in {"postgres", "valkey", "redis"}:
|
||||
raise RuntimeError("SESSION_STORE must be postgres, valkey or redis.")
|
||||
|
||||
cache_url = None
|
||||
if session_store != "postgres":
|
||||
cache_url = read_value("CACHE_URL")
|
||||
|
||||
return Settings(
|
||||
app_host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
app_port=int(os.getenv("APP_PORT", "8080")),
|
||||
database_url=read_value("DATABASE_URL"),
|
||||
session_ttl_seconds=read_positive_int("SESSION_TTL_SECONDS", "3600"),
|
||||
db_pool_size=read_positive_int("DB_POOL_SIZE", "10"),
|
||||
db_max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "5")),
|
||||
db_pool_timeout_seconds=read_positive_float("DB_POOL_TIMEOUT_SECONDS", "3"),
|
||||
db_connect_timeout_seconds=read_positive_float(
|
||||
"DB_CONNECT_TIMEOUT_SECONDS", "3"
|
||||
),
|
||||
db_statement_timeout_ms=read_positive_int("DB_STATEMENT_TIMEOUT_MS", "5000"),
|
||||
cache_url=cache_url,
|
||||
session_cache_ttl_seconds=read_positive_int(
|
||||
"SESSION_CACHE_TTL_SECONDS", "60"
|
||||
),
|
||||
cache_timeout_seconds=read_positive_float("CACHE_TIMEOUT_SECONDS", "0.25"),
|
||||
cache_max_connections=read_positive_int("CACHE_MAX_CONNECTIONS", "20"),
|
||||
cache_retry_after_seconds=read_positive_float(
|
||||
"CACHE_RETRY_AFTER_SECONDS", "5"
|
||||
),
|
||||
instance_id=os.getenv("INSTANCE_ID") or socket.gethostname(),
|
||||
idempotency_ttl_seconds=read_positive_int(
|
||||
"IDEMPOTENCY_TTL_SECONDS", "86400"
|
||||
),
|
||||
shutdown_drain_seconds=read_positive_float(
|
||||
"SHUTDOWN_DRAIN_SECONDS", "3"
|
||||
),
|
||||
shutdown_timeout_seconds=read_positive_int(
|
||||
"SHUTDOWN_TIMEOUT_SECONDS", "8"
|
||||
),
|
||||
s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"),
|
||||
s3_region=os.getenv("S3_REGION", "us-east-1"),
|
||||
s3_bucket=read_value("S3_BUCKET", "messenger-attachments"),
|
||||
s3_access_key_id=read_value("S3_ACCESS_KEY_ID"),
|
||||
s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"),
|
||||
s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"),
|
||||
s3_connect_timeout_seconds=read_positive_float(
|
||||
"S3_CONNECT_TIMEOUT_SECONDS", "2"
|
||||
),
|
||||
s3_read_timeout_seconds=read_positive_float(
|
||||
"S3_READ_TIMEOUT_SECONDS", "5"
|
||||
),
|
||||
s3_operation_timeout_seconds=read_positive_float(
|
||||
"S3_OPERATION_TIMEOUT_SECONDS", "6"
|
||||
),
|
||||
s3_put_min_bytes_per_second=read_positive_int(
|
||||
"S3_PUT_MIN_BYTES_PER_SECOND", "1048576"
|
||||
),
|
||||
max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"),
|
||||
max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"),
|
||||
upload_spool_memory_bytes=read_positive_int(
|
||||
"UPLOAD_SPOOL_MEMORY_BYTES", "1048576"
|
||||
),
|
||||
upload_timeout_seconds=read_positive_float(
|
||||
"UPLOAD_TIMEOUT_SECONDS", "60"
|
||||
),
|
||||
max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"),
|
||||
upload_slot_wait_seconds=read_positive_float(
|
||||
"UPLOAD_SLOT_WAIT_SECONDS", "10"
|
||||
),
|
||||
upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"),
|
||||
janitor_interval_seconds=read_positive_float(
|
||||
"JANITOR_INTERVAL_SECONDS", "60"
|
||||
),
|
||||
fault_injection=os.getenv("FAULT_INJECTION", ""),
|
||||
)
|
||||
|
||||
|
||||
settings = load_settings()
|
||||
@@ -0,0 +1,63 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from app.config import settings
|
||||
from app.advanced import ObservedSession, install_gauges
|
||||
from app.telemetry import span
|
||||
|
||||
|
||||
database_url = settings.database_url.replace(
|
||||
"postgresql://",
|
||||
"postgresql+asyncpg://",
|
||||
1,
|
||||
)
|
||||
|
||||
# Bounded pool: at most db_pool_size + db_max_overflow connections per API
|
||||
# process. Waiting for a free connection, connecting and every statement are
|
||||
# limited, so an unavailable database produces a fast 503 instead of a hang.
|
||||
engine = create_async_engine(
|
||||
database_url,
|
||||
pool_pre_ping=True,
|
||||
pool_size=settings.db_pool_size,
|
||||
max_overflow=settings.db_max_overflow,
|
||||
pool_timeout=settings.db_pool_timeout_seconds,
|
||||
pool_recycle=1800,
|
||||
connect_args={
|
||||
"timeout": settings.db_connect_timeout_seconds,
|
||||
"command_timeout": settings.db_statement_timeout_ms / 1000 + 1,
|
||||
"server_settings": {
|
||||
"statement_timeout": str(settings.db_statement_timeout_ms),
|
||||
"idle_in_transaction_session_timeout": "10000",
|
||||
"application_name": "messenger-api",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
SessionLocal = async_sessionmaker(
|
||||
bind=engine,
|
||||
class_=ObservedSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
)
|
||||
|
||||
install_gauges(engine)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with SessionLocal() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def check_database_ready() -> bool:
|
||||
try:
|
||||
with span("db.health", {"db.system": "postgresql"}):
|
||||
async with engine.connect() as connection:
|
||||
await connection.exec_driver_sql("SELECT 1")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
FastAPI dependencies for Lab2:
|
||||
- Bearer authentication;
|
||||
- pagination query parameters.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, Query, Request, Security
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBasic, HTTPBearer
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import get_db
|
||||
from app.errors import Unauthorized
|
||||
from app.services import current_user_from_bearer_token
|
||||
|
||||
|
||||
# Security schemes make the "Authorize" button in /docs send real
|
||||
# Authorization headers. Errors keep the contract format: auto_error=False
|
||||
# and our own 401 responses.
|
||||
bearer_scheme = HTTPBearer(
|
||||
scheme_name="Bearer",
|
||||
auto_error=False,
|
||||
description="Opaque session token from POST /api/v1/auth/sessions.",
|
||||
)
|
||||
|
||||
|
||||
class BasicForDocs(HTTPBasic):
|
||||
"""Documents Basic auth; the header is parsed by app.security."""
|
||||
|
||||
async def __call__(self, request: Request) -> None: # type: ignore[override]
|
||||
return None
|
||||
|
||||
|
||||
basic_scheme = BasicForDocs(
|
||||
scheme_name="Basic",
|
||||
auto_error=False,
|
||||
description="Only for POST /api/v1/auth/sessions.",
|
||||
)
|
||||
|
||||
|
||||
async def require_user(
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Security(
|
||||
bearer_scheme
|
||||
),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
if credentials is None or not credentials.credentials.strip():
|
||||
raise Unauthorized("Bearer")
|
||||
|
||||
return await current_user_from_bearer_token(
|
||||
db,
|
||||
credentials.credentials.strip(),
|
||||
)
|
||||
|
||||
|
||||
def pagination_params(
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
) -> tuple[int, Optional[str]]:
|
||||
return limit, cursor
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
import re
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from uuid import uuid4
|
||||
|
||||
from app.config import settings
|
||||
from app.lifecycle import draining
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
def __init__(self, message: str = "Invalid request.") -> None:
|
||||
super().__init__(400, "invalid_request", message)
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
def __init__(self, scheme: str = "Bearer") -> None:
|
||||
if scheme == "Basic":
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"'
|
||||
else:
|
||||
challenge = 'Bearer realm="messenger"'
|
||||
|
||||
super().__init__(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication is required.",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
def __init__(self, message: str = "Object not found.") -> None:
|
||||
super().__init__(404, "not_found", message)
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
def __init__(self, message: str = "Conflict.") -> None:
|
||||
super().__init__(409, "conflict", message)
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(503, "unavailable", "Service is unavailable.")
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
def __init__(self, message: str = "Payload is too large.") -> None:
|
||||
super().__init__(413, "payload_too_large", message)
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
def __init__(
|
||||
self,
|
||||
message: str = "Only PNG and JPEG images are accepted.",
|
||||
) -> None:
|
||||
super().__init__(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
def __init__(self, message: str, retry_after_seconds: int = 1) -> None:
|
||||
super().__init__(
|
||||
429,
|
||||
"rate_limited",
|
||||
message,
|
||||
{"Retry-After": str(retry_after_seconds)},
|
||||
)
|
||||
|
||||
# X-Request-Id from HAProxy (or the client) is reused only if it is short and
|
||||
# printable; anything else is replaced, so it cannot inject into logs.
|
||||
REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def incoming_request_id(request: Request) -> str:
|
||||
value = request.headers.get("X-Request-Id", "")
|
||||
if REQUEST_ID_PATTERN.fullmatch(value):
|
||||
return value
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
def request_id(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", str(uuid4()))
|
||||
|
||||
|
||||
def common_headers(request: Request) -> dict[str, str]:
|
||||
"""Headers of every API response, including errors (lab3: X-Instance-Id)."""
|
||||
headers = {
|
||||
"X-Request-Id": request_id(request),
|
||||
"X-Instance-Id": settings.instance_id,
|
||||
}
|
||||
if draining.is_set():
|
||||
# Ask HAProxy not to reuse the connection to a replica that is stopping.
|
||||
headers["Connection"] = "close"
|
||||
return headers
|
||||
|
||||
|
||||
async def api_error_handler(
|
||||
request: Request,
|
||||
exc: ApiError,
|
||||
) -> JSONResponse:
|
||||
rid = request_id(request)
|
||||
|
||||
headers = {
|
||||
**common_headers(request),
|
||||
**exc.headers,
|
||||
}
|
||||
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"request_id": rid,
|
||||
}
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Disposable, resource-limited image decoder. No app credentials/config.
|
||||
|
||||
Usage: python -m app.image_worker INPUT OUTPUT_DIR. Result is manifest.json.
|
||||
All input comes from a trusted staging object selected by the worker in PG.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
|
||||
def fit_size(width: int, height: int, limit: int) -> tuple[int, int]:
|
||||
largest = max(width, height)
|
||||
if largest <= limit:
|
||||
return width, height
|
||||
return max(1, width * limit // largest), max(1, height * limit // largest)
|
||||
|
||||
|
||||
def convert(source: Path, directory: Path) -> list[dict]:
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(source) as opened:
|
||||
if opened.format not in {"PNG", "JPEG"}:
|
||||
raise ValueError("invalid_image")
|
||||
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
|
||||
raise ValueError("image_too_large")
|
||||
opened.load()
|
||||
oriented = ImageOps.exif_transpose(opened)
|
||||
# Detach metadata and use fixed output modes. The original file
|
||||
# itself is never rewritten; EXIF orientation affects variants only.
|
||||
image = oriented.convert("RGBA" if "A" in oriented.getbands() or
|
||||
"transparency" in oriented.info else "RGB")
|
||||
oriented.close()
|
||||
image.info.clear()
|
||||
if image.mode == "RGBA":
|
||||
rgb = Image.new("RGB", image.size, "white")
|
||||
rgb.paste(image, mask=image.getchannel("A"))
|
||||
else:
|
||||
rgb = image
|
||||
|
||||
variants = []
|
||||
|
||||
def save(name, result, filename, content_type, **options):
|
||||
path = directory / filename
|
||||
result.info.clear()
|
||||
result.save(path, **options)
|
||||
data = path.read_bytes()
|
||||
variants.append({"name": name, "filename": filename, "content_type": content_type,
|
||||
"width": result.width, "height": result.height,
|
||||
"size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
|
||||
if result is not rgb and result is not image:
|
||||
result.close()
|
||||
|
||||
save("thumbnail", rgb.resize(fit_size(*rgb.size, 256), Image.Resampling.LANCZOS),
|
||||
"thumbnail.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
|
||||
save("crop", ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS,
|
||||
centering=(0.5, 0.5)),
|
||||
"crop.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
|
||||
save("optimized", image.resize(fit_size(*image.size, 1280), Image.Resampling.LANCZOS),
|
||||
"optimized.webp", "image/webp", format="WEBP", quality=80, method=4)
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
image.close()
|
||||
return variants
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Linux container limits are inherited by every native decoder. Parent
|
||||
# enforces a wall timeout and kills/reaps this process on expiration.
|
||||
if sys.platform == "linux":
|
||||
import resource
|
||||
memory = int(os.getenv("WORKER_MEMORY_MB", "768")) * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
cpu = math.ceil(float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))) + 1
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu))
|
||||
try:
|
||||
source, directory = Path(sys.argv[1]), Path(sys.argv[2])
|
||||
manifest = convert(source, directory)
|
||||
(directory / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return 0
|
||||
except MemoryError:
|
||||
return 21
|
||||
except Exception:
|
||||
return 20
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
"""Content-based validation of uploaded PNG/JPEG images.
|
||||
|
||||
The declared Content-Type is never trusted: the format is detected from the
|
||||
first bytes and must match it. Dimensions are read from the header and the
|
||||
pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file
|
||||
cannot make the API allocate gigabytes.
|
||||
|
||||
- PNG is validated by our own streaming reader: critical chunks, CRC of every
|
||||
chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces.
|
||||
Scanline sizes and filter bytes (also in Adam7 passes) are checked without
|
||||
retaining any row. Memory stays O(64 KiB) for any image.
|
||||
- JPEG: Pillow reads the header (lazy open); then the image is decoded with
|
||||
draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded
|
||||
stream (a truncated/corrupted file raises) while allocating only 1/64 of
|
||||
the full-size pixel buffer. Large progressive JPEGs can still retain DCT
|
||||
coefficients; the caller must bound concurrent validation and memory.
|
||||
Their entropy stream is checked as well: there is no header-only bypass.
|
||||
|
||||
The functions are synchronous (CPU work); call them via asyncio.to_thread.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
# Our own limit is applied before Pillow decodes anything.
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIGNATURE = b"\xff\xd8\xff"
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
"""Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415)."""
|
||||
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
"""Decoded image would exceed MAX_IMAGE_PIXELS (-> 413)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def sniff_content_type(head: bytes) -> str | None:
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SIGNATURE):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_image(
|
||||
file: BinaryIO,
|
||||
declared_content_type: str,
|
||||
max_pixels: int,
|
||||
) -> ImageInfo:
|
||||
file.seek(0)
|
||||
detected = sniff_content_type(file.read(len(PNG_SIGNATURE)))
|
||||
file.seek(0)
|
||||
|
||||
if detected is None:
|
||||
raise InvalidImage("Content is not a PNG or JPEG image.")
|
||||
|
||||
if detected != declared_content_type:
|
||||
raise InvalidImage("Content-Type does not match the image content.")
|
||||
|
||||
if detected == "image/png":
|
||||
width, height = _validate_png(file, max_pixels)
|
||||
else:
|
||||
width, height = _validate_jpeg(file, max_pixels)
|
||||
|
||||
return ImageInfo(detected, width, height)
|
||||
|
||||
|
||||
# --- PNG -------------------------------------------------------------------
|
||||
|
||||
# color type -> (channels, allowed bit depths)
|
||||
_PNG_COLOR_TYPES = {
|
||||
0: (1, {1, 2, 4, 8, 16}),
|
||||
2: (3, {8, 16}),
|
||||
3: (1, {1, 2, 4, 8}),
|
||||
4: (2, {8, 16}),
|
||||
6: (4, {8, 16}),
|
||||
}
|
||||
_ADAM7_PASSES = (
|
||||
(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4),
|
||||
(0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
def _read_exact(file: BinaryIO, size: int) -> bytes:
|
||||
data = file.read(size)
|
||||
if len(data) != size:
|
||||
raise InvalidImage("Image file is truncated.")
|
||||
return data
|
||||
|
||||
|
||||
def _png_scanlines(width: int, height: int, bits_per_pixel: int,
|
||||
interlace: int) -> list[tuple[int, int]]:
|
||||
"""At most seven (packed row bytes, row count) pairs; no row allocation."""
|
||||
if interlace == 0:
|
||||
return [((width * bits_per_pixel + 7) // 8, height)]
|
||||
passes = []
|
||||
for x0, y0, dx, dy in _ADAM7_PASSES:
|
||||
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if w and h:
|
||||
passes.append(((w * bits_per_pixel + 7) // 8, h))
|
||||
return passes
|
||||
|
||||
|
||||
class _PngScanlines:
|
||||
"""Validate filter positions across arbitrary decompression boundaries."""
|
||||
def __init__(self, passes: list[tuple[int, int]]) -> None:
|
||||
self.passes = iter(passes)
|
||||
self.row_bytes, self.rows_left = next(self.passes)
|
||||
self.row_remaining = 0
|
||||
self.expected = sum((row_bytes + 1) * count for row_bytes, count in passes)
|
||||
self.produced = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
self.produced += len(data)
|
||||
if self.produced > self.expected:
|
||||
raise InvalidImage("PNG image data is larger than IHDR declares.")
|
||||
position = 0
|
||||
while position < len(data):
|
||||
if self.row_remaining == 0:
|
||||
if self.rows_left == 0:
|
||||
try:
|
||||
self.row_bytes, self.rows_left = next(self.passes)
|
||||
except StopIteration:
|
||||
raise InvalidImage("PNG has too many scanlines.") from None
|
||||
if data[position] > 4:
|
||||
raise InvalidImage("Invalid PNG scanline filter.")
|
||||
position += 1
|
||||
self.rows_left -= 1
|
||||
self.row_remaining = self.row_bytes
|
||||
consumed = min(self.row_remaining, len(data) - position)
|
||||
self.row_remaining -= consumed
|
||||
position += consumed
|
||||
|
||||
|
||||
def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
_read_exact(file, len(PNG_SIGNATURE))
|
||||
|
||||
width = height = 0
|
||||
scanlines = None
|
||||
inflater = zlib.decompressobj()
|
||||
color_type = -1
|
||||
bit_depth = 0
|
||||
seen_plte = False
|
||||
palette_entries = 0
|
||||
seen_trns = False
|
||||
idat_state = "before" # before -> inside -> after
|
||||
first = True
|
||||
|
||||
while True:
|
||||
header = _read_exact(file, 8)
|
||||
length, chunk_type = struct.unpack(">I4s", header)
|
||||
|
||||
if length > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG chunk length.")
|
||||
|
||||
if first and chunk_type != b"IHDR":
|
||||
raise InvalidImage("PNG must start with IHDR.")
|
||||
if not all(65 <= ch <= 90 or 97 <= ch <= 122 for ch in chunk_type) \
|
||||
or not 65 <= chunk_type[2] <= 90:
|
||||
raise InvalidImage("Invalid PNG chunk type.")
|
||||
if chunk_type == b"IHDR" and (not first or length != 13):
|
||||
# Check before reading/buffering: an attacker cannot turn a
|
||||
# bogus multi-megabyte IHDR into a large allocation.
|
||||
raise InvalidImage("Invalid or duplicate PNG IHDR.")
|
||||
if chunk_type[0] < 97 and chunk_type not in {b"IHDR", b"PLTE", b"IDAT", b"IEND"}:
|
||||
raise InvalidImage("Unknown critical PNG chunk.")
|
||||
if chunk_type == b"IEND" and length != 0:
|
||||
raise InvalidImage("Invalid PNG IEND.")
|
||||
if chunk_type == b"PLTE":
|
||||
if seen_plte or seen_trns or idat_state != "before" or color_type in {0, 4} \
|
||||
or not 3 <= length <= 768 or length % 3:
|
||||
raise InvalidImage("Invalid PNG palette.")
|
||||
palette_entries = length // 3
|
||||
if color_type == 3 and palette_entries > 1 << bit_depth:
|
||||
raise InvalidImage("PNG palette exceeds its bit depth.")
|
||||
if chunk_type == b"tRNS":
|
||||
valid_length = ((color_type == 0 and length == 2) or
|
||||
(color_type == 2 and length == 6) or
|
||||
(color_type == 3 and seen_plte and 1 <= length <= palette_entries))
|
||||
if seen_trns or idat_state != "before" or not valid_length:
|
||||
raise InvalidImage("Invalid PNG transparency chunk.")
|
||||
seen_trns = True
|
||||
|
||||
crc = zlib.crc32(chunk_type)
|
||||
is_idat = chunk_type == b"IDAT"
|
||||
|
||||
if is_idat:
|
||||
if idat_state == "after":
|
||||
raise InvalidImage("PNG IDAT chunks are not consecutive.")
|
||||
if color_type == 3 and not seen_plte:
|
||||
raise InvalidImage("PNG palette image without PLTE.")
|
||||
idat_state = "inside"
|
||||
elif idat_state == "inside":
|
||||
idat_state = "after"
|
||||
|
||||
data = b""
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = _read_exact(file, min(CHUNK, remaining))
|
||||
remaining -= len(piece)
|
||||
crc = zlib.crc32(piece, crc)
|
||||
|
||||
if is_idat:
|
||||
_inflate_scanlines(inflater, piece, scanlines)
|
||||
elif chunk_type == b"IHDR":
|
||||
data += piece
|
||||
|
||||
(stored_crc,) = struct.unpack(">I", _read_exact(file, 4))
|
||||
if stored_crc != crc:
|
||||
raise InvalidImage("PNG chunk CRC mismatch.")
|
||||
|
||||
if first:
|
||||
first = False
|
||||
(width, height, bit_depth, color_type,
|
||||
compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data)
|
||||
|
||||
if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Before a single IDAT byte is decompressed.
|
||||
raise TooManyPixels()
|
||||
if color_type not in _PNG_COLOR_TYPES:
|
||||
raise InvalidImage("Invalid PNG color type.")
|
||||
channels, depths = _PNG_COLOR_TYPES[color_type]
|
||||
if bit_depth not in depths or compression != 0 or filter_method != 0 \
|
||||
or interlace not in (0, 1):
|
||||
raise InvalidImage("Invalid PNG IHDR.")
|
||||
scanlines = _PngScanlines(
|
||||
_png_scanlines(width, height, channels * bit_depth, interlace)
|
||||
)
|
||||
elif chunk_type == b"PLTE":
|
||||
seen_plte = True
|
||||
elif chunk_type == b"IEND":
|
||||
break
|
||||
|
||||
if idat_state == "before":
|
||||
raise InvalidImage("PNG has no image data.")
|
||||
|
||||
# decompress(max_length) drains every unconsumed_tail above. No flush()
|
||||
# is needed (and its output would not have a strict allocation bound).
|
||||
if not inflater.eof or scanlines.produced != scanlines.expected:
|
||||
raise InvalidImage("PNG image data is corrupted or truncated.")
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
def _inflate_scanlines(inflater, data: bytes, scanlines: _PngScanlines) -> None:
|
||||
"""Decompress at most 64 KiB at once, validate, then discard."""
|
||||
try:
|
||||
while data:
|
||||
if inflater.eof:
|
||||
raise InvalidImage("PNG has data after the end of the zlib stream.")
|
||||
out = inflater.decompress(data, CHUNK)
|
||||
scanlines.feed(out)
|
||||
if inflater.unused_data:
|
||||
raise InvalidImage("PNG has data after the end of the zlib stream.")
|
||||
data = inflater.unconsumed_tail
|
||||
except zlib.error:
|
||||
raise InvalidImage("PNG image data is corrupted.") from None
|
||||
|
||||
|
||||
# --- JPEG ------------------------------------------------------------------
|
||||
|
||||
def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
try:
|
||||
with Image.open(file, formats=["JPEG"]) as image:
|
||||
width, height = image.size
|
||||
|
||||
if width < 1 or height < 1:
|
||||
raise InvalidImage("Invalid JPEG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Header only: no pixel has been decoded yet.
|
||||
raise TooManyPixels()
|
||||
|
||||
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
image.load()
|
||||
except (InvalidImage, TooManyPixels):
|
||||
raise
|
||||
except Exception:
|
||||
# Pillow raises OSError/SyntaxError/ValueError/... for broken data.
|
||||
raise InvalidImage("JPEG image is corrupted or truncated.") from None
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
"""Transactional outbox relay, expired-lease recovery and staging janitor.
|
||||
|
||||
DB COMMIT and RabbitMQ publish cannot be one transaction. Keep outbox rows
|
||||
until a publisher confirm, accepting harmless duplicate delivery on crash.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import (RETENTION_SECONDS, arm_from_env, broker_channel,
|
||||
crash_if_armed, envelope, publish, retry_or_fail)
|
||||
from app.storage import storage
|
||||
from app.telemetry import configure, emit, install_termination_handler, shutdown
|
||||
from app import advanced
|
||||
|
||||
logger = logging.getLogger("messenger.jobs")
|
||||
|
||||
|
||||
async def recover_leases() -> int:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE status='processing' AND lease_until<now()
|
||||
ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED
|
||||
"""))
|
||||
rows = list(result.mappings())
|
||||
for row in rows:
|
||||
await retry_or_fail(db, row, "lease_expired")
|
||||
await db.commit()
|
||||
if rows:
|
||||
logger.warning("recovered expired leases count=%s", len(rows))
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def cleanup_staging() -> None:
|
||||
# One object transaction at a time: S3's bounded deadline is shorter
|
||||
# than idle_in_transaction_session_timeout. Replay takes the same lock.
|
||||
for _ in range(10):
|
||||
async with SessionLocal() as db:
|
||||
# Failed retention may remove five objects. Keep its row lock
|
||||
# until all bounded deletes finish, so replay cannot race it.
|
||||
await db.execute(text("SET LOCAL idle_in_transaction_session_timeout='60s'"))
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND NOT staging_deleted AND
|
||||
(status='ready' OR (status='failed' AND updated_at <
|
||||
now()-make_interval(secs => :retention)))
|
||||
ORDER BY updated_at LIMIT 1 FOR UPDATE SKIP LOCKED
|
||||
"""), {"retention": RETENTION_SECONDS})
|
||||
row = result.mappings().first()
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
return
|
||||
await storage.delete(row["staging_key"])
|
||||
# Partial final objects of an exhausted job are no longer
|
||||
# useful after retention; ready objects are never deleted.
|
||||
if row["status"] == "failed":
|
||||
prefix = row["object_key"].rsplit("/", 1)[0]
|
||||
for suffix in ("original", "thumbnail.jpg", "crop.jpg", "optimized.webp"):
|
||||
await storage.delete(f"{prefix}/{suffix}")
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET staging_deleted=true WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await db.commit()
|
||||
logger.info("staging cleaned attachment=%s", row["id"])
|
||||
|
||||
|
||||
async def relay_one(channel) -> bool:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT o.*,a.pipeline_version FROM outbox o
|
||||
JOIN attachments a ON a.id=o.attachment_id
|
||||
WHERE o.published_at IS NULL AND o.available_at<=now()
|
||||
ORDER BY o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED
|
||||
"""))
|
||||
row = result.mappings().first()
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
return False
|
||||
# The upload COMMIT already exists; the pending row is untouched
|
||||
# by this crash, so the restarted relay finds it automatically.
|
||||
await crash_if_armed("crash_after_commit_before_publish")
|
||||
await publish(channel, envelope(row))
|
||||
await db.execute(text("UPDATE outbox SET published_at=now() WHERE id=:id"),
|
||||
{"id": row["id"]})
|
||||
await db.commit()
|
||||
logger.info("published job=%s attempt=%s", row["job_id"], row["attempt"])
|
||||
return True
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
configure("messenger-publisher")
|
||||
install_termination_handler()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
for name in ("aio_pika", "aiormq"):
|
||||
logging.getLogger(name).setLevel(logging.CRITICAL)
|
||||
await storage.start()
|
||||
backlog = asyncio.create_task(advanced.backlog_loop(SessionLocal))
|
||||
connection = None
|
||||
try:
|
||||
await arm_from_env("PUBLISHER_FAULT")
|
||||
while True:
|
||||
try:
|
||||
connection, channel, _ = await broker_channel()
|
||||
last_maintenance = 0.0
|
||||
while True:
|
||||
now = asyncio.get_running_loop().time()
|
||||
if now - last_maintenance > 1:
|
||||
await recover_leases()
|
||||
try:
|
||||
await cleanup_staging()
|
||||
except Exception as exc:
|
||||
# Retention is best effort. S3 failure must not
|
||||
# starve outbox delivery or exhaustible retries.
|
||||
logger.warning("staging cleanup deferred: %s", type(exc).__name__)
|
||||
last_maintenance = now
|
||||
if not await relay_one(channel):
|
||||
await asyncio.sleep(0.25)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Exception text can contain connection URLs. Log type only.
|
||||
logger.warning("relay unavailable: %s", type(exc).__name__)
|
||||
if connection:
|
||||
await connection.close()
|
||||
connection = None
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
backlog.cancel()
|
||||
await asyncio.gather(backlog, return_exceptions=True)
|
||||
if connection:
|
||||
await connection.close()
|
||||
await storage.close()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(shutdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except asyncio.CancelledError:
|
||||
pass # SIGTERM has run finally/SDK flush; exit cleanly.
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Process lifecycle state shared by the server and request handlers."""
|
||||
import threading
|
||||
|
||||
# Set on the first SIGTERM/SIGINT. While draining, /health/ready answers 503
|
||||
# and responses carry "Connection: close", but requests are still served.
|
||||
draining = threading.Event()
|
||||
+461
@@ -0,0 +1,461 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import UUID
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from sqlalchemy import exc as sa_exc
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.attachments import (
|
||||
get_attachment,
|
||||
janitor_loop,
|
||||
open_attachment_content,
|
||||
upload_attachment,
|
||||
)
|
||||
from app.cache import session_cache
|
||||
from app.db import check_database_ready, engine, get_db
|
||||
from app.deps import basic_scheme, pagination_params, require_user
|
||||
from app.errors import (
|
||||
ApiError,
|
||||
InvalidRequest,
|
||||
Unavailable,
|
||||
api_error_handler,
|
||||
common_headers,
|
||||
)
|
||||
from app.lifecycle import draining
|
||||
from app.config import settings
|
||||
from app.telemetry import HttpTelemetryMiddleware, configure, shutdown
|
||||
from app.models import (
|
||||
Attachment,
|
||||
Chat,
|
||||
ChatPage,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
Health,
|
||||
Message,
|
||||
MessagePage,
|
||||
Session,
|
||||
User,
|
||||
)
|
||||
from app.storage import storage
|
||||
from app.services import (
|
||||
create_chat,
|
||||
create_message,
|
||||
create_user_session,
|
||||
get_chat,
|
||||
list_chats,
|
||||
list_messages,
|
||||
logout_current_session,
|
||||
register_user,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
configure("messenger-api", settings.instance_id)
|
||||
await storage.start()
|
||||
janitor = asyncio.create_task(janitor_loop())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
janitor.cancel()
|
||||
await asyncio.gather(janitor, return_exceptions=True)
|
||||
await storage.close()
|
||||
await session_cache.close()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(shutdown)
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger — лабораторная 6",
|
||||
version="1.6.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
if not isinstance(exc, ApiError):
|
||||
raise exc
|
||||
|
||||
return await api_error_handler(request, exc)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(
|
||||
request: Request,
|
||||
exc: StarletteHTTPException,
|
||||
) -> JSONResponse:
|
||||
# Unknown routes/methods: the contract error format instead of
|
||||
# FastAPI's default {"detail": ...}.
|
||||
if exc.status_code == 404:
|
||||
error = ApiError(404, "not_found", "Route not found.")
|
||||
else:
|
||||
error = ApiError(exc.status_code, "invalid_request", "Invalid request.")
|
||||
|
||||
return await api_error_handler(request, error)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(
|
||||
request: Request,
|
||||
_exc: RequestValidationError,
|
||||
) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": {
|
||||
"code": "invalid_request",
|
||||
"message": "Invalid request.",
|
||||
"request_id": common_headers(request)["X-Request-Id"],
|
||||
}
|
||||
},
|
||||
headers=common_headers(request),
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.api")
|
||||
|
||||
|
||||
def is_dependency_failure(exc: Exception) -> bool:
|
||||
"""PostgreSQL is unreachable, restarting, overloaded or timed out."""
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
sa_exc.OperationalError,
|
||||
sa_exc.InterfaceError,
|
||||
sa_exc.TimeoutError, # connection pool exhausted
|
||||
OSError,
|
||||
TimeoutError,
|
||||
),
|
||||
):
|
||||
return True
|
||||
|
||||
return isinstance(exc, sa_exc.DBAPIError) and exc.connection_invalidated
|
||||
|
||||
|
||||
async def dependency_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
if not is_dependency_failure(exc):
|
||||
return await unexpected_error_handler(request, exc)
|
||||
|
||||
logger.warning("database unavailable: %s", type(exc).__name__)
|
||||
return await api_error_handler(request, Unavailable())
|
||||
|
||||
|
||||
for _exc_class in (sa_exc.DBAPIError, sa_exc.TimeoutError, OSError, TimeoutError):
|
||||
app.add_exception_handler(_exc_class, dependency_error_handler)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
headers = common_headers(request)
|
||||
# Only the type: SQL parameters and headers may contain secrets.
|
||||
logger.error("unexpected error: %s", type(exc).__name__)
|
||||
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"error": {
|
||||
"code": "internal_error",
|
||||
"message": "Internal server error.",
|
||||
"request_id": headers["X-Request-Id"],
|
||||
}
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/health/live",
|
||||
response_model=Health,
|
||||
)
|
||||
async def liveness() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get(
|
||||
"/health/ready",
|
||||
response_model=Health,
|
||||
)
|
||||
async def readiness() -> dict[str, str]:
|
||||
# A draining replica reports "not ready" so HAProxy stops sending new
|
||||
# requests to it before the listener is closed.
|
||||
# S3 is deliberately NOT checked: without it only attachments degrade
|
||||
# (503), users/chats/messages keep working. If readiness depended on S3,
|
||||
# an S3 outage would take every replica out of the LB.
|
||||
if draining.is_set() or not await check_database_ready():
|
||||
raise Unavailable()
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=User,
|
||||
)
|
||||
async def register_user_route(
|
||||
payload: CreateUser,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await register_user(
|
||||
db,
|
||||
username=payload.username,
|
||||
display_name=payload.display_name,
|
||||
password=payload.password,
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/auth/sessions",
|
||||
status_code=201,
|
||||
response_model=Session,
|
||||
)
|
||||
async def login(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_basic: None = Security(basic_scheme),
|
||||
):
|
||||
# Contract: login takes only "Authorization: Basic", no request body.
|
||||
# Any body (malformed JSON, "{}", ...) is rejected before the password check.
|
||||
if await request.body():
|
||||
raise InvalidRequest("POST /auth/sessions does not accept a request body.")
|
||||
|
||||
return await create_user_session(
|
||||
db,
|
||||
request.headers.get("Authorization"),
|
||||
)
|
||||
|
||||
|
||||
@app.delete(
|
||||
"/api/v1/auth/sessions/current",
|
||||
status_code=204,
|
||||
)
|
||||
async def logout(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_current_user: dict = Depends(require_user),
|
||||
) -> Response:
|
||||
_, _, token = request.headers["Authorization"].partition(" ")
|
||||
|
||||
await logout_current_session(
|
||||
db,
|
||||
token.strip(),
|
||||
)
|
||||
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/users/me",
|
||||
response_model=User,
|
||||
)
|
||||
async def current_user_route(
|
||||
current_user: dict = Depends(require_user),
|
||||
):
|
||||
return current_user
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=Chat,
|
||||
)
|
||||
async def create_chat_route(
|
||||
payload: CreateChat,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await create_chat(
|
||||
db,
|
||||
title=payload.title,
|
||||
member_ids=payload.member_ids,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats",
|
||||
response_model=ChatPage,
|
||||
)
|
||||
async def list_chats_route(
|
||||
page: tuple[int, str | None] = Depends(pagination_params),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
limit, cursor = page
|
||||
|
||||
return await list_chats(
|
||||
db,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}",
|
||||
response_model=Chat,
|
||||
)
|
||||
async def get_chat_route(
|
||||
chat_id: UUID,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=Message,
|
||||
)
|
||||
async def create_message_route(
|
||||
chat_id: UUID,
|
||||
payload: CreateMessage,
|
||||
response: Response,
|
||||
idempotency_key: str | None = Header(
|
||||
default=None,
|
||||
alias="Idempotency-Key",
|
||||
),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
message, replayed = await create_message(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
text_value=payload.text,
|
||||
attachment_ids=payload.attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
if replayed:
|
||||
response.headers["Idempotent-Replayed"] = "true"
|
||||
|
||||
return message
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=MessagePage,
|
||||
)
|
||||
async def list_messages_route(
|
||||
chat_id: UUID,
|
||||
page: tuple[int, str | None] = Depends(pagination_params),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
limit, cursor = page
|
||||
|
||||
return await list_messages(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
UPLOAD_BODY = {
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"description": "Raw image bytes, not multipart and not JSON/base64.",
|
||||
"content": {
|
||||
"image/png": {"schema": {"type": "string", "format": "binary"}},
|
||||
"image/jpeg": {"schema": {"type": "string", "format": "binary"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
status_code=202,
|
||||
response_model=Attachment,
|
||||
openapi_extra=UPLOAD_BODY,
|
||||
)
|
||||
async def upload_attachment_route(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(min_length=1, max_length=255),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await upload_attachment(
|
||||
request,
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
filename=filename,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
response_model=Attachment,
|
||||
)
|
||||
async def get_attachment_route(
|
||||
attachment_id: UUID,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"image/png": {}, "image/jpeg": {}, "image/webp": {}}}},
|
||||
)
|
||||
async def download_attachment_route(
|
||||
attachment_id: UUID,
|
||||
# Known variant names are accepted so that a ready attachment without
|
||||
# that variant answers 404 (contract); any other value is 400.
|
||||
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(
|
||||
default="original"
|
||||
),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
chunks, headers = await open_attachment_content(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
variant=variant,
|
||||
current_user=current_user,
|
||||
)
|
||||
return StreamingResponse(
|
||||
chunks,
|
||||
media_type=headers.pop("Content-Type"),
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
# Wrap outside FastAPI ServerErrorMiddleware to observe its final 500 response.
|
||||
app = HttpTelemetryMiddleware(app)
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class ApiModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(ApiModel):
|
||||
username: str = Field(
|
||||
min_length=3,
|
||||
max_length=32,
|
||||
pattern=r"^[a-z0-9_]{3,32}$",
|
||||
)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("username must not contain only spaces.")
|
||||
return value
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def display_name_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError(
|
||||
"display_name must not contain only spaces."
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def password_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("password must not contain only spaces.")
|
||||
return value
|
||||
|
||||
|
||||
class User(ApiModel):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def user_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class CreateChat(ApiModel):
|
||||
# Contract: 1-100 code points (lab2 allowed 255 by mistake).
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
# Contract: 1-100 members (minItems/maxItems in openapi.json).
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def title_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title must not contain only spaces.")
|
||||
return value
|
||||
|
||||
|
||||
class Chat(ApiModel):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def chat_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class ChatPage(ApiModel):
|
||||
items: list[Chat]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class CreateMessage(ApiModel):
|
||||
# Lab4: text may be "" when the message carries attachments.
|
||||
text: str = Field(max_length=4000)
|
||||
# 0-10 unique IDs of attachments of the same chat; missing == [].
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("attachment_ids must be unique.")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def text_or_attachments(self) -> "CreateMessage":
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError(
|
||||
"text must not be blank when there are no attachments."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class Message(ApiModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID] = Field(default_factory=list)
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def message_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class MessagePage(ApiModel):
|
||||
items: list[Message]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class Session(ApiModel):
|
||||
token: str = Field(min_length=1)
|
||||
token_type: Literal["Bearer"]
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
@field_validator("expires_at")
|
||||
@classmethod
|
||||
def expiry_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("expires_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Variant(ApiModel):
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(ApiModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["queued", "processing", "ready", "failed"]
|
||||
variants: list[Variant]
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@field_validator("created_at", "updated_at")
|
||||
@classmethod
|
||||
def attachment_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("timestamps must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Health(ApiModel):
|
||||
status: Literal["ok"]
|
||||
|
||||
|
||||
class ErrorInner(ApiModel):
|
||||
code: str = Field(min_length=1)
|
||||
message: str = Field(min_length=1)
|
||||
request_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class Error(ApiModel):
|
||||
error: ErrorInner
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Explicit local operator commands; no new public HTTP endpoints."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import event
|
||||
|
||||
POINTS = ("crash_after_commit_before_publish", "crash_after_objects_before_commit",
|
||||
"crash_after_commit_before_ack")
|
||||
|
||||
|
||||
async def run(args) -> None:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
if args.command == "fault":
|
||||
await db.execute(text("""
|
||||
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
|
||||
ON CONFLICT(point) DO UPDATE SET remaining=1,updated_at=now()
|
||||
"""), {"point": args.point})
|
||||
await db.commit()
|
||||
print(f"ARMED one-shot fault: {args.point}")
|
||||
return
|
||||
if args.command == "failed":
|
||||
result = await db.execute(text("""
|
||||
SELECT id,job_id,attempt,attempts_run,error_code,staging_deleted,updated_at
|
||||
FROM attachments WHERE status='failed' ORDER BY updated_at
|
||||
"""))
|
||||
print(json.dumps([dict(row) for row in result.mappings()], default=str, indent=2))
|
||||
return
|
||||
result = await db.execute(text("SELECT * FROM attachments WHERE id=:id FOR UPDATE"),
|
||||
{"id": args.attachment_id})
|
||||
row = result.mappings().first()
|
||||
if row is None or row["job_id"] is None:
|
||||
raise ValueError("Unknown asynchronous attachment")
|
||||
if args.command == "replay":
|
||||
if row["status"] == "ready":
|
||||
print("NO-OP: attachment is already ready")
|
||||
return
|
||||
if row["status"] != "failed":
|
||||
raise ValueError("Replay is allowed only for failed attachments")
|
||||
if row["staging_deleted"]:
|
||||
raise ValueError("Replay source expired; upload the original again")
|
||||
if not args.reason.strip() or len(args.reason) > 200:
|
||||
raise ValueError("Replay reason must have 1-200 characters")
|
||||
result = await db.execute(text("""
|
||||
UPDATE attachments SET status='queued',attempt=attempt+1,attempts_run=0,
|
||||
error_code=NULL,lease_token=NULL,lease_until=NULL,
|
||||
next_attempt_at=now(),updated_at=now() WHERE id=:id RETURNING *
|
||||
"""), {"id": row["id"]})
|
||||
row = result.mappings().one()
|
||||
await event(db, row, "operator_replay", args.reason)
|
||||
else:
|
||||
# Real duplicate envelope: same job ID and current attempt.
|
||||
# Ready effects are checked in PG by the worker and acked.
|
||||
await event(db, row, "operator_redeliver")
|
||||
await db.execute(text("""
|
||||
INSERT INTO outbox(job_id,attachment_id,attempt,available_at,traceparent,tracestate)
|
||||
SELECT job_id,id,attempt,GREATEST(now(),next_attempt_at),traceparent,tracestate
|
||||
FROM attachments WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await db.commit()
|
||||
print(f"QUEUED: {args.command} attachment={row['id']} job={row['job_id']}")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("failed")
|
||||
fault = commands.add_parser("fault")
|
||||
fault.add_argument("point", choices=POINTS)
|
||||
replay = commands.add_parser("replay")
|
||||
replay.add_argument("attachment_id", type=UUID)
|
||||
replay.add_argument("--reason", required=True)
|
||||
redeliver = commands.add_parser("redeliver")
|
||||
redeliver.add_argument("attachment_id", type=UUID)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
asyncio.run(run(args))
|
||||
except ValueError as exc:
|
||||
parser.exit(2, f"REFUSED: {exc}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
"""Shared durable-job plumbing, deliberately separate from the HTTP API."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
import aio_pika
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import read_value
|
||||
from app.db import SessionLocal
|
||||
from app import advanced
|
||||
from app.telemetry import emit, span
|
||||
from app.trace_context import carrier, parent_context
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
logger = logging.getLogger("messenger.pipeline")
|
||||
QUEUE = "attachments"
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120"))
|
||||
IMAGE_TIMEOUT = float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
||||
TOTAL_TIMEOUT = float(os.getenv("JOB_TOTAL_TIMEOUT_SECONDS", "90"))
|
||||
RETENTION_SECONDS = int(os.getenv("STAGING_RETENTION_SECONDS", "604800"))
|
||||
RETRY_BASE_SECONDS = float(os.getenv("RETRY_BASE_SECONDS", "1"))
|
||||
if (MAX_ATTEMPTS < 1 or not math.isfinite(IMAGE_TIMEOUT) or IMAGE_TIMEOUT <= 0
|
||||
or not 0 < TOTAL_TIMEOUT < LEASE_SECONDS):
|
||||
raise RuntimeError("Invalid job limits; total timeout must be below lease.")
|
||||
if not math.isfinite(RETRY_BASE_SECONDS) or RETRY_BASE_SECONDS <= 0:
|
||||
raise RuntimeError("RETRY_BASE_SECONDS must be finite and positive.")
|
||||
|
||||
|
||||
def retry_delay(attempts_run: int) -> float:
|
||||
# Equal jitter: never zero, capped exponential delay, one durable due date.
|
||||
upper = min(60.0, RETRY_BASE_SECONDS * 2 ** min(max(attempts_run - 1, 0), 10))
|
||||
return random.uniform(upper / 2, upper)
|
||||
|
||||
|
||||
async def broker_channel():
|
||||
connection = await aio_pika.connect_robust(read_value("BROKER_URL"), timeout=5)
|
||||
try:
|
||||
channel = await connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await channel.set_qos(prefetch_count=1)
|
||||
dead = await channel.declare_exchange("attachments.dead", aio_pika.ExchangeType.FANOUT,
|
||||
durable=True)
|
||||
dlq = await channel.declare_queue("attachments.dlq", durable=True)
|
||||
await dlq.bind(dead)
|
||||
queue = await channel.declare_queue(
|
||||
QUEUE, durable=True, arguments={"x-dead-letter-exchange": "attachments.dead"}
|
||||
)
|
||||
return connection, channel, queue
|
||||
except BaseException:
|
||||
await connection.close()
|
||||
raise
|
||||
|
||||
|
||||
def envelope(row) -> dict:
|
||||
job = {
|
||||
"schema_version": 1, "job_id": str(row["job_id"]),
|
||||
"type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": int(row.get("pipeline_version", 1)),
|
||||
"attempt": int(row["attempt"]),
|
||||
"created_at": row["created_at"].isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
job.update({name: row[name] for name in ("traceparent", "tracestate") if row.get(name)})
|
||||
return job
|
||||
|
||||
|
||||
def parse_envelope(raw: bytes) -> dict:
|
||||
job = json.loads(raw)
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at"}
|
||||
if not isinstance(job, dict) or not required <= job.keys() or (
|
||||
job.keys() - required - {"traceparent", "tracestate"}
|
||||
):
|
||||
raise ValueError("invalid job fields")
|
||||
if type(job["schema_version"]) is not int or job["schema_version"] != 1 or job["type"] != "attachment.process.v1":
|
||||
raise ValueError("unsupported job schema")
|
||||
if type(job["attempt"]) is not int or job["attempt"] < 1:
|
||||
raise ValueError("invalid attempt")
|
||||
if type(job["pipeline_version"]) is not int or job["pipeline_version"] != 1:
|
||||
raise ValueError("unsupported pipeline")
|
||||
UUID(job["job_id"])
|
||||
UUID(job["attachment_id"])
|
||||
if datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).tzinfo is None:
|
||||
raise ValueError("job timestamp must include timezone")
|
||||
if "traceparent" in job and not re.fullmatch(
|
||||
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", job["traceparent"]
|
||||
):
|
||||
raise ValueError("invalid traceparent")
|
||||
if "tracestate" in job and (not isinstance(job["tracestate"], str) or
|
||||
len(job["tracestate"]) > 512):
|
||||
raise ValueError("invalid tracestate")
|
||||
return job
|
||||
|
||||
|
||||
async def publish(channel, job: dict) -> None:
|
||||
with span("attachments publish", kind=SpanKind.PRODUCER, parent=parent_context(job), attributes={
|
||||
"messaging.system": "rabbitmq", "messaging.destination.name": QUEUE,
|
||||
"messaging.operation": "publish", "job_id": job["job_id"], "attempt": job["attempt"],
|
||||
}):
|
||||
# The confirmed envelope names this producer span as the consumer's
|
||||
# parent. Durable outbox remains unchanged if confirmation is lost.
|
||||
outgoing = {**job, **carrier()}
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(json.dumps(outgoing).encode("utf-8"), content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=job["job_id"]),
|
||||
routing_key=QUEUE, mandatory=True, timeout=5,
|
||||
)
|
||||
emit("job.published", job_id=job["job_id"], attempt=job["attempt"], queue=QUEUE)
|
||||
|
||||
|
||||
async def event(db, row, name: str, detail: str = "") -> None:
|
||||
await db.execute(text("""
|
||||
INSERT INTO job_events(attachment_id,event,attempt,detail)
|
||||
VALUES (:id,:event,:attempt,:detail)
|
||||
"""), {"id": row["id"], "attempt": row["attempt"], "event": name, "detail": detail})
|
||||
|
||||
|
||||
async def arm_from_env(variable: str) -> None:
|
||||
point = os.getenv(variable, "")
|
||||
if point:
|
||||
async with SessionLocal() as db:
|
||||
await db.execute(text("""
|
||||
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
|
||||
ON CONFLICT (point) DO NOTHING
|
||||
"""), {"point": point})
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def crash_if_armed(point: str) -> None:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
UPDATE fault_injections SET remaining=remaining-1,hits=hits+1,updated_at=now()
|
||||
WHERE point=:point AND remaining>0 RETURNING point
|
||||
"""), {"point": point})
|
||||
hit = result.first() is not None
|
||||
await db.commit()
|
||||
if hit:
|
||||
logger.error("FAULT: %s (one-shot durable hit)", point)
|
||||
os._exit(70)
|
||||
|
||||
|
||||
async def retry_or_fail(db, row, code: str, *, permanent: bool = False) -> None:
|
||||
"""Caller owns the attachment row lock. Commit before acking old delivery."""
|
||||
if permanent or row["attempts_run"] >= MAX_ATTEMPTS:
|
||||
final_code = code if permanent else "retry_exhausted"
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='failed',error_code=:code,
|
||||
lease_token=NULL,lease_until=NULL,updated_at=now()
|
||||
WHERE id=:id
|
||||
"""), {"id": row["id"], "code": final_code})
|
||||
await event(db, row, "failed", final_code)
|
||||
logger.warning("failed attachment=%s code=%s", row["id"], final_code)
|
||||
return
|
||||
delay = retry_delay(row["attempts_run"])
|
||||
# A consumer retry continues from its active span. Lease recovery may have
|
||||
# no active span, so retain the context durably recorded by the old claim.
|
||||
context = carrier() or {name: row.get(name) for name in ("traceparent", "tracestate")}
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='queued',error_code=NULL,attempt=attempt+1,
|
||||
lease_token=NULL,lease_until=NULL,
|
||||
traceparent=:traceparent,tracestate=:tracestate,
|
||||
next_attempt_at=now()+make_interval(secs => CAST(:delay AS double precision)),
|
||||
updated_at=now() WHERE id=:id
|
||||
"""), {"id": row["id"], "delay": delay,
|
||||
"traceparent": context.get("traceparent"), "tracestate": context.get("tracestate")})
|
||||
await db.execute(text("""
|
||||
INSERT INTO outbox(job_id,attachment_id,attempt,available_at,traceparent,tracestate)
|
||||
SELECT job_id,id,attempt,next_attempt_at,traceparent,tracestate FROM attachments WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await event(db, row, "retry", f"{code}; delay={delay:.3f}s")
|
||||
logger.info("retry attachment=%s after=%.3fs", row["id"], delay)
|
||||
advanced.retries.add(1, {"job_type": "attachment.process.v1"})
|
||||
emit("job.retry", job_id=str(row["job_id"]), attachment_id=str(row["id"]),
|
||||
attempt=row["attempt"] + 1, outcome="retry", queue=QUEUE)
|
||||
@@ -0,0 +1,726 @@
|
||||
"""PostgreSQL repository functions for MTUSI Messenger Lab4."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.trace_context import carrier
|
||||
|
||||
|
||||
async def create_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
username: str,
|
||||
display_name: str,
|
||||
password_hash: str,
|
||||
) -> dict[str, Any] | None:
|
||||
user_id = uuid4()
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (:id, :username, :display_name, :password_hash)
|
||||
RETURNING id, username, display_name, created_at
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": user_id,
|
||||
"username": username,
|
||||
"display_name": display_name,
|
||||
"password_hash": password_hash,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
return dict(result.mappings().one())
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
return None
|
||||
|
||||
|
||||
async def get_user_by_username(
|
||||
db: AsyncSession,
|
||||
username: str,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = :username
|
||||
"""
|
||||
),
|
||||
{"username": username},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def get_user_by_id(
|
||||
db: AsyncSession,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, username, display_name, created_at
|
||||
FROM users
|
||||
WHERE id = :user_id
|
||||
"""
|
||||
),
|
||||
{"user_id": user_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def user_ids_that_exist(
|
||||
db: AsyncSession,
|
||||
member_ids: Iterable[UUID],
|
||||
) -> set[UUID]:
|
||||
values = list(member_ids)
|
||||
if not values:
|
||||
return set()
|
||||
|
||||
result = await db.execute(
|
||||
text("SELECT id FROM users WHERE id = ANY(CAST(:member_ids AS uuid[]))"),
|
||||
{"member_ids": values},
|
||||
)
|
||||
return {row.id for row in result}
|
||||
|
||||
|
||||
async def create_session(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: UUID,
|
||||
token_hash: str,
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO sessions (id, user_id, token_hash, expires_at)
|
||||
VALUES (:id, :user_id, :token_hash, :expires_at)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": uuid4(),
|
||||
"user_id": user_id,
|
||||
"token_hash": token_hash,
|
||||
"expires_at": expires_at,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_active_session_user(
|
||||
db: AsyncSession,
|
||||
token_hash: str,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT
|
||||
s.id AS session_id,
|
||||
s.token_hash,
|
||||
s.expires_at,
|
||||
u.id,
|
||||
u.username,
|
||||
u.display_name,
|
||||
u.created_at
|
||||
FROM sessions AS s
|
||||
JOIN users AS u ON u.id = s.user_id
|
||||
WHERE s.token_hash = :token_hash
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > now()
|
||||
"""
|
||||
),
|
||||
{"token_hash": token_hash},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def revoke_active_session(
|
||||
db: AsyncSession,
|
||||
token_hash: str,
|
||||
) -> bool:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET revoked_at = now()
|
||||
WHERE token_hash = :token_hash
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > now()
|
||||
"""
|
||||
),
|
||||
{"token_hash": token_hash},
|
||||
)
|
||||
await db.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def create_chat_with_members(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
title: str,
|
||||
member_ids: Iterable[UUID],
|
||||
) -> dict[str, Any]:
|
||||
chat_id = uuid4()
|
||||
members = list(member_ids)
|
||||
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO chats (id, title)
|
||||
VALUES (:id, :title)
|
||||
RETURNING id, title, created_at
|
||||
"""
|
||||
),
|
||||
{"id": chat_id, "title": title},
|
||||
)
|
||||
chat = dict(result.mappings().one())
|
||||
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO chat_members (chat_id, user_id)
|
||||
VALUES (:chat_id, :user_id)
|
||||
"""
|
||||
),
|
||||
[{"chat_id": chat_id, "user_id": user_id} for user_id in members],
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
chat["member_ids"] = members
|
||||
return chat
|
||||
|
||||
|
||||
async def get_accessible_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats AS c
|
||||
JOIN chat_members AS own_membership
|
||||
ON own_membership.chat_id = c.id
|
||||
AND own_membership.user_id = :user_id
|
||||
WHERE c.id = :chat_id
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id, "user_id": user_id},
|
||||
)
|
||||
chat_row = result.mappings().first()
|
||||
if chat_row is None:
|
||||
return None
|
||||
|
||||
members_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = :chat_id
|
||||
ORDER BY user_id ASC
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id},
|
||||
)
|
||||
chat = dict(chat_row)
|
||||
chat["member_ids"] = [row.user_id for row in members_result]
|
||||
return chat
|
||||
|
||||
|
||||
async def list_chats_for_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: UUID,
|
||||
limit: int,
|
||||
cursor_created_at: datetime | None,
|
||||
cursor_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = """
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats AS c
|
||||
JOIN chat_members AS own_membership
|
||||
ON own_membership.chat_id = c.id
|
||||
AND own_membership.user_id = :user_id \
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"user_id": user_id,
|
||||
"limit_plus_one": limit + 1,
|
||||
}
|
||||
|
||||
if cursor_created_at is not None and cursor_id is not None:
|
||||
query += """
|
||||
WHERE (c.created_at, c.id) > (:cursor_created_at, :cursor_id)
|
||||
"""
|
||||
params["cursor_created_at"] = cursor_created_at
|
||||
params["cursor_id"] = cursor_id
|
||||
|
||||
query += " ORDER BY c.created_at ASC, c.id ASC LIMIT :limit_plus_one"
|
||||
|
||||
result = await db.execute(text(query), params)
|
||||
chats = [dict(row) for row in result.mappings().all()]
|
||||
if not chats:
|
||||
return []
|
||||
|
||||
chat_ids = [chat["id"] for chat in chats]
|
||||
members_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT chat_id, user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = ANY(CAST(:chat_ids AS uuid[]))
|
||||
ORDER BY chat_id ASC, user_id ASC
|
||||
"""
|
||||
),
|
||||
{"chat_ids": chat_ids},
|
||||
)
|
||||
|
||||
members_by_chat: dict[UUID, list[UUID]] = {chat_id: [] for chat_id in chat_ids}
|
||||
for row in members_result:
|
||||
members_by_chat[row.chat_id].append(row.user_id)
|
||||
|
||||
for chat in chats:
|
||||
chat["member_ids"] = members_by_chat[chat["id"]]
|
||||
|
||||
return chats
|
||||
|
||||
|
||||
# Attachment IDs of a message in the order of the request.
|
||||
MESSAGE_ATTACHMENT_IDS = """
|
||||
COALESCE(
|
||||
(SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments AS ma
|
||||
WHERE ma.message_id = m.id),
|
||||
CAST('{}' AS uuid[])
|
||||
) AS attachment_ids
|
||||
"""
|
||||
|
||||
|
||||
class AttachmentsNotFound(Exception):
|
||||
"""Some attachment_ids do not exist or belong to another chat (-> 404)."""
|
||||
|
||||
|
||||
class AttachmentFailed(Exception):
|
||||
"""A failed attachment cannot be added to a new message (-> 409)."""
|
||||
|
||||
|
||||
async def create_message_idempotent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str,
|
||||
request_hash: str,
|
||||
ttl_seconds: int,
|
||||
) -> tuple[dict[str, Any], bool] | None:
|
||||
"""Insert a message at most once per idempotency key; one transaction.
|
||||
|
||||
1. INSERT the key row. Its primary key (user, chat, method, key) lives in
|
||||
the shared PostgreSQL, so a concurrent request with the same key on
|
||||
ANY replica blocks on this INSERT until the first transaction ends.
|
||||
2. If the key was claimed: check that every attachment belongs to this
|
||||
chat, INSERT the message and its attachment links, link the message
|
||||
to the key and COMMIT everything together. A failure before COMMIT
|
||||
(also AttachmentsNotFound) rolls back all of it, so an error is never
|
||||
stored as the final answer and a retry with the same key runs again.
|
||||
3. If the key already exists (committed by someone else): return the
|
||||
stored message, or None when the body differs (-> 409).
|
||||
|
||||
A key older than ttl_seconds is reclaimed by the ON CONFLICT ... WHERE.
|
||||
Returns (message, replayed) or None on a body mismatch.
|
||||
"""
|
||||
key_params = {
|
||||
"user_id": sender_id,
|
||||
"chat_id": chat_id,
|
||||
"key": idempotency_key,
|
||||
}
|
||||
|
||||
try:
|
||||
claimed = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO idempotency_keys AS k
|
||||
(user_id, chat_id, method, key, request_hash)
|
||||
VALUES (:user_id, :chat_id, 'POST', :key, :request_hash)
|
||||
ON CONFLICT (user_id, chat_id, method, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash,
|
||||
message_id = NULL,
|
||||
created_at = now()
|
||||
WHERE k.created_at < now() - make_interval(secs => :ttl)
|
||||
RETURNING 1
|
||||
"""
|
||||
),
|
||||
{**key_params, "request_hash": request_hash, "ttl": ttl_seconds},
|
||||
)
|
||||
|
||||
if claimed.first() is not None:
|
||||
if attachment_ids:
|
||||
# Unknown and foreign IDs look the same (404): the response
|
||||
# does not reveal that an attachment of another chat exists.
|
||||
found = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, status FROM attachments
|
||||
WHERE chat_id = :chat_id
|
||||
AND id = ANY(CAST(:ids AS uuid[]))
|
||||
FOR SHARE
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id, "ids": attachment_ids},
|
||||
)
|
||||
attachments = list(found)
|
||||
if {row.id for row in attachments} != set(attachment_ids):
|
||||
raise AttachmentsNotFound()
|
||||
if any(row.status == "failed" for row in attachments):
|
||||
raise AttachmentFailed()
|
||||
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES (:id, :chat_id, :sender_id, :text)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": uuid4(),
|
||||
"chat_id": chat_id,
|
||||
"sender_id": sender_id,
|
||||
"text": text_value,
|
||||
},
|
||||
)
|
||||
message = dict(result.mappings().one())
|
||||
|
||||
if attachment_ids:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO message_attachments
|
||||
(message_id, attachment_id, position)
|
||||
VALUES (:message_id, :attachment_id, :position)
|
||||
"""
|
||||
),
|
||||
[
|
||||
{
|
||||
"message_id": message["id"],
|
||||
"attachment_id": attachment_id,
|
||||
"position": position,
|
||||
}
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
],
|
||||
)
|
||||
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE idempotency_keys
|
||||
SET message_id = :message_id
|
||||
WHERE user_id = :user_id
|
||||
AND chat_id = :chat_id
|
||||
AND method = 'POST'
|
||||
AND key = :key
|
||||
"""
|
||||
),
|
||||
{**key_params, "message_id": message["id"]},
|
||||
)
|
||||
await db.commit()
|
||||
message["attachment_ids"] = list(attachment_ids)
|
||||
return message, False
|
||||
|
||||
stored = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT k.request_hash,
|
||||
m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM idempotency_keys AS k
|
||||
JOIN messages AS m ON m.id = k.message_id
|
||||
WHERE k.user_id = :user_id
|
||||
AND k.chat_id = :chat_id
|
||||
AND k.method = 'POST'
|
||||
AND k.key = :key
|
||||
"""
|
||||
),
|
||||
key_params,
|
||||
)
|
||||
row = stored.mappings().first()
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
if row is None:
|
||||
# Not reachable while the key row and the message commit together.
|
||||
raise RuntimeError("idempotency key without a message")
|
||||
|
||||
message = dict(row)
|
||||
if message.pop("request_hash") != request_hash:
|
||||
return None
|
||||
|
||||
return message, True
|
||||
|
||||
|
||||
async def list_messages_keyset(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
limit: int,
|
||||
cursor_created_at: datetime | None,
|
||||
cursor_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = f"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM messages AS m
|
||||
WHERE m.chat_id = :chat_id
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"chat_id": chat_id,
|
||||
"limit_plus_one": limit + 1,
|
||||
}
|
||||
|
||||
if cursor_created_at is not None and cursor_id is not None:
|
||||
query += """
|
||||
AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id)
|
||||
"""
|
||||
params["cursor_created_at"] = cursor_created_at
|
||||
params["cursor_id"] = cursor_id
|
||||
|
||||
query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one"
|
||||
result = await db.execute(text(query), params)
|
||||
return [dict(row) for row in result.mappings().all()]
|
||||
|
||||
|
||||
# --- Attachments ------------------------------------------------------------
|
||||
|
||||
ATTACHMENT_COLUMNS = """
|
||||
a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
|
||||
a.sha256, a.status, a.error_code, a.object_key, a.variants, a.created_at, a.updated_at
|
||||
"""
|
||||
|
||||
|
||||
async def create_upload_intent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
object_key: str,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
) -> None:
|
||||
"""Committed BEFORE the S3 PUT: from now on the object can be found and
|
||||
cleaned up even if this process dies right after the PUT."""
|
||||
try:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO upload_intents (object_key, attachment_id, chat_id)
|
||||
VALUES (:object_key, :attachment_id, :chat_id)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"object_key": object_key,
|
||||
"attachment_id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def finalize_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Delete the intent, insert queued metadata + outbox in ONE transaction.
|
||||
|
||||
Returns None if the intent is already gone: the janitor decided this
|
||||
upload was abandoned and deleted the object, so the row must not be
|
||||
created. The DELETE waits on the janitor's row lock, so the two can
|
||||
never both succeed.
|
||||
"""
|
||||
try:
|
||||
claimed = await db.execute(
|
||||
text(
|
||||
"DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1"
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
if claimed.first() is None:
|
||||
await db.rollback()
|
||||
return None
|
||||
|
||||
job_id = uuid4()
|
||||
final_key = f"originals/{attachment_id}/v1/original"
|
||||
context = carrier()
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
INSERT INTO attachments AS a
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, width, height, object_key, staging_key, job_id, status,
|
||||
traceparent, tracestate)
|
||||
VALUES (:id, :chat_id, :owner_id, :filename, :content_type,
|
||||
:size_bytes, :sha256, :width, :height, :final_key, :object_key, :job_id, 'queued',
|
||||
:traceparent, :tracestate)
|
||||
RETURNING {ATTACHMENT_COLUMNS}
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": owner_id,
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size_bytes": size_bytes,
|
||||
"sha256": sha256,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": object_key,
|
||||
"final_key": final_key,
|
||||
"job_id": job_id,
|
||||
"traceparent": context.get("traceparent"),
|
||||
"tracestate": context.get("tracestate"),
|
||||
},
|
||||
)
|
||||
row = dict(result.mappings().one())
|
||||
await db.execute(
|
||||
text("""INSERT INTO outbox (job_id, attachment_id, attempt, traceparent, tracestate)
|
||||
VALUES (:job_id, :id, 1, :traceparent, :tracestate)"""),
|
||||
{"job_id": job_id, "id": attachment_id,
|
||||
"traceparent": context.get("traceparent"), "tracestate": context.get("tracestate")},
|
||||
)
|
||||
await db.commit()
|
||||
return row
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def delete_upload_intent(db: AsyncSession, object_key: str) -> None:
|
||||
try:
|
||||
await db.execute(
|
||||
text("DELETE FROM upload_intents WHERE object_key = :object_key"),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def get_accessible_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Only members of the attachment's chat see it; others get None (404)."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT {ATTACHMENT_COLUMNS}
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm
|
||||
ON cm.chat_id = a.chat_id
|
||||
AND cm.user_id = :user_id
|
||||
WHERE a.id = :attachment_id
|
||||
"""
|
||||
),
|
||||
{"attachment_id": attachment_id, "user_id": user_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_upload_intent(
|
||||
db: AsyncSession,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock the intent of one upload (waits for a finishing transaction);
|
||||
the transaction stays open. None: no intent, nothing to compensate."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.object_key = :object_key
|
||||
FOR UPDATE OF i
|
||||
"""
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_stale_intent(
|
||||
db: AsyncSession,
|
||||
grace_seconds: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock ONE intent older than the grace period; the transaction stays open.
|
||||
|
||||
SKIP LOCKED: an upload that is finishing right now holds the row lock in
|
||||
finalize_attachment and is skipped; several replicas can run the janitor
|
||||
at once without touching the same intent.
|
||||
"""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.created_at < now() - make_interval(secs => :grace)
|
||||
ORDER BY i.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF i SKIP LOCKED
|
||||
"""
|
||||
),
|
||||
{"grace": grace_seconds},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def attachment_keys(db: AsyncSession) -> set[str]:
|
||||
result = await db.execute(text("SELECT object_key FROM attachments"))
|
||||
return {row.object_key for row in result}
|
||||
|
||||
|
||||
async def intent_keys(db: AsyncSession) -> dict[str, datetime]:
|
||||
result = await db.execute(
|
||||
text("SELECT object_key, created_at FROM upload_intents")
|
||||
)
|
||||
return {row.object_key: row.created_at for row in result}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return password_hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return password_hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def generate_session_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_session_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def parse_basic_authorization(value: str | None) -> tuple[str, str] | None:
|
||||
if value is None or not value.startswith("Basic "):
|
||||
return None
|
||||
|
||||
try:
|
||||
encoded_credentials = value[len("Basic "):]
|
||||
decoded = base64.b64decode(
|
||||
encoded_credentials,
|
||||
validate=True,
|
||||
).decode("utf-8")
|
||||
|
||||
username, password = decoded.split(":", 1)
|
||||
except (
|
||||
ValueError,
|
||||
UnicodeDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
return None
|
||||
|
||||
if not username or not username.isascii():
|
||||
return None
|
||||
|
||||
return username, password
|
||||
|
||||
|
||||
def _encode_cursor_payload(payload: dict[str, str]) -> str:
|
||||
raw = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii")
|
||||
|
||||
|
||||
def _decode_cursor_payload(cursor: str) -> dict[str, str] | None:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
return payload
|
||||
except (
|
||||
ValueError,
|
||||
TypeError,
|
||||
UnicodeDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
return None
|
||||
|
||||
|
||||
def _utc_timestamp(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("Cursor datetime must be timezone-aware.")
|
||||
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode_chat_cursor(
|
||||
created_at: datetime,
|
||||
chat_id: UUID,
|
||||
) -> str:
|
||||
return _encode_cursor_payload(
|
||||
{
|
||||
"kind": "chat",
|
||||
"created_at": _utc_timestamp(created_at),
|
||||
"id": str(chat_id),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def decode_chat_cursor(
|
||||
cursor: str,
|
||||
) -> tuple[datetime, UUID] | None:
|
||||
payload = _decode_cursor_payload(cursor)
|
||||
|
||||
if payload is None or payload.get("kind") != "chat":
|
||||
return None
|
||||
|
||||
try:
|
||||
created_at = datetime.fromisoformat(
|
||||
payload["created_at"].replace("Z", "+00:00")
|
||||
)
|
||||
chat_id = UUID(payload["id"])
|
||||
except (KeyError, ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
return created_at, chat_id
|
||||
|
||||
|
||||
def encode_message_cursor(
|
||||
chat_id: UUID,
|
||||
created_at: datetime,
|
||||
message_id: UUID,
|
||||
) -> str:
|
||||
return _encode_cursor_payload(
|
||||
{
|
||||
"kind": "message",
|
||||
"chat_id": str(chat_id),
|
||||
"created_at": _utc_timestamp(created_at),
|
||||
"id": str(message_id),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def decode_message_cursor(
|
||||
cursor: str,
|
||||
) -> tuple[UUID, datetime, UUID] | None:
|
||||
payload = _decode_cursor_payload(cursor)
|
||||
|
||||
if payload is None or payload.get("kind") != "message":
|
||||
return None
|
||||
|
||||
try:
|
||||
chat_id = UUID(payload["chat_id"])
|
||||
created_at = datetime.fromisoformat(
|
||||
payload["created_at"].replace("Z", "+00:00")
|
||||
)
|
||||
message_id = UUID(payload["id"])
|
||||
except (KeyError, ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
return chat_id, created_at, message_id
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Entry point: uvicorn with a drain phase before the graceful shutdown.
|
||||
|
||||
Plain uvicorn stops accepting connections immediately on SIGTERM. Behind
|
||||
HAProxy that turns every stop into a burst of refused connections until the
|
||||
active health check notices. Instead, on the first signal:
|
||||
|
||||
t = 0 readiness -> 503, keep serving (Connection: close);
|
||||
t = drain (3 s) HAProxy has marked the server DOWN (inter 1 s x fall 2),
|
||||
uvicorn closes the listener and waits for in-flight
|
||||
requests up to SHUTDOWN_TIMEOUT_SECONDS (8 s).
|
||||
|
||||
stop_grace_period in compose.yaml (15 s) is larger than 3 + 8 s.
|
||||
A second signal skips the drain phase.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from app.config import settings
|
||||
from app.lifecycle import draining
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.server")
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
def handle_exit(self, sig, frame) -> None:
|
||||
if draining.is_set():
|
||||
super().handle_exit(sig, frame)
|
||||
return
|
||||
|
||||
draining.set()
|
||||
logger.warning(
|
||||
"signal %s: draining for %.1f s before shutdown",
|
||||
sig,
|
||||
settings.shutdown_drain_seconds,
|
||||
)
|
||||
# Set should_exit directly instead of super().handle_exit(): uvicorn
|
||||
# would re-raise the captured SIGTERM after shutdown and the process
|
||||
# would exit with 143 instead of a clean 0.
|
||||
timer = threading.Timer(
|
||||
settings.shutdown_drain_seconds,
|
||||
self._begin_shutdown,
|
||||
)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
def _begin_shutdown(self) -> None:
|
||||
logger.warning("drain finished: closing listener")
|
||||
self.should_exit = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
config = uvicorn.Config(
|
||||
"app.main:app",
|
||||
host=settings.app_host,
|
||||
port=settings.app_port,
|
||||
proxy_headers=False,
|
||||
# Longer than HAProxy "timeout http-keep-alive 10s": the LB closes an
|
||||
# idle connection first, so it never reuses one the API just closed.
|
||||
timeout_keep_alive=30,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout_seconds,
|
||||
log_level="info",
|
||||
access_log=False, # raw URL can contain filename; safe events are exported instead
|
||||
)
|
||||
DrainingServer(config).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
"""Business rules for MTUSI Messenger Lab4 (users, chats, messages)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import repositories
|
||||
from app.cache import REVOKED_SESSION, session_cache
|
||||
from app.config import settings
|
||||
from app.errors import ApiError
|
||||
from app.security import (
|
||||
decode_chat_cursor,
|
||||
decode_message_cursor,
|
||||
encode_chat_cursor,
|
||||
encode_message_cursor,
|
||||
generate_session_token,
|
||||
hash_password,
|
||||
hash_session_token,
|
||||
parse_basic_authorization,
|
||||
verify_password,
|
||||
)
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _to_user(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
|
||||
def _to_chat(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"title": row["title"],
|
||||
"member_ids": row["member_ids"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
|
||||
def _to_message(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"sender_id": row["sender_id"],
|
||||
"text": row["text"],
|
||||
"created_at": row["created_at"],
|
||||
"attachment_ids": list(row.get("attachment_ids") or []),
|
||||
}
|
||||
|
||||
|
||||
def _bearer_unauthorized() -> ApiError:
|
||||
return ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication is required.",
|
||||
headers={
|
||||
"WWW-Authenticate": 'Bearer realm="messenger"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _basic_unauthorized() -> ApiError:
|
||||
return ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials.",
|
||||
headers={
|
||||
"WWW-Authenticate": (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# Lab3 contract: ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def _invalid_request(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def _not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Object not found.")
|
||||
|
||||
|
||||
async def register_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
username: str,
|
||||
display_name: str,
|
||||
password: str,
|
||||
) -> dict[str, Any]:
|
||||
# Argon2 is CPU-bound (~0.5-1 s); off the event loop so /health/ready keeps answering.
|
||||
password_hash = await asyncio.to_thread(hash_password, password)
|
||||
user = await repositories.create_user(
|
||||
db,
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=password_hash,
|
||||
)
|
||||
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
409,
|
||||
"conflict",
|
||||
"Username is already taken.",
|
||||
)
|
||||
|
||||
return _to_user(user)
|
||||
|
||||
|
||||
async def create_user_session(
|
||||
db: AsyncSession,
|
||||
authorization: str | None,
|
||||
) -> dict[str, Any]:
|
||||
credentials = parse_basic_authorization(authorization)
|
||||
|
||||
if credentials is None:
|
||||
raise _basic_unauthorized()
|
||||
|
||||
username, password = credentials
|
||||
user = await repositories.get_user_by_username(db, username)
|
||||
|
||||
if user is None or not await asyncio.to_thread(
|
||||
verify_password,
|
||||
user["password_hash"],
|
||||
password,
|
||||
):
|
||||
raise _basic_unauthorized()
|
||||
|
||||
token = generate_session_token()
|
||||
expires_at = _utc_now() + timedelta(
|
||||
seconds=settings.session_ttl_seconds
|
||||
)
|
||||
|
||||
await repositories.create_session(
|
||||
db,
|
||||
user_id=user["id"],
|
||||
token_hash=hash_session_token(token),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at,
|
||||
"user": _to_user(user),
|
||||
}
|
||||
|
||||
|
||||
async def current_user_from_bearer_token(
|
||||
db: AsyncSession,
|
||||
token: str,
|
||||
) -> dict[str, Any]:
|
||||
token_hash = hash_session_token(token)
|
||||
cached = await session_cache.get(token_hash)
|
||||
|
||||
if cached is REVOKED_SESSION:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# Cache miss or cache unavailable: PostgreSQL decides.
|
||||
session_user = await repositories.get_active_session_user(db, token_hash)
|
||||
|
||||
if session_user is None:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
user = _to_user(session_user)
|
||||
await session_cache.put(token_hash, user, session_user["expires_at"])
|
||||
return user
|
||||
|
||||
|
||||
async def logout_current_session(
|
||||
db: AsyncSession,
|
||||
token: str,
|
||||
) -> None:
|
||||
token_hash = hash_session_token(token)
|
||||
revoked = await repositories.revoke_active_session(db, token_hash)
|
||||
|
||||
if not revoked:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
# Invalidate after COMMIT in the source of truth.
|
||||
await session_cache.revoke(token_hash)
|
||||
|
||||
|
||||
async def create_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
title: str,
|
||||
member_ids: list[UUID],
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
unique_member_ids = set(member_ids)
|
||||
|
||||
if len(unique_member_ids) != len(member_ids):
|
||||
raise _invalid_request(
|
||||
"member_ids must not contain duplicates."
|
||||
)
|
||||
|
||||
if current_user["id"] not in unique_member_ids:
|
||||
raise _invalid_request(
|
||||
"The current user must be included in member_ids."
|
||||
)
|
||||
|
||||
existing_ids = await repositories.user_ids_that_exist(
|
||||
db,
|
||||
unique_member_ids,
|
||||
)
|
||||
|
||||
if existing_ids != unique_member_ids:
|
||||
raise _invalid_request(
|
||||
"One or more member_ids do not exist."
|
||||
)
|
||||
|
||||
chat = await repositories.create_chat_with_members(
|
||||
db,
|
||||
title=title,
|
||||
member_ids=member_ids,
|
||||
)
|
||||
|
||||
return _to_chat(chat)
|
||||
|
||||
|
||||
async def get_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
return _to_chat(chat)
|
||||
|
||||
|
||||
async def list_chats(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
cursor_created_at: datetime | None = None
|
||||
cursor_id: UUID | None = None
|
||||
|
||||
if cursor is not None:
|
||||
decoded = decode_chat_cursor(cursor)
|
||||
|
||||
if decoded is None:
|
||||
raise _invalid_request("Invalid cursor.")
|
||||
|
||||
cursor_created_at, cursor_id = decoded
|
||||
|
||||
rows = await repositories.list_chats_for_user(
|
||||
db,
|
||||
user_id=current_user["id"],
|
||||
limit=limit,
|
||||
cursor_created_at=cursor_created_at,
|
||||
cursor_id=cursor_id,
|
||||
)
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
next_cursor = encode_chat_cursor(
|
||||
last["created_at"],
|
||||
last["id"],
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [_to_chat(row) for row in page],
|
||||
"next_cursor": next_cursor,
|
||||
}
|
||||
|
||||
|
||||
def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str:
|
||||
"""Hash of the validated body: key order and whitespace do not matter,
|
||||
a missing attachment_ids equals []. Without attachments the canonical
|
||||
form is the lab3 one, so keys stored by lab3 still replay correctly."""
|
||||
body: dict[str, Any] = {"text": text_value}
|
||||
if attachment_ids:
|
||||
body["attachment_ids"] = [str(item) for item in attachment_ids]
|
||||
canonical = json.dumps(
|
||||
body,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def create_message(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""Send a message once per (user, POST, chat, Idempotency-Key).
|
||||
|
||||
Returns (message, replayed). A replay re-checks the session (done by
|
||||
require_user) and chat membership before returning the stored result.
|
||||
"""
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_PATTERN.fullmatch(
|
||||
idempotency_key
|
||||
):
|
||||
raise _invalid_request(
|
||||
"Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}."
|
||||
)
|
||||
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
try:
|
||||
result = await repositories.create_message_idempotent(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
sender_id=current_user["id"],
|
||||
text_value=text_value,
|
||||
attachment_ids=attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
request_hash=_request_fingerprint(text_value, attachment_ids),
|
||||
ttl_seconds=settings.idempotency_ttl_seconds,
|
||||
)
|
||||
except repositories.AttachmentsNotFound:
|
||||
raise _not_found() from None
|
||||
except repositories.AttachmentFailed:
|
||||
raise ApiError(409, "conflict", "Failed attachment cannot be attached.") from None
|
||||
|
||||
if result is None:
|
||||
raise ApiError(
|
||||
409,
|
||||
"conflict",
|
||||
"Idempotency-Key was already used with a different body.",
|
||||
)
|
||||
|
||||
message, replayed = result
|
||||
return _to_message(message), replayed
|
||||
|
||||
|
||||
async def list_messages(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
cursor_created_at: datetime | None = None
|
||||
cursor_id: UUID | None = None
|
||||
|
||||
if cursor is not None:
|
||||
decoded = decode_message_cursor(cursor)
|
||||
|
||||
if decoded is None:
|
||||
raise _invalid_request("Invalid cursor.")
|
||||
|
||||
cursor_chat_id, cursor_created_at, cursor_id = decoded
|
||||
|
||||
if cursor_chat_id != chat_id:
|
||||
raise _invalid_request(
|
||||
"Cursor belongs to another chat."
|
||||
)
|
||||
|
||||
rows = await repositories.list_messages_keyset(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
limit=limit,
|
||||
cursor_created_at=cursor_created_at,
|
||||
cursor_id=cursor_id,
|
||||
)
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
next_cursor = encode_message_cursor(
|
||||
chat_id,
|
||||
last["created_at"],
|
||||
last["id"],
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [_to_message(row) for row in page],
|
||||
"next_cursor": next_cursor,
|
||||
}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
|
||||
|
||||
One long-lived aiobotocore client per API process, path-style addressing,
|
||||
short connect/read timeouts, at most 2 attempts and an overall deadline
|
||||
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
|
||||
fast 503 on the attachment endpoints only. Any client-side or
|
||||
network error is raised as StorageUnavailable; a missing key is
|
||||
ObjectMissing. Object keys are generated by the server (see services.py);
|
||||
the user's filename never becomes part of a key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import AsyncExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, BinaryIO
|
||||
from functools import wraps
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app.config import settings
|
||||
from app.telemetry import span
|
||||
|
||||
|
||||
def observed(operation):
|
||||
"""Export operation/outcome only, never S3 URL, key or credentials."""
|
||||
def decorate(method):
|
||||
@wraps(method)
|
||||
async def run(*args, **kwargs):
|
||||
with span("s3." + operation, {"rpc.system": "aws-api", "rpc.method": operation}):
|
||||
return await method(*args, **kwargs)
|
||||
return run
|
||||
return decorate
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.storage")
|
||||
|
||||
DOWNLOAD_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 is unreachable, timed out or refused the request."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""The key does not exist in the bucket."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: Any
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self) -> None:
|
||||
self._stack: AsyncExitStack | None = None
|
||||
self._client = None
|
||||
self.bucket = settings.s3_bucket
|
||||
|
||||
async def start(self) -> None:
|
||||
self._stack = AsyncExitStack()
|
||||
session = get_session()
|
||||
self._client = await self._stack.enter_async_context(
|
||||
session.create_client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id,
|
||||
aws_secret_access_key=settings.s3_secret_access_key,
|
||||
config=AioConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_read_timeout_seconds,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self._stack = None
|
||||
self._client = None
|
||||
|
||||
@staticmethod
|
||||
def _deadline(size: int = 0):
|
||||
return asyncio.timeout(
|
||||
settings.s3_operation_timeout_seconds
|
||||
+ size / settings.s3_put_min_bytes_per_second
|
||||
)
|
||||
|
||||
def _require_client(self):
|
||||
if self._client is None:
|
||||
raise StorageUnavailable("S3 client is not started")
|
||||
return self._client
|
||||
|
||||
@observed("PutObject")
|
||||
async def put(
|
||||
self,
|
||||
key: str,
|
||||
body: BinaryIO,
|
||||
size: int,
|
||||
content_type: str,
|
||||
) -> None:
|
||||
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
|
||||
itself rejects bytes that were damaged on the way."""
|
||||
client = self._require_client()
|
||||
body.seek(0)
|
||||
try:
|
||||
async with self._deadline(size):
|
||||
await client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
@observed("GetObject")
|
||||
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
|
||||
"""GET the object; S3 errors surface here, before any byte is sent."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
response = await client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise ObjectMissing(key) from exc
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
body = response["Body"]
|
||||
|
||||
async def chunks() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return chunks(), int(response["ContentLength"])
|
||||
|
||||
@observed("DeleteObject")
|
||||
async def delete(self, key: str) -> None:
|
||||
"""Idempotent: deleting a missing key succeeds in S3."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
@observed("HeadObject")
|
||||
async def exists(self, key: str) -> bool:
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return False
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
|
||||
client = self._require_client()
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
|
||||
for item in page.get("Contents", []):
|
||||
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
|
||||
storage = ObjectStorage()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Explicit, bounded OTLP instrumentation; no body/headers/SQL auto-capture.
|
||||
|
||||
Only this module's safe event logger is exported. Existing application and
|
||||
third-party loggers are deliberately not attached to the OTLP handler.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.propagate import extract
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
meter = metrics.get_meter("messenger", "1.6.0")
|
||||
tracer = trace.get_tracer("messenger", "1.6.0")
|
||||
http_requests = meter.create_counter("messenger.http.requests")
|
||||
http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
||||
jobs_processed = meter.create_counter("messenger.jobs.processed")
|
||||
jobs_duration = meter.create_histogram("messenger.jobs.duration", unit="s")
|
||||
export_failures = meter.create_counter("messenger.telemetry.export.failures")
|
||||
dropped_records = meter.create_counter("messenger.telemetry.dropped.records")
|
||||
|
||||
_providers: list = []
|
||||
_service = "messenger"
|
||||
_event_logger = logging.getLogger("messenger.telemetry.events")
|
||||
_event_logger.propagate = False
|
||||
_event_logger.setLevel(logging.INFO)
|
||||
_ALLOWED_FIELDS = {
|
||||
"event", "request_id", "trace_id", "span_id", "method", "route", "status_code",
|
||||
"outcome", "duration_seconds", "job_type", "job_id", "attachment_id", "attempt",
|
||||
"exception_type", "operation", "queue", "count",
|
||||
}
|
||||
|
||||
|
||||
class _NoHttpRetry:
|
||||
# Verified against pinned OTel 1.28.2. Retry/backpressure is the Collector's
|
||||
# responsibility; a failing SDK export gets one HTTP attempt, <=2 s socket
|
||||
# timeout, then discards its bounded batch. No request waits on export.
|
||||
@staticmethod
|
||||
def _retryable(_response):
|
||||
return False
|
||||
|
||||
|
||||
class _SpanExporter(_NoHttpRetry, OTLPSpanExporter):
|
||||
def export(self, spans):
|
||||
try:
|
||||
result = super().export(spans)
|
||||
except Exception:
|
||||
result = SpanExportResult.FAILURE
|
||||
if result != SpanExportResult.SUCCESS:
|
||||
export_failures.add(1, {"signal": "traces"})
|
||||
dropped_records.add(len(spans), {"signal": "traces"})
|
||||
return result
|
||||
|
||||
|
||||
class _LogExporter(_NoHttpRetry, OTLPLogExporter):
|
||||
def export(self, batch):
|
||||
try:
|
||||
result = super().export(batch)
|
||||
except Exception:
|
||||
result = LogExportResult.FAILURE
|
||||
if result != LogExportResult.SUCCESS:
|
||||
export_failures.add(1, {"signal": "logs"})
|
||||
dropped_records.add(len(batch), {"signal": "logs"})
|
||||
return result
|
||||
|
||||
|
||||
class _MetricExporter(_NoHttpRetry, OTLPMetricExporter):
|
||||
def export(self, metrics_data, timeout_millis=10000, **kwargs):
|
||||
try:
|
||||
result = super().export(metrics_data, timeout_millis, **kwargs)
|
||||
except Exception:
|
||||
result = MetricExportResult.FAILURE
|
||||
if result != MetricExportResult.SUCCESS:
|
||||
export_failures.add(1, {"signal": "metrics"})
|
||||
return result
|
||||
|
||||
|
||||
class _JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
value = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)),
|
||||
"severity": record.levelname, "service.name": _service,
|
||||
}
|
||||
value.update({key: getattr(record, key) for key in _ALLOWED_FIELDS
|
||||
if hasattr(record, key)})
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def configure(service: str, instance_id: str | None = None) -> None:
|
||||
"""One resource/provider set per process. OTEL_SDK_DISABLED supports off runs."""
|
||||
global _service
|
||||
if _providers:
|
||||
return
|
||||
_service = service
|
||||
console = logging.StreamHandler()
|
||||
console.setFormatter(_JsonFormatter())
|
||||
_event_logger.addHandler(console)
|
||||
if os.getenv("OTEL_SDK_DISABLED", "false").lower() == "true":
|
||||
return
|
||||
resource = Resource({
|
||||
"service.name": service,
|
||||
"service.version": os.getenv("APP_VERSION", "1.6.0"),
|
||||
"service.instance.id": instance_id or uuid4().hex,
|
||||
"deployment.environment.name": "lab",
|
||||
})
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
trace_provider = TracerProvider(resource=resource)
|
||||
trace_provider.add_span_processor(BatchSpanProcessor(
|
||||
_SpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64,
|
||||
schedule_delay_millis=1000, export_timeout_millis=2000,
|
||||
))
|
||||
trace.set_tracer_provider(trace_provider)
|
||||
log_provider = LoggerProvider(resource=resource)
|
||||
log_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
_LogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64,
|
||||
schedule_delay_millis=1000, export_timeout_millis=2000,
|
||||
))
|
||||
set_logger_provider(log_provider)
|
||||
_event_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
|
||||
reader = PeriodicExportingMetricReader(
|
||||
_MetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
|
||||
export_interval_millis=5000, export_timeout_millis=2500,
|
||||
)
|
||||
# Boundaries are seconds. Quantiles are computed over merged replica buckets.
|
||||
boundaries = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
|
||||
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
|
||||
View(instrument_type=metrics.Histogram,
|
||||
aggregation=ExplicitBucketHistogramAggregation(boundaries)),
|
||||
])
|
||||
metrics.set_meter_provider(metric_provider)
|
||||
_providers.extend((metric_provider, log_provider, trace_provider))
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
"""Flush bounded background buffers on normal termination (not SIGKILL)."""
|
||||
for provider in _providers:
|
||||
provider.shutdown()
|
||||
_providers.clear()
|
||||
|
||||
|
||||
def install_termination_handler() -> None:
|
||||
"""Turn worker/publisher SIGTERM into cancellation so their finally runs."""
|
||||
import asyncio
|
||||
loop = asyncio.get_running_loop()
|
||||
task = asyncio.current_task()
|
||||
if task is not None:
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGTERM, task.cancel)
|
||||
except NotImplementedError: # Windows host; Compose containers use POSIX.
|
||||
signal.signal(signal.SIGTERM, lambda *_: loop.call_soon_threadsafe(task.cancel))
|
||||
|
||||
|
||||
def emit(event: str, *, severity: int = logging.INFO, **fields) -> None:
|
||||
"""Call with a source-defined event and technical fields, never free text."""
|
||||
current = trace.get_current_span().get_span_context()
|
||||
attrs = {key: value for key, value in fields.items()
|
||||
if key in _ALLOWED_FIELDS and value is not None}
|
||||
attrs.update(event=event, trace_id=f"{current.trace_id:032x}", span_id=f"{current.span_id:016x}")
|
||||
_event_logger.log(severity, event, extra=attrs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, attributes=None, kind=SpanKind.INTERNAL, parent=None):
|
||||
"""Safe span helper: never export exception messages or stack traces."""
|
||||
with tracer.start_as_current_span(
|
||||
name, attributes=attributes, kind=kind, context=parent,
|
||||
record_exception=False, set_status_on_exception=False,
|
||||
) as active:
|
||||
try:
|
||||
yield active
|
||||
except BaseException as exc:
|
||||
active.set_attribute("exception_type", type(exc).__name__)
|
||||
active.set_status(Status(StatusCode.ERROR))
|
||||
raise
|
||||
|
||||
|
||||
class HttpTelemetryMiddleware:
|
||||
"""Outer ASGI wrapper sees final status, streaming duration and 500 responses."""
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
from starlette.requests import Request
|
||||
from app.errors import common_headers, incoming_request_id
|
||||
|
||||
request = Request(scope)
|
||||
request.state.request_id = incoming_request_id(request)
|
||||
method = scope.get("method", "OTHER")
|
||||
if method not in {"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}:
|
||||
method = "OTHER"
|
||||
status = 500
|
||||
started = time.perf_counter()
|
||||
# Only W3C propagation fields are consumed; headers are never exported.
|
||||
parent = extract(dict(request.headers))
|
||||
with span("HTTP request", kind=SpanKind.SERVER, parent=parent) as active:
|
||||
async def observed_send(message):
|
||||
nonlocal status
|
||||
if message["type"] == "http.response.start":
|
||||
status = message["status"]
|
||||
headers = dict(message.get("headers", []))
|
||||
for key, value in common_headers(request).items():
|
||||
headers[key.lower().encode()] = value.encode()
|
||||
headers[b"x-trace-id"] = f"{active.get_span_context().trace_id:032x}".encode()
|
||||
message = {**message, "headers": list(headers.items())}
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, observed_send)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
duration = time.perf_counter() - started
|
||||
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
|
||||
active.update_name(f"{method} {route}")
|
||||
active.set_attributes({**attrs, "http.response.status_code": status})
|
||||
if status >= 500:
|
||||
active.set_status(Status(StatusCode.ERROR))
|
||||
http_requests.add(1, attrs)
|
||||
http_duration.record(duration, attrs)
|
||||
emit("http.request.complete", severity=logging.ERROR if status >= 500 else logging.INFO,
|
||||
request_id=request.state.request_id, method=method, route=route,
|
||||
status_code=status, outcome="error" if status >= 500 else "success",
|
||||
duration_seconds=duration)
|
||||
@@ -0,0 +1,40 @@
|
||||
"""W3C context carried durably alongside a job, never its HTTP credentials.
|
||||
|
||||
The upload transaction stores ``carrier()``; publisher and retries restore it
|
||||
with ``parent_context``. Only a new producer/consumer span changes the parent
|
||||
span ID. Context-less legacy jobs deliberately start a new trace.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import get_current_span
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
_PROPAGATOR = TraceContextTextMapPropagator()
|
||||
_FIELDS = ("traceparent", "tracestate")
|
||||
|
||||
|
||||
def carrier() -> dict[str, str]:
|
||||
"""Return the active span's W3C fields, or {} outside a valid span."""
|
||||
headers: dict[str, str] = {}
|
||||
_PROPAGATOR.inject(headers)
|
||||
return {name: headers[name] for name in _FIELDS if headers.get(name)}
|
||||
|
||||
|
||||
def parent_context(row_or_job: Mapping | None) -> Context:
|
||||
"""Extract nullable DB/envelope fields without inheriting ambient context.
|
||||
|
||||
W3C parsing validates version, nonzero trace/span IDs, flags and tracestate.
|
||||
Malformed or missing parent IDs become a new root, even if a publisher's
|
||||
unrelated maintenance span happens to be active.
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
if isinstance(row_or_job, Mapping):
|
||||
for name in _FIELDS:
|
||||
value = row_or_job.get(name)
|
||||
if isinstance(value, str) and value:
|
||||
headers[name] = value
|
||||
context = _PROPAGATOR.extract(headers, context=Context())
|
||||
return context if get_current_span(context).get_span_context().is_valid else Context()
|
||||
+266
@@ -0,0 +1,266 @@
|
||||
"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
|
||||
arm_from_env, broker_channel, crash_if_armed,
|
||||
event, parse_envelope, retry_or_fail)
|
||||
from app.storage import ObjectMissing, storage
|
||||
from app.telemetry import (configure, emit, install_termination_handler, jobs_duration,
|
||||
jobs_processed, shutdown, span)
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from app import advanced
|
||||
from app.trace_context import carrier, parent_context
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def claim(job: dict) -> dict | None:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE id=:id FOR UPDATE
|
||||
"""), {"id": UUID(job["attachment_id"])})
|
||||
row = result.mappings().first()
|
||||
if row is None or row["job_id"] != UUID(job["job_id"]) or (
|
||||
row["pipeline_version"] != job["pipeline_version"] or
|
||||
row["attempt"] != job["attempt"] or row["status"] != "queued"
|
||||
):
|
||||
await db.rollback()
|
||||
return None
|
||||
# An early/manual duplicate cannot bypass the durable retry date.
|
||||
result = await db.execute(text("""
|
||||
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
|
||||
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
|
||||
traceparent=:traceparent,tracestate=:tracestate,
|
||||
updated_at=now()
|
||||
WHERE id=:id AND next_attempt_at<=now() RETURNING *
|
||||
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS,
|
||||
"traceparent": carrier().get("traceparent"), "tracestate": carrier().get("tracestate")})
|
||||
claimed = result.mappings().first()
|
||||
if claimed is None:
|
||||
await db.rollback()
|
||||
return None
|
||||
claimed = dict(claimed)
|
||||
await event(db, claimed, "processing")
|
||||
await db.commit()
|
||||
return claimed
|
||||
|
||||
|
||||
async def generate(source: Path, directory: Path) -> list[dict]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.image_worker", str(source), str(directory),
|
||||
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
try:
|
||||
async with asyncio.timeout(IMAGE_TIMEOUT):
|
||||
code = await process.wait()
|
||||
except TimeoutError:
|
||||
raise PermanentFailure("image_timeout") from None
|
||||
if code == 21:
|
||||
raise PermanentFailure("image_resource_limit")
|
||||
if code != 0:
|
||||
raise PermanentFailure("invalid_image")
|
||||
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
finally:
|
||||
# Includes cancellation of the entire job. Native image code cannot
|
||||
# continue using CPU/RAM after its parent timed out.
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def build_objects(row: dict) -> list[dict]:
|
||||
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
|
||||
directory = Path(temporary)
|
||||
source = directory / "source"
|
||||
if not row["staging_key"] or row["staging_deleted"]:
|
||||
raise PermanentFailure("source_missing")
|
||||
try:
|
||||
chunks, declared_length = await storage.open_download(row["staging_key"])
|
||||
except ObjectMissing:
|
||||
raise PermanentFailure("source_missing") from None
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
try:
|
||||
with source.open("wb") as output:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise PermanentFailure("source_too_large")
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
finally:
|
||||
await chunks.aclose()
|
||||
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
raise PermanentFailure("source_checksum_mismatch")
|
||||
with span("image.generate"):
|
||||
variants = await generate(source, directory)
|
||||
# API never publishes directly to this final prefix. Repeating one
|
||||
# v1 job writes the same bytes to the same four keys, never appends.
|
||||
with source.open("rb") as body:
|
||||
await storage.put(row["object_key"], body, size, row["content_type"])
|
||||
prefix = row["object_key"].rsplit("/", 1)[0]
|
||||
for variant in variants:
|
||||
filename = variant.pop("filename")
|
||||
key = f"{prefix}/{filename}"
|
||||
with (directory / filename).open("rb") as body:
|
||||
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
|
||||
variant["object_key"] = key
|
||||
await crash_if_armed("crash_after_objects_before_commit")
|
||||
return variants
|
||||
|
||||
|
||||
async def finish(row: dict, *, variants=None, code: str | None = None,
|
||||
permanent: bool = False) -> bool:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE id=:id AND status='processing'
|
||||
AND lease_token=:token AND lease_until>now() FOR UPDATE
|
||||
"""), {"id": row["id"], "token": row["lease_token"]})
|
||||
current = result.mappings().first()
|
||||
if current is None:
|
||||
# Lease recovery or explicit operator action already owns a
|
||||
# newer attempt. Never overwrite its state with an old worker.
|
||||
await db.rollback()
|
||||
return False
|
||||
if code is not None:
|
||||
await retry_or_fail(db, current, code, permanent=permanent)
|
||||
else:
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
|
||||
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
|
||||
WHERE id=:id
|
||||
"""), {"id": row["id"], "variants": json.dumps(variants)})
|
||||
await event(db, current, "ready")
|
||||
await db.commit()
|
||||
if code is None:
|
||||
advanced.observe_ready(current)
|
||||
return True
|
||||
|
||||
|
||||
async def _handle(message) -> str:
|
||||
try:
|
||||
job = parse_envelope(message.body)
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
logger.warning("invalid envelope sent to durable DLQ")
|
||||
await message.reject(requeue=False)
|
||||
return "invalid"
|
||||
row = await claim(job)
|
||||
if row is None:
|
||||
# A ready/failed/stale attempt is a durable no-op. A duplicate of
|
||||
# active processing is covered by the DB lease recovery, so it too
|
||||
# can be acknowledged without running a second concurrent decoder.
|
||||
await message.ack()
|
||||
return "duplicate"
|
||||
outcome = "error"
|
||||
try:
|
||||
async with asyncio.timeout(TOTAL_TIMEOUT):
|
||||
variants = await build_objects(row)
|
||||
except PermanentFailure as exc:
|
||||
await finish(row, code=str(exc), permanent=True)
|
||||
except Exception as exc:
|
||||
logger.warning("temporary processing failure attachment=%s type=%s",
|
||||
row["id"], type(exc).__name__)
|
||||
await finish(row, code="processing_failed")
|
||||
else:
|
||||
if await finish(row, variants=variants):
|
||||
await crash_if_armed("crash_after_commit_before_ack")
|
||||
logger.info("ready attachment=%s", row["id"])
|
||||
outcome = "success"
|
||||
else:
|
||||
outcome = "duplicate"
|
||||
# If finish failed, the exception propagates and the connection closes:
|
||||
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
|
||||
await message.ack()
|
||||
return outcome
|
||||
|
||||
|
||||
async def handle(message) -> None:
|
||||
# Technical IDs in logs/spans only; metric dimensions stay low-cardinality.
|
||||
job = {}
|
||||
try:
|
||||
job = parse_envelope(message.body)
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
pass
|
||||
started = time.perf_counter()
|
||||
outcome = "error"
|
||||
with span("attachments process", kind=SpanKind.CONSUMER, parent=parent_context(job), attributes={
|
||||
"messaging.system": "rabbitmq", "messaging.destination.name": "attachments",
|
||||
"messaging.operation": "process", "job_type": "attachment.process.v1",
|
||||
}) as active:
|
||||
try:
|
||||
advanced.busy = 1
|
||||
outcome = await _handle(message)
|
||||
finally:
|
||||
advanced.busy = 0
|
||||
duration = time.perf_counter() - started
|
||||
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
|
||||
jobs_processed.add(1, attrs)
|
||||
if outcome in {"success", "error"}:
|
||||
jobs_duration.record(duration, attrs)
|
||||
active.set_attribute("outcome", outcome)
|
||||
if outcome in {"error", "invalid"}:
|
||||
active.set_status(Status(StatusCode.ERROR))
|
||||
for name in ("job_id", "attachment_id", "attempt"):
|
||||
if name in job:
|
||||
active.set_attribute(name, job[name])
|
||||
emit("job.complete", severity=logging.WARNING if outcome == "error" else logging.INFO,
|
||||
**attrs, job_id=job.get("job_id"), attachment_id=job.get("attachment_id"),
|
||||
attempt=job.get("attempt"), duration_seconds=duration)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
configure("messenger-worker")
|
||||
install_termination_handler()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
for name in ("aio_pika", "aiormq"):
|
||||
logging.getLogger(name).setLevel(logging.CRITICAL)
|
||||
await storage.start()
|
||||
connection = None
|
||||
try:
|
||||
await arm_from_env("WORKER_FAULT")
|
||||
while True:
|
||||
try:
|
||||
connection, _, queue = await broker_channel()
|
||||
async with queue.iterator() as deliveries:
|
||||
async for message in deliveries:
|
||||
await handle(message)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("worker reconnect: %s", type(exc).__name__)
|
||||
if connection:
|
||||
await connection.close()
|
||||
connection = None
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
if connection:
|
||||
await connection.close()
|
||||
await storage.close()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(shutdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except asyncio.CancelledError:
|
||||
pass # SIGTERM has run finally/SDK flush; exit cleanly.
|
||||
+455
@@ -0,0 +1,455 @@
|
||||
#file: noinspection ComposeUnknownValues
|
||||
# Lab6: API x2 -> durable S3 staging + PostgreSQL outbox -> publisher
|
||||
# -> RabbitMQ -> worker -> final private S3 objects + PostgreSQL metadata.
|
||||
# Start: see REPORT.md, "Запуск из чистого clone".
|
||||
name: messenger-lab6
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
|
||||
SESSION_STORE: valkey
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_CACHE_TTL_SECONDS: "60"
|
||||
CACHE_TIMEOUT_SECONDS: "0.25"
|
||||
CACHE_MAX_CONNECTIONS: "20"
|
||||
CACHE_RETRY_AFTER_SECONDS: "5"
|
||||
DB_POOL_SIZE: "10"
|
||||
DB_MAX_OVERFLOW: "5"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_CONNECT_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
|
||||
SHUTDOWN_DRAIN_SECONDS: "3"
|
||||
SHUTDOWN_TIMEOUT_SECONDS: "8"
|
||||
# Lab4: S3 with the API's own restricted credentials (not admin).
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT_SECONDS: "2"
|
||||
S3_READ_TIMEOUT_SECONDS: "5"
|
||||
S3_OPERATION_TIMEOUT_SECONDS: "6"
|
||||
S3_PUT_MIN_BYTES_PER_SECOND: "1048576"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
UPLOAD_SPOOL_MEMORY_BYTES: "1048576"
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
MAX_CONCURRENT_UPLOADS: "8"
|
||||
UPLOAD_SLOT_WAIT_SECONDS: "10"
|
||||
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
|
||||
JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}"
|
||||
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
|
||||
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-30}"
|
||||
JOB_TOTAL_TIMEOUT_SECONDS: "${JOB_TOTAL_TIMEOUT_SECONDS:-90}"
|
||||
WORKER_MEMORY_MB: "${WORKER_MEMORY_MB:-768}"
|
||||
JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-120}"
|
||||
RETRY_BASE_SECONDS: "${RETRY_BASE_SECONDS:-1}"
|
||||
STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-604800}"
|
||||
PUBLISHER_FAULT: "${PUBLISHER_FAULT:-}"
|
||||
WORKER_FAULT: "${WORKER_FAULT:-}"
|
||||
# API fault-injection is disabled during normal operation.
|
||||
FAULT_INJECTION: "${FAULT_INJECTION:-}"
|
||||
OTEL_SDK_DISABLED: "${OTEL_SDK_DISABLED:-false}"
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES: service.version=1.6.0,deployment.environment.name=lab
|
||||
OTEL_METRIC_EXPORT_INTERVAL: "5000"
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
networks: [backend]
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 512M }
|
||||
|
||||
# Session cache only (lab2). PostgreSQL stays the source of truth, so the
|
||||
# cache is intentionally not persisted. It is shared by all API replicas:
|
||||
# a logout on one replica writes the "revoked" tombstone seen by all.
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.1-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
exec valkey-server
|
||||
--requirepass "$$(cat /run/secrets/valkey_password)"
|
||||
--save ""
|
||||
--appendonly no
|
||||
--maxmemory 64mb
|
||||
--maxmemory-policy allkeys-lru
|
||||
secrets: [valkey_password]
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- valkey-cli -a "$$(cat /run/secrets/valkey_password)" --no-auth-warning ping | grep -q PONG
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
|
||||
# S3: SeaweedFS all-in-one. No "ports": reachable only from the internal
|
||||
# backend network; its admin web UIs are bound to 127.0.0.1 inside the
|
||||
# container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.97
|
||||
entrypoint: ["/bin/sh", "/opt/lab/start.sh"]
|
||||
environment:
|
||||
# Soft limit of the Go heap: without it the GC lets SeaweedFS grow to
|
||||
# the container limit under parallel 10 MiB PUTs (measured, REPORT.md).
|
||||
GOMEMLIMIT: 600MiB
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/seaweedfs:/opt/lab:ro
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 60s
|
||||
networks: [backend]
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
# 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed
|
||||
# SeaweedFS to the limit and the kernel killed it (REPORT.md).
|
||||
limits: { cpus: "1.0", memory: 1G }
|
||||
|
||||
# Idempotent bucket bootstrap + self-checks (private bucket, least
|
||||
# privilege of the API key). The only container with the S3 admin key.
|
||||
s3-init:
|
||||
build: .
|
||||
image: messenger-lab6-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/s3_bootstrap.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
# Runs the migrations once, before any API replica starts (no race of
|
||||
# replicas). A repeated run only prints SKIP.
|
||||
migrate:
|
||||
build: .
|
||||
image: messenger-lab6-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
db: { condition: service_healthy }
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: messenger-lab6-api:${APP_VERSION:-dev}
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
# No "ports": replicas are reachable only through the LB.
|
||||
# No "container_name": the service is scaled by deploy.replicas / --scale.
|
||||
networks: [edge, backend]
|
||||
depends_on:
|
||||
db: { condition: service_healthy }
|
||||
valkey: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 2
|
||||
start_period: 5s
|
||||
stop_signal: SIGTERM
|
||||
# > SHUTDOWN_DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (3 + 8 s).
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
|
||||
# A stable nodename keeps durable queues in the same RabbitMQ database
|
||||
# even after the container is re-created. No AMQP/management port is public.
|
||||
broker:
|
||||
image: rabbitmq:3.13.7-alpine
|
||||
hostname: broker
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server'
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_VHOST: /
|
||||
# Bound Erlang schedulers to this small container, rather than the
|
||||
# host's core count; health-check CLI VMs need only one scheduler.
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
|
||||
secrets: [broker_password]
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
image: messenger-lab6-api:${APP_VERSION:-dev}
|
||||
command: ["python", "-m", "app.jobs"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
OTEL_SERVICE_NAME: messenger-publisher
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
broker: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
|
||||
worker:
|
||||
build: .
|
||||
image: messenger-lab6-api:${APP_VERSION:-dev}
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
broker: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
# One active job and one reserved delivery; image subprocess has its own
|
||||
# address-space/time limits in addition to this container memory limit.
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 1G }
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.11-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
# Backend state page for the operator, only on the host loopback.
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||||
networks: [edge]
|
||||
depends_on:
|
||||
api: { condition: service_healthy }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
# Telemetry is deliberately absent from API depends_on/readiness.
|
||||
# No OTLP ports on the host: application traffic stays in backend.
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.111.0
|
||||
command: ["--config=/etc/otelcol/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=512MB
|
||||
volumes:
|
||||
- ./infra/prometheus:/etc/prometheus:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
|
||||
networks: [backend, observability]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.2.1
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports: ["127.0.0.1:${LOKI_PORT:-3100}:3100"]
|
||||
networks: [backend, observability]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 384M }
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1
|
||||
environment:
|
||||
# Bound the Go heap while searching many retained blocks on this small stand.
|
||||
GOMEMLIMIT: 256MiB
|
||||
# A new named volume needs write access for the local evaluation backend.
|
||||
user: "0:0"
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports: ["127.0.0.1:${TEMPO_PORT:-3200}:3200"]
|
||||
networks: [backend, observability]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 512M }
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.2.2
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
|
||||
networks: [backend, observability]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
|
||||
# Runs the public and own checks in Linux (needed on Windows: the hooks are
|
||||
# POSIX .sh scripts that call the docker CLI). Profile "tools": not started
|
||||
# by a plain "up". The Docker socket grants control over the Docker host —
|
||||
# use it only on the lab stand.
|
||||
tester:
|
||||
profiles: [tools]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: student-tests/Dockerfile.tester
|
||||
working_dir: /work
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
COMPOSE_PROJECT: messenger-lab6
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
volumes:
|
||||
- ./:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# backend: only for the "private bucket from outside the API" check.
|
||||
networks: [edge, backend]
|
||||
|
||||
networks:
|
||||
observability: {}
|
||||
edge: {}
|
||||
# PostgreSQL, Valkey and S3: no route outside, the LB is not attached.
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data: {}
|
||||
s3_data: {}
|
||||
broker_data: {}
|
||||
prometheus_data: {}
|
||||
loki_data: {}
|
||||
tempo_data: {}
|
||||
grafana_data: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./infra/secrets/postgres_password.txt
|
||||
database_url:
|
||||
file: ./infra/secrets/database_url.txt
|
||||
valkey_password:
|
||||
file: ./infra/secrets/valkey_password.txt
|
||||
cache_url:
|
||||
file: ./infra/secrets/cache_url.txt
|
||||
s3_admin_access_key_id:
|
||||
file: ./infra/secrets/s3_admin_access_key_id.txt
|
||||
s3_admin_secret_access_key:
|
||||
file: ./infra/secrets/s3_admin_secret_access_key.txt
|
||||
s3_access_key_id:
|
||||
file: ./infra/secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./infra/secrets/s3_secret_access_key.txt
|
||||
broker_password:
|
||||
file: ./infra/secrets/broker_password.txt
|
||||
broker_url:
|
||||
file: ./infra/secrets/broker_url.txt
|
||||
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.14954320399965582",
|
||||
"event": "http.request.complete",
|
||||
"flags": "1",
|
||||
"method": "POST",
|
||||
"observed_timestamp": "1795252636588490258",
|
||||
"outcome": "success",
|
||||
"request_id": "observe-60a6c2da00814bf08cb3a801b8388ef4",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "b33ff621673d",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "7f43d8d88f960fff",
|
||||
"status_code": "202",
|
||||
"trace_id": "115550cd6d2c36ddf2bea96f48e89c6b"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795252636588292864",
|
||||
"http.request.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 3571455,
|
||||
"linesProcessedPerSecond": 18129,
|
||||
"totalBytesProcessed": 18124,
|
||||
"totalLinesProcessed": 92,
|
||||
"execTime": 0.005075,
|
||||
"queueTime": 0.000262,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 16192
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 1,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 18124,
|
||||
"headChunkLines": 92,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 16192,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 81728,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "eb62cd3e-75a1-4dbe-8965-9e012b312282",
|
||||
"attempt": "1",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.42506639099974564",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "e1b6a557-f9fa-462c-ac79-1ab71f0b35bf",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795252637174010973",
|
||||
"outcome": "success",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "4d9015b23ab54bde9d37384ddc4bf89b",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "c71f9d5977e9bd59",
|
||||
"trace_id": "115550cd6d2c36ddf2bea96f48e89c6b"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795252637173831168",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 79386,
|
||||
"linesProcessedPerSecond": 422,
|
||||
"totalBytesProcessed": 376,
|
||||
"totalLinesProcessed": 2,
|
||||
"execTime": 0.004736,
|
||||
"queueTime": 0.000244,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 352
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 1,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 376,
|
||||
"headChunkLines": 2,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 352,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 138740,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
|
||||
UTC: 2026-11-21T09:16:46Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:46Z
|
||||
@@ -0,0 +1,5 @@
|
||||
|
||||
UTC: 2026-11-21T09:16:45Z
|
||||
Command: docker compose config --quiet
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:46Z
|
||||
@@ -0,0 +1,220 @@
|
||||
|
||||
Before public persistence — 2026-11-21T09:14:39Z
|
||||
|
||||
UTC: 2026-11-21T09:14:39Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
|
||||
|
||||
UTC: 2026-11-21T09:14:40Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:10:22.045070464Z finished=2026-11-21T09:10:21.647750535Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:10:18.318585259Z finished=2026-11-21T09:10:17.940277751Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:10:24.158928332Z finished=2026-11-21T09:10:23.907977242Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T08:59:12.528963564Z finished=2026-11-21T08:58:31.826616128Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T08:51:52.621809468Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:10:55.825987305Z finished=2026-11-21T09:10:55.547390235Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:09.569875679Z finished=2026-11-21T09:13:44.880718518Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T08:45:28.390651046Z finished=2026-11-21T08:45:27.109421727Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:13:45.19691868Z finished=2026-11-21T09:13:41.499885247Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
|
||||
|
||||
After public persistence — 2026-11-21T09:15:30Z
|
||||
|
||||
UTC: 2026-11-21T09:15:30Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
Before public failover — 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
After public failover, before API restoration — 2026-11-21T09:15:52Z
|
||||
|
||||
UTC: 2026-11-21T09:15:52Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
|
||||
|
||||
UTC: 2026-11-21T09:15:53Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=exited started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:15:40.679293151Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
|
||||
|
||||
After API restoration — 2026-11-21T09:16:00Z
|
||||
|
||||
UTC: 2026-11-21T09:16:00Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
|
||||
UTC: 2026-11-21T09:16:01Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:54.07308443Z finished=2026-11-21T09:15:40.679293151Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
@@ -0,0 +1,187 @@
|
||||
{
|
||||
"captured_at": "2026-11-21T09:17:47.509294Z",
|
||||
"live_worker_capacity": 1.0,
|
||||
"panels": 24,
|
||||
"promql_checks": [
|
||||
{
|
||||
"panel": "Business API requests / second",
|
||||
"query": "sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m]))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Business API 5xx / all responses",
|
||||
"query": "(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "API mean duration (seconds)",
|
||||
"query": "sum(rate(messenger_http_duration_seconds_sum{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_duration_seconds_count{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])), 1e-9)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker attempt outcomes / second",
|
||||
"query": "sum by (outcome) (rate(messenger_jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Worker mean attempt duration (seconds)",
|
||||
"query": "sum(rate(messenger_jobs_duration_seconds_sum{service_name=\"messenger-worker\"}[1m])) / clamp_min(sum(rate(messenger_jobs_duration_seconds_count{service_name=\"messenger-worker\"}[1m])), 1e-9)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker completed attempts (counter)",
|
||||
"query": "sum by (outcome) (messenger_jobs_processed_total{service_name=\"messenger-worker\"})",
|
||||
"series": 3
|
||||
},
|
||||
{
|
||||
"panel": "Structured application events (expand for trace_id / span_id / request_id)",
|
||||
"query": "{service_name=~\"messenger-api|messenger-worker|messenger-publisher\"}",
|
||||
"series": 1343
|
||||
},
|
||||
{
|
||||
"panel": "API duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "API duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "API duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{service_name=\"messenger-api\",route!~\"/health/.*\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker duration p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Durable unfinished jobs (count)",
|
||||
"query": "max(messenger_queue_depth{service_name=\"messenger-publisher\"})",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Oldest unfinished job / observer sample age (seconds)",
|
||||
"query": "max(messenger_queue_oldest_age_seconds{service_name=\"messenger-publisher\"})",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Oldest unfinished job / observer sample age (seconds)",
|
||||
"query": "max(messenger_queue_sample_age_seconds{service_name=\"messenger-publisher\"})",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "DB pool used / capacity (connections)",
|
||||
"query": "messenger_db_pool_used",
|
||||
"series": 13
|
||||
},
|
||||
{
|
||||
"panel": "DB pool used / capacity (connections)",
|
||||
"query": "messenger_db_pool_capacity",
|
||||
"series": 13
|
||||
},
|
||||
{
|
||||
"panel": "DB acquisitions in progress (requests)",
|
||||
"query": "messenger_db_pool_waiting",
|
||||
"series": 13
|
||||
},
|
||||
{
|
||||
"panel": "Worker saturation (busy / capacity)",
|
||||
"query": "(sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)) / (sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker busy / capacity (jobs)",
|
||||
"query": "sum(messenger_worker_busy{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Worker busy / capacity (jobs)",
|
||||
"query": "sum(messenger_worker_capacity{service_name=\"messenger-worker\"} and on(service_instance_id) (time() - messenger_worker_heartbeat_seconds{service_name=\"messenger-worker\"} < 15)) or vector(0)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Application CPU (cores)",
|
||||
"query": "rate(messenger_process_cpu_seconds_total[1m])",
|
||||
"series": 13
|
||||
},
|
||||
{
|
||||
"panel": "Application resident RAM (bytes)",
|
||||
"query": "messenger_process_memory_bytes",
|
||||
"series": 13
|
||||
},
|
||||
{
|
||||
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Acceptance to ready p50 / p95 / p99 (seconds)",
|
||||
"query": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "API availability SLI / SLO (5 minutes)",
|
||||
"query": "1 - (sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{service_name=\"messenger-api\",route!~\"/health/.*\"}[5m]))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "API availability SLI / SLO (5 minutes)",
|
||||
"query": "vector(0.99)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
|
||||
"query": "sum(rate(messenger_attachment_time_to_ready_seconds_bucket{service_name=\"messenger-worker\",le=\"60\"}[5m])) / sum(rate(messenger_attachment_time_to_ready_seconds_count{service_name=\"messenger-worker\"}[5m]))",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Ready within 60 seconds SLI / SLO (5 minutes)",
|
||||
"query": "vector(0.95)",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Local alert state: pending / firing",
|
||||
"query": "ALERTS{alertname=~\"MessengerHighErrorRate|MessengerQueueLag\"}",
|
||||
"series": 0
|
||||
},
|
||||
{
|
||||
"panel": "Collector queue occupancy (batches)",
|
||||
"query": "otelcol_exporter_queue_size{exporter=~\"otlphttp/.*\"}",
|
||||
"series": 2
|
||||
},
|
||||
{
|
||||
"panel": "Collector queue occupancy (batches)",
|
||||
"query": "otelcol_exporter_queue_capacity{exporter=~\"otlphttp/.*\"}",
|
||||
"series": 2
|
||||
},
|
||||
{
|
||||
"panel": "Collector failed export attempts / second",
|
||||
"query": "rate(otelcol_exporter_send_failed_spans[1m])",
|
||||
"series": 1
|
||||
},
|
||||
{
|
||||
"panel": "Collector failed export attempts / second",
|
||||
"query": "rate(otelcol_exporter_send_failed_log_records[1m])",
|
||||
"series": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode failover --action-script scripts/stop-one.sh
|
||||
Container messenger-lab6-tester-run-0af389d5dbc3 Creating
|
||||
Container messenger-lab6-tester-run-0af389d5dbc3 Created
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.189}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:52Z
|
||||
|
||||
UTC: 2026-11-21T09:15:53Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester sh scripts/start-api.sh
|
||||
Container messenger-lab6-tester-run-b9e9496603b9 Creating
|
||||
Container messenger-lab6-tester-run-b9e9496603b9 Created
|
||||
started api container 81ebb23d65ec
|
||||
running api replicas: 2
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:00Z
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:53Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/image_validation_unit.py
|
||||
Container messenger-lab6-tester-run-56217a03ad55 Creating
|
||||
Container messenger-lab6-tester-run-56217a03ad55 Created
|
||||
test_adam7_pass_filter_boundaries (__main__.ImageValidationTests.test_adam7_pass_filter_boundaries) ... ok
|
||||
test_crc_compressed_tail_and_pixel_bomb (__main__.ImageValidationTests.test_crc_compressed_tail_and_pixel_bomb) ... ok
|
||||
test_critical_headers_and_chunk_order (__main__.ImageValidationTests.test_critical_headers_and_chunk_order) ... ok
|
||||
test_filter_values_and_row_boundaries (__main__.ImageValidationTests.test_filter_values_and_row_boundaries) ... ok
|
||||
test_jpeg_and_truncated_progressive_above_old_bypass_limit (__main__.ImageValidationTests.test_jpeg_and_truncated_progressive_above_old_bypass_limit) ... ok
|
||||
test_palette_shape_and_transparency (__main__.ImageValidationTests.test_palette_shape_and_transparency) ... ok
|
||||
test_valid_png_and_trailing_padding (__main__.ImageValidationTests.test_valid_png_and_trailing_padding) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 0.462s
|
||||
|
||||
OK
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:55Z
|
||||
@@ -0,0 +1,14 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:55Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_images.py
|
||||
Container messenger-lab6-tester-run-d0860115a1da Creating
|
||||
Container messenger-lab6-tester-run-d0860115a1da Created
|
||||
PASS centered crop, three real variants, checksums, original bytes and ACL
|
||||
PASS no upscaling and floor rounding for thumbnail/optimized
|
||||
PASS transparent PNG composited onto white JPEG
|
||||
PASS EXIF orientation applied to variants and metadata removed
|
||||
PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202
|
||||
PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400
|
||||
PASS: lab5 grade4 image checks
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:05Z
|
||||
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "error",
|
||||
"duration_seconds": "3.3880072909998944",
|
||||
"event": "http.request.complete",
|
||||
"flags": "1",
|
||||
"method": "POST",
|
||||
"observed_timestamp": "1795251516034833239",
|
||||
"outcome": "error",
|
||||
"request_id": "f44f1b74-f3f1-4cbe-8b44-510cfbe21ebd",
|
||||
"route": "/api/v1/users",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "0d13c5814dab",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "17",
|
||||
"severity_text": "ERROR",
|
||||
"span_id": "6389f79f96d8c840",
|
||||
"status_code": "503",
|
||||
"trace_id": "819eab187881e315742feb7f2ebb70ff"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251516034664192",
|
||||
"http.request.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 4455963,
|
||||
"linesProcessedPerSecond": 22619,
|
||||
"totalBytesProcessed": 70329,
|
||||
"totalLinesProcessed": 357,
|
||||
"execTime": 0.015783,
|
||||
"queueTime": 0.002886,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 2,
|
||||
"shards": 2,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 62832
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 2,
|
||||
"totalChunksMatched": 2,
|
||||
"totalBatches": 3,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 70329,
|
||||
"headChunkLines": 357,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 62832,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 584920,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"batches": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "1.6.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "0d13c5814dab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-api"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger",
|
||||
"version": "1.6.0"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "gZ6rGHiB4xV0L+t/Lrtw/w==",
|
||||
"spanId": "z5kgvNQvytc=",
|
||||
"parentSpanId": "Y4n3n5bYyEA=",
|
||||
"name": "db.acquire",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"startTimeUnixNano": "1795251513033234050",
|
||||
"endTimeUnixNano": "1795251516033386648",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "exception_type",
|
||||
"value": {
|
||||
"stringValue": "TimeoutError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "gZ6rGHiB4xV0L+t/Lrtw/w==",
|
||||
"spanId": "Y4n3n5bYyEA=",
|
||||
"name": "POST /api/v1/users",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1795251512647658833",
|
||||
"endTimeUnixNano": "1795251516034910448",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/users"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "status_class",
|
||||
"value": {
|
||||
"stringValue": "5xx"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "503"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:39Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/test_lab5.py
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools ps --status running --services
|
||||
api
|
||||
broker
|
||||
db
|
||||
grafana
|
||||
lb
|
||||
loki
|
||||
otel-collector
|
||||
prometheus
|
||||
publisher
|
||||
s3
|
||||
tempo
|
||||
valkey
|
||||
worker
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
|
||||
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
|
||||
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 34 minutes ago Up 24 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 34 minutes ago Up 10 minutes (healthy) 5432/tcp
|
||||
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 19 minutes ago Up 18 minutes 127.0.0.1:3000->3000/tcp
|
||||
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 17 minutes ago Up 17 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
|
||||
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 19 minutes ago Up 8 minutes 127.0.0.1:3100->3100/tcp
|
||||
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate 18 minutes ago Exited (0) 16 minutes ago
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 20 minutes ago Up 20 minutes 4317/tcp, 55678-55679/tcp
|
||||
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 18 minutes 127.0.0.1:9090->9090/tcp
|
||||
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher 18 minutes ago Up 17 minutes 8080/tcp
|
||||
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 34 minutes ago Up 13 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init 18 minutes ago Exited (0) 16 minutes ago
|
||||
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes 127.0.0.1:3200->3200/tcp
|
||||
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 34 minutes ago Up 24 minutes (healthy) 6379/tcp
|
||||
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker 18 minutes ago Up 5 minutes 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools build tester
|
||||
Image messenger-lab6-tester Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 587B 0.0s done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile.tester
|
||||
#2 transferring dockerfile: 405B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.11-alpine3.22
|
||||
#3 DONE 0.2s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/docker:28.5.1-cli
|
||||
#4 DONE 0.2s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 286B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [stage-1 1/5] FROM docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
|
||||
#6 resolve docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 0.0s done
|
||||
#6 DONE 0.1s
|
||||
|
||||
#7 [docker-cli 1/1] FROM docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad
|
||||
#7 resolve docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad 0.0s done
|
||||
#7 DONE 0.1s
|
||||
|
||||
#8 [stage-1 2/5] RUN apk add --no-cache make
|
||||
#8 CACHED
|
||||
|
||||
#9 [stage-1 3/5] RUN pip install --no-cache-dir Pillow==11.0.0
|
||||
#9 CACHED
|
||||
|
||||
#10 [stage-1 4/5] COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||
#10 CACHED
|
||||
|
||||
#11 [stage-1 5/5] WORKDIR /work
|
||||
#11 CACHED
|
||||
|
||||
#12 exporting to image
|
||||
#12 exporting layers done
|
||||
#12 exporting manifest sha256:1c93bba11e9ded8b819bbcadafd9217f8b6b4fc5301deeca1e720c49a45f1b2e done
|
||||
#12 exporting config sha256:9ad2daad56b9cfccc6c03e7be0468faf51994e1604e86b502077a03f28844325 done
|
||||
#12 exporting attestation manifest sha256:96fa2816d4a538ed8e6e3ea5c702033990470ee27e842b91c5f2979f0304e95a 0.1s done
|
||||
#12 exporting manifest list sha256:3e586fe0bf4aeb3814ad182a780c6173a9e252da1c33e7e61f20187228c8b23f 0.0s done
|
||||
#12 naming to docker.io/library/messenger-lab6-tester:latest
|
||||
#12 naming to docker.io/library/messenger-lab6-tester:latest done
|
||||
#12 unpacking to docker.io/library/messenger-lab6-tester:latest 0.0s done
|
||||
#12 DONE 0.2s
|
||||
|
||||
#13 resolving provenance for metadata file
|
||||
#13 DONE 0.0s
|
||||
Image messenger-lab6-tester Built
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:42Z
|
||||
|
||||
UTC: 2026-11-21T09:09:42Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u scripts/check_contract.py
|
||||
Container messenger-lab6-tester-run-e5ea96fea8b0 Creating
|
||||
Container messenger-lab6-tester-run-e5ea96fea8b0 Created
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:46Z
|
||||
|
||||
UTC: 2026-11-21T09:09:46Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/smoke.py
|
||||
Container messenger-lab6-tester-run-42a477b98467 Creating
|
||||
Container messenger-lab6-tester-run-42a477b98467 Created
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:53Z
|
||||
|
||||
UTC: 2026-11-21T09:09:53Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/image_validation_unit.py
|
||||
Container messenger-lab6-tester-run-56217a03ad55 Creating
|
||||
Container messenger-lab6-tester-run-56217a03ad55 Created
|
||||
test_adam7_pass_filter_boundaries (__main__.ImageValidationTests.test_adam7_pass_filter_boundaries) ... ok
|
||||
test_crc_compressed_tail_and_pixel_bomb (__main__.ImageValidationTests.test_crc_compressed_tail_and_pixel_bomb) ... ok
|
||||
test_critical_headers_and_chunk_order (__main__.ImageValidationTests.test_critical_headers_and_chunk_order) ... ok
|
||||
test_filter_values_and_row_boundaries (__main__.ImageValidationTests.test_filter_values_and_row_boundaries) ... ok
|
||||
test_jpeg_and_truncated_progressive_above_old_bypass_limit (__main__.ImageValidationTests.test_jpeg_and_truncated_progressive_above_old_bypass_limit) ... ok
|
||||
test_palette_shape_and_transparency (__main__.ImageValidationTests.test_palette_shape_and_transparency) ... ok
|
||||
test_valid_png_and_trailing_padding (__main__.ImageValidationTests.test_valid_png_and_trailing_padding) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 0.462s
|
||||
|
||||
OK
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:55Z
|
||||
|
||||
UTC: 2026-11-21T09:09:55Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_images.py
|
||||
Container messenger-lab6-tester-run-d0860115a1da Creating
|
||||
Container messenger-lab6-tester-run-d0860115a1da Created
|
||||
PASS centered crop, three real variants, checksums, original bytes and ACL
|
||||
PASS no upscaling and floor rounding for thumbnail/optimized
|
||||
PASS transparent PNG composited onto white JPEG
|
||||
PASS EXIF orientation applied to variants and metadata removed
|
||||
PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202
|
||||
PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400
|
||||
PASS: lab5 grade4 image checks
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:05Z
|
||||
|
||||
UTC: 2026-11-21T09:10:05Z
|
||||
Command: docker compose --profile tools exec -T worker python -u - < student-tests/lab5_limits.py
|
||||
PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB
|
||||
PASS timeout: decoder killed and reaped; parent continues
|
||||
PASS: lab5 decoder resource limits
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:08Z
|
||||
|
||||
UTC: 2026-11-21T09:10:08Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
Container messenger-lab6-tester-run-100a83886eb9 Creating
|
||||
Container messenger-lab6-tester-run-100a83886eb9 Created
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:09Z
|
||||
|
||||
UTC: 2026-11-21T09:10:09Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_recovery.py
|
||||
Container messenger-lab6-tester-run-6606f20c462f Creating
|
||||
Container messenger-lab6-tester-run-6606f20c462f Created
|
||||
PASS persistence: attachment=9d7de7ac-016a-49d7-a0f1-9b880cee3c83 same session/ID after API+broker restart; volumes preserved; queued content409; messages201; ready
|
||||
PASS crash_after_commit_before_publish: attachment=af40fc00-4da4-4fe6-80b3-27a8879d7661 hits=1 recovered_seconds=4.0; same original; three variants
|
||||
PASS crash_after_objects_before_commit: attachment=30ee3a05-086c-45fd-bbf1-78df42514413 hits=1 recovered_seconds=118.6; same original; three variants
|
||||
PASS crash_after_commit_before_ack: attachment=4aa3f19d-638f-492a-b17e-ccb103f1cae9 hits=1 recovered_seconds=0.4; same original; three variants
|
||||
PASS failed/replay: attachment=00b66546-aa21-4115-aca1-13fbc7f47f2c; permanent error fails once; failed registry; content/link409; audited replay preserves job_id; ready duplicate no-op
|
||||
PASS transient S3 outage: attachment=242a36c2-40f8-440f-899b-228164e5ddc4; attempts=5; durable retries, retry_exhausted, explicit replay after recovery
|
||||
PASS: lab5 grade4 recovery checks (all)
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:39Z
|
||||
|
||||
UTC: 2026-11-21T09:14:39Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
|
||||
|
||||
UTC: 2026-11-21T09:14:40Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:10:22.045070464Z finished=2026-11-21T09:10:21.647750535Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:10:18.318585259Z finished=2026-11-21T09:10:17.940277751Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:10:24.158928332Z finished=2026-11-21T09:10:23.907977242Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T08:59:12.528963564Z finished=2026-11-21T08:58:31.826616128Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T08:51:52.621809468Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:10:55.825987305Z finished=2026-11-21T09:10:55.547390235Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:09.569875679Z finished=2026-11-21T09:13:44.880718518Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T08:45:28.390651046Z finished=2026-11-21T08:45:27.109421727Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:13:45.19691868Z finished=2026-11-21T09:13:41.499885247Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:40Z
|
||||
|
||||
UTC: 2026-11-21T09:14:40Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
Container messenger-lab6-tester-run-e3bdaf9250f1 Creating
|
||||
Container messenger-lab6-tester-run-e3bdaf9250f1 Created
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:30Z
|
||||
|
||||
UTC: 2026-11-21T09:15:30Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:14:49.40226195Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:31Z
|
||||
|
||||
UTC: 2026-11-21T09:15:31Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode failover --action-script scripts/stop-one.sh
|
||||
Container messenger-lab6-tester-run-0af389d5dbc3 Creating
|
||||
Container messenger-lab6-tester-run-0af389d5dbc3 Created
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.189}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:52Z
|
||||
|
||||
UTC: 2026-11-21T09:15:52Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
|
||||
|
||||
UTC: 2026-11-21T09:15:53Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=exited started=2026-11-21T09:15:18.227427331Z finished=2026-11-21T09:15:40.679293151Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:53Z
|
||||
|
||||
UTC: 2026-11-21T09:15:53Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester sh scripts/start-api.sh
|
||||
Container messenger-lab6-tester-run-b9e9496603b9 Creating
|
||||
Container messenger-lab6-tester-run-b9e9496603b9 Created
|
||||
started api container 81ebb23d65ec
|
||||
running api replicas: 2
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:00Z
|
||||
|
||||
UTC: 2026-11-21T09:16:00Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85
|
||||
0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8
|
||||
d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d
|
||||
0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084
|
||||
b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026
|
||||
5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4
|
||||
278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300
|
||||
68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf
|
||||
93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c
|
||||
13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b
|
||||
cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb
|
||||
a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612
|
||||
00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007
|
||||
fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05
|
||||
837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053
|
||||
27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
|
||||
UTC: 2026-11-21T09:16:01Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 81ebb23d65ec0d8f5eb9ae2d4da96e25e039c243048d325762805276860b5a85 0d13c5814dabaec1e9c2fab5b0199548668d44d3545aea4275250859d25b7dd8 d7b60f9558f079a28b76494c527c9e9341263729f5a0118ac6db1d1d388be54d 0cfc9b11ef3caed56dd7e4b22337e7c92163cc447dba971c29c277f07790a084 b40de06dc1d77aa83f1f0ca68bbc1dfe9fe337be16be9f3340993d781060b026 5448a293692bfe7b4297906534f74dc8e65793746d349d6bf7e674766ef943e4 278dcc4f37a2344d41b2a924fa14e9f8282a3671719c516121850092b741f300 68d796ac925f0ca1f88927a1a31486f228248d80c7a38966b22c14b48a5d07bf 93a2b85ca3d137ee5c98fe69c918a1496988c811eef4c8563d7018a128b1546c 13523d87d42a41c573081eede09a9e8afe9378e030c850214b2f063f7a9abe5b cdc87d3378f2c06dba602760661e60141929e2fb1b7248d59769a57987f2b2bb a30fa6baae40e97264da0406c72178f0d2e5e194d8c7dbb916b1bf9f69ff0612 00e5826b0d93e9be4fd82cd2210d4589ecd1d27368d80e2f7a609e9920604007 fc5561671768ddd4b9a405d5be8dbc5f0d19335520b95218d4a2f519ef590a05 837eda8499aa16593a8c42b89e87fa1b5d834d240f8d5247a7c49027d015a053 27b1583583d52b794e554a5df74bebd61c4118bf3e2de5aef38295ba8bc780dd
|
||||
/messenger-lab6-api-1 status=running started=2026-11-21T09:15:54.07308443Z finished=2026-11-21T09:15:40.679293151Z
|
||||
/messenger-lab6-api-2 status=running started=2026-11-21T09:15:17.875504619Z finished=2026-11-21T09:14:49.181726236Z
|
||||
/messenger-lab6-broker-1 status=running started=2026-11-21T09:14:51.311241575Z finished=2026-11-21T09:14:49.824073591Z
|
||||
/messenger-lab6-db-1 status=running started=2026-11-21T09:14:50.494933266Z finished=2026-11-21T09:14:49.814681938Z
|
||||
/messenger-lab6-grafana-1 status=running started=2026-11-21T08:51:11.412805651Z finished=2026-11-21T08:51:10.520984964Z
|
||||
/messenger-lab6-lb-1 status=running started=2026-11-21T09:15:24.826206264Z finished=2026-11-21T09:14:45.274514498Z
|
||||
/messenger-lab6-loki-1 status=running started=2026-11-21T09:01:34.707920007Z finished=2026-11-21T09:00:28.644856083Z
|
||||
/messenger-lab6-migrate-1 status=exited started=2026-11-21T08:52:49.834748909Z finished=2026-11-21T08:52:50.79798573Z
|
||||
/messenger-lab6-otel-collector-1 status=running started=2026-11-21T08:49:03.407564143Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-prometheus-1 status=running started=2026-11-21T08:51:11.311294483Z finished=2026-11-21T08:51:10.512998838Z
|
||||
/messenger-lab6-publisher-1 status=running started=2026-11-21T09:15:17.332690697Z finished=2026-11-21T09:14:45.830823803Z
|
||||
/messenger-lab6-s3-1 status=running started=2026-11-21T09:14:51.024302658Z finished=2026-11-21T09:14:49.817292594Z
|
||||
/messenger-lab6-s3-init-1 status=exited started=2026-11-21T08:53:15.109417522Z finished=2026-11-21T08:53:17.005092363Z
|
||||
/messenger-lab6-tempo-1 status=running started=2026-11-21T09:06:37.030642992Z finished=0001-01-01T00:00:00Z
|
||||
/messenger-lab6-valkey-1 status=running started=2026-11-21T09:14:50.777977548Z finished=2026-11-21T09:14:49.803932669Z
|
||||
/messenger-lab6-worker-1 status=running started=2026-11-21T09:15:17.592030927Z finished=2026-11-21T09:14:45.554953067Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
|
||||
UTC: 2026-11-21T09:16:01Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab6-api-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 7 seconds (healthy) 8080/tcp
|
||||
messenger-lab6-api-2 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 43 seconds (healthy) 8080/tcp
|
||||
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 40 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 40 minutes ago Up About a minute (healthy) 5432/tcp
|
||||
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 25 minutes ago Up 24 minutes 127.0.0.1:3000->3000/tcp
|
||||
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 24 minutes ago Up 36 seconds (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
|
||||
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 25 minutes ago Up 14 minutes 127.0.0.1:3100->3100/tcp
|
||||
messenger-lab6-migrate-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/migr…" migrate 24 minutes ago Exited (0) 23 minutes ago
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 27 minutes ago Up 26 minutes 4317/tcp, 55678-55679/tcp
|
||||
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 25 minutes ago Up 24 minutes 127.0.0.1:9090->9090/tcp
|
||||
messenger-lab6-publisher-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.jobs" publisher 24 minutes ago Up 44 seconds 8080/tcp
|
||||
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 40 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab6-s3-init-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/s3_b…" s3-init 24 minutes ago Exited (0) 22 minutes ago
|
||||
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 9 minutes ago Up 9 minutes 127.0.0.1:3200->3200/tcp
|
||||
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 40 minutes ago Up About a minute (healthy) 6379/tcp
|
||||
messenger-lab6-worker-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.worker" worker 24 minutes ago Up 44 seconds 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
|
||||
PASS: lab5 full checks. UTF-8 logs: C:\Users\masha\WebstormProjects\mtusi2026lab6\evidence
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-21T09:10:05Z
|
||||
Command: docker compose --profile tools exec -T worker python -u - < student-tests/lab5_limits.py
|
||||
PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB
|
||||
PASS timeout: decoder killed and reaped; parent continues
|
||||
PASS: lab5 decoder resource limits
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:08Z
|
||||
@@ -0,0 +1,562 @@
|
||||
{
|
||||
"start": "2026-11-21T08:51:58Z",
|
||||
"end": "2026-11-21T08:52:21Z",
|
||||
"queries": {
|
||||
"rss_bytes": "messenger_process_memory_bytes",
|
||||
"cpu_cores": "rate(messenger_process_cpu_seconds_total[15s])"
|
||||
},
|
||||
"results": {
|
||||
"rss_bytes": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "matrix",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "0d13c5814dab",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "0d13c5814dab",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"120827904"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"121270272"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"122515456"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"125259776"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"127827968"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "0fa6e7066e3145a993d3264494361be3",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "0fa6e7066e3145a993d3264494361be3",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"89513984"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"89513984"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"89513984"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"89513984"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"89513984"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "551e5017a02c4426af0773950c83c10e",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "551e5017a02c4426af0773950c83c10e",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"88707072"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"88707072"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"88707072"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"88707072"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"88707072"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "6cacaff48df9",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "6cacaff48df9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"122683392"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"122683392"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"122683392"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"122683392"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"122683392"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "81ebb23d65ec",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "81ebb23d65ec",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"120791040"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"121085952"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"122593280"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"125325312"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"128040960"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "a068b747445b44439e7e1b7bc72675c2",
|
||||
"exported_job": "messenger-publisher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "a068b747445b44439e7e1b7bc72675c2",
|
||||
"service_name": "messenger-publisher",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"88813568"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"88813568"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"88813568"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"88813568"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"88813568"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "da2c8073a2144ecc9965ea4553926bf3",
|
||||
"exported_job": "messenger-publisher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "da2c8073a2144ecc9965ea4553926bf3",
|
||||
"service_name": "messenger-publisher",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"89305088"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"89305088"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"89305088"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"89305088"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"89305088"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "messenger_process_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "e16c50966c56",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "e16c50966c56",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"123514880"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"123514880"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"123514880"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"123514880"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"123514880"
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"cpu_cores": {
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "matrix",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "0d13c5814dab",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "0d13c5814dab",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251123,
|
||||
"0.022415557507320008"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0.17908626842684267"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0.3572547659765976"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0.3890083826"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "0fa6e7066e3145a993d3264494361be3",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "0fa6e7066e3145a993d3264494361be3",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "551e5017a02c4426af0773950c83c10e",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "551e5017a02c4426af0773950c83c10e",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251123,
|
||||
"0.0005782936565199904"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0.0007851155115511435"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0.0005319559955995547"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0.0004686850999999992"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "6cacaff48df9",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "6cacaff48df9",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "81ebb23d65ec",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "81ebb23d65ec",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251123,
|
||||
"0.038017601540733366"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0.17548991699169916"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0.34144971107110705"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0.3803775972000001"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "a068b747445b44439e7e1b7bc72675c2",
|
||||
"exported_job": "messenger-publisher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "a068b747445b44439e7e1b7bc72675c2",
|
||||
"service_name": "messenger-publisher",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "da2c8073a2144ecc9965ea4553926bf3",
|
||||
"exported_job": "messenger-publisher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "da2c8073a2144ecc9965ea4553926bf3",
|
||||
"service_name": "messenger-publisher",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251123,
|
||||
"0.01116402495005333"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0.020074461846184624"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0.021264563956395647"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0.021109743999999986"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "e16c50966c56",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "messenger",
|
||||
"service_instance_id": "e16c50966c56",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
1795251118,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251123,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251128,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251133,
|
||||
"0"
|
||||
],
|
||||
[
|
||||
1795251138,
|
||||
"0"
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 20.0,
|
||||
"elapsed_seconds": 20.27,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1645,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 81.154,
|
||||
"successful_latency_ms": {
|
||||
"p50": 38.796,
|
||||
"p95": 114.354,
|
||||
"p99": 194.742
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
|
||||
UTC: 2026-11-21T08:51:58Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 tests/load.py --base-url http://localhost:8086 --duration 20 --concurrency 4 --output evidence/load-summary.json
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 20.0,
|
||||
"elapsed_seconds": 20.27,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 1645,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 81.154,
|
||||
"successful_latency_ms": {
|
||||
"p50": 38.796,
|
||||
"p95": 114.354,
|
||||
"p99": 194.742
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T08:52:21Z
|
||||
@@ -0,0 +1,11 @@
|
||||
|
||||
UTC: 2026-11-21T09:16:46Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester make check test
|
||||
Container messenger-lab6-tester-run-f02cd878c6ba Creating
|
||||
Container messenger-lab6-tester-run-f02cd878c6ba Created
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://lb:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:55Z
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
|
||||
UTC: 2026-11-21T09:16:58Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --baseline
|
||||
PASS: real API/worker logs, traces and metrics; evidence/observability-baseline.json
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:17:18Z
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-21T08:58:20Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --incidents
|
||||
PASS: worker stopped
|
||||
PASS: PostgreSQL unavailable
|
||||
PASS: Tempo and Loki unavailable
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:02:41Z
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"scenario": "S3 unavailable during first attempt, then restored",
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"request_id": "observe-927e3831c3d241ac80d3d43391ad1ff4",
|
||||
"api_trace_id": "94af84b9ec1663416ac05526995fbd91",
|
||||
"accepted_at": "2026-11-21T08:56:07.085875Z",
|
||||
"worker_trace_id": "94af84b9ec1663416ac05526995fbd91",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"worker_attempts": [
|
||||
"1",
|
||||
"2",
|
||||
"2",
|
||||
"3"
|
||||
],
|
||||
"same_trace": true,
|
||||
"correlated_at": "2026-11-21T08:58:20.029926Z",
|
||||
"content": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"original_bytes": 111817,
|
||||
"thumbnail_size": [
|
||||
256,
|
||||
160
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
UTC: 2026-11-21T08:56:05Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 scripts/observe_lab6.py --chain-with-retry
|
||||
PASS: propagated upload/publish/consume/S3/DB trace including retry
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T08:58:20Z
|
||||
@@ -0,0 +1,15 @@
|
||||
|
||||
UTC: 2026-11-21T09:16:55Z
|
||||
Command: docker compose exec -T api python - < student-tests/observability_unit.py
|
||||
test_http_route_template_and_request_trace_log_correlation_without_secrets (__main__.ObservabilityTests.test_http_route_template_and_request_trace_log_correlation_without_secrets) ... ok
|
||||
test_missing_or_invalid_context_starts_new_root_even_inside_span (__main__.ObservabilityTests.test_missing_or_invalid_context_starts_new_root_even_inside_span) ... ok
|
||||
test_unknown_route_and_exception_message_do_not_escape_into_telemetry (__main__.ObservabilityTests.test_unknown_route_and_exception_message_do_not_escape_into_telemetry) ... ok
|
||||
test_unsampled_trace_flags_and_tracestate_survive (__main__.ObservabilityTests.test_unsampled_trace_flags_and_tracestate_survive) ... ok
|
||||
test_w3c_parent_child_and_durable_retry_preserve_trace (__main__.ObservabilityTests.test_w3c_parent_child_and_durable_retry_preserve_trace) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 5 tests in 0.694s
|
||||
|
||||
OK
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:58Z
|
||||
@@ -0,0 +1,9 @@
|
||||
|
||||
UTC: 2026-11-21T09:14:40Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
Container messenger-lab6-tester-run-e3bdaf9250f1 Creating
|
||||
Container messenger-lab6-tester-run-e3bdaf9250f1 Created
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:15:30Z
|
||||
@@ -0,0 +1,14 @@
|
||||
|
||||
UTC: 2026-11-21T09:10:09Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_recovery.py
|
||||
Container messenger-lab6-tester-run-6606f20c462f Creating
|
||||
Container messenger-lab6-tester-run-6606f20c462f Created
|
||||
PASS persistence: attachment=9d7de7ac-016a-49d7-a0f1-9b880cee3c83 same session/ID after API+broker restart; volumes preserved; queued content409; messages201; ready
|
||||
PASS crash_after_commit_before_publish: attachment=af40fc00-4da4-4fe6-80b3-27a8879d7661 hits=1 recovered_seconds=4.0; same original; three variants
|
||||
PASS crash_after_objects_before_commit: attachment=30ee3a05-086c-45fd-bbf1-78df42514413 hits=1 recovered_seconds=118.6; same original; three variants
|
||||
PASS crash_after_commit_before_ack: attachment=4aa3f19d-638f-492a-b17e-ccb103f1cae9 hits=1 recovered_seconds=0.4; same original; three variants
|
||||
PASS failed/replay: attachment=00b66546-aa21-4115-aca1-13fbc7f47f2c; permanent error fails once; failed registry; content/link409; audited replay preserves job_id; ready duplicate no-op
|
||||
PASS transient S3 outage: attachment=242a36c2-40f8-440f-899b-228164e5ddc4; attempts=5; durable retries, retry_exhausted, explicit replay after recovery
|
||||
PASS: lab5 grade4 recovery checks (all)
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:14:39Z
|
||||
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.06001993599988964",
|
||||
"event": "http.request.complete",
|
||||
"flags": "1",
|
||||
"method": "POST",
|
||||
"observed_timestamp": "1795251367084401557",
|
||||
"outcome": "success",
|
||||
"request_id": "observe-927e3831c3d241ac80d3d43391ad1ff4",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "81ebb23d65ec",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "10167513c3917aba",
|
||||
"status_code": "202",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251367084238848",
|
||||
"http.request.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 6425057,
|
||||
"linesProcessedPerSecond": 32614,
|
||||
"totalBytesProcessed": 86680,
|
||||
"totalLinesProcessed": 440,
|
||||
"execTime": 0.013491,
|
||||
"queueTime": 0.001181,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 77440
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 1,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 86680,
|
||||
"headChunkLines": 440,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 77440,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 190898,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,318 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"attempt": "1",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"duration_seconds": "1.1696733750000021",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795251373761214222",
|
||||
"outcome": "error",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "604b1094f4b8ea5a",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251373761101312",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"attempt": "2",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.04622686900006556",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795251405312162771",
|
||||
"outcome": "duplicate",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "860a8bda1f144a65878ee73824659b78",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "ba1e0fd901467175",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251405311928576",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"attempt": "2",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"event": "job.retry",
|
||||
"flags": "1",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"observed_timestamp": "1795251373753026344",
|
||||
"outcome": "retry",
|
||||
"queue": "attachments",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "604b1094f4b8ea5a",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251373752916480",
|
||||
"job.retry"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"attempt": "2",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "warn",
|
||||
"duration_seconds": "1.4581017409998367",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795251375868803562",
|
||||
"outcome": "error",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "fbfa8249e8c241d98e76ac1b870fab3e",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "13",
|
||||
"severity_text": "WARN",
|
||||
"span_id": "83adf2393f95e484",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251375868673792",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "8da6897d-7ad0-4a47-92a1-63e8820c6fb3",
|
||||
"attempt": "3",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.36952172699989205",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "c1561046-eb77-42b6-a915-e8f6e4c0a1bc",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795251497430982971",
|
||||
"outcome": "success",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "860a8bda1f144a65878ee73824659b78",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "ba826640e32b3505",
|
||||
"trace_id": "94af84b9ec1663416ac05526995fbd91"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251497430907392",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 142743,
|
||||
"linesProcessedPerSecond": 766,
|
||||
"totalBytesProcessed": 1117,
|
||||
"totalLinesProcessed": 6,
|
||||
"execTime": 0.007825,
|
||||
"queueTime": 0.000908,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 5,
|
||||
"splits": 0,
|
||||
"shards": 1,
|
||||
"totalPostFilterLines": 5,
|
||||
"totalStructuredMetadataBytesProcessed": 1048
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 1,
|
||||
"totalChunksMatched": 1,
|
||||
"totalBatches": 2,
|
||||
"totalLinesSent": 5,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 1117,
|
||||
"headChunkLines": 6,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 5,
|
||||
"headChunkStructuredMetadataBytes": 1048,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 103240,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"captured_at": "2026-11-21T09:17:48.990874Z",
|
||||
"host": "Windows, Intel Core Ultra 5 125U",
|
||||
"docker": {
|
||||
"ServerVersion": "29.3.0",
|
||||
"NCPU": 14,
|
||||
"MemTotal": 7965700096,
|
||||
"OperatingSystem": "Docker Desktop"
|
||||
},
|
||||
"revision_before_final_commit": "d0a062e2d2fbeda54c2a24f518eb3687650f7c9e",
|
||||
"working_tree_grade4": true,
|
||||
"api_url": "http://localhost:8086",
|
||||
"code_sha256": {
|
||||
".env.example": "f4d81ed6a4559e9f26c0c5a3d8accc495adb330c107c4f47cc34d47321a81dfe",
|
||||
".github/workflows/application.yml": "b486fe8a8299d363cf4e403dc2987de70c35a3ac6cc7e39f7656b6b77e6763ea",
|
||||
".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8",
|
||||
"app/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"app/advanced.py": "27762f863185cb6b51a64ae7698e60350b5aba0f924992e7eecaf58511d6e5cc",
|
||||
"app/attachments.py": "fae87bae0835fe6cafe8e85b685d1a1a0039d67516d52a633bac66e26c7fb8c1",
|
||||
"app/cache.py": "c7d5a210c4b85624ec089c2626cb69a84154790d325a6308cc8b0f578db3178b",
|
||||
"app/config.py": "0ee78d89095878150de4c195e28ffc17c44ab40f5e6b40fc6cb9ef23672d10aa",
|
||||
"app/db.py": "28eade9a5dda08887f9efeb786e52040832dd28975619a0c4b23988be2ba1ca3",
|
||||
"app/deps.py": "40b9353387289b7b3ff964a216446f13bbde66ee5a26bf5c847e39fe1948b062",
|
||||
"app/errors.py": "b06a0001aa2f1410cdcd4c385719ad3b90a0dacae62784649129c1e1c6dd933c",
|
||||
"app/image_worker.py": "66821f54709707fb8ad7d336eeceaa97cb69d6e6b1ae369147b59241a1e099f5",
|
||||
"app/images.py": "9e0cd83ef5d673356bde672d77ec106f8f3bb86812aaf46fbe3c932db2f7df54",
|
||||
"app/jobs.py": "b1b6dd46529042b72a088a61a3e153f6c8861d41550dbe713056be301dc5c3dc",
|
||||
"app/lifecycle.py": "097fcd74d178654fa91b3704d45c5215cfdec2f0ab3bb21b432e028160c9aa5e",
|
||||
"app/main.py": "fabcf21af59875defa0c130b03bf14df8db3ac9352df654dc3da4fcf38b84f50",
|
||||
"app/models.py": "69eb6bd83683c7ab1329e3d0d331577ce40612c156dc17c8ae1bdd231a921ecc",
|
||||
"app/operator.py": "d9046d7c757bc9c40721f11569f4ca1a7c9e52586ad05a04f0b6179b31b551b6",
|
||||
"app/pipeline.py": "3bcdd1484d4633c901a7bcfcd9f800db9f5e53db6d1c610d82d556d1337fbd4c",
|
||||
"app/repositories.py": "86b2774b89e2f7b875283cd684c7b346eeba1da2dd76d836661f6b45812ebcd8",
|
||||
"app/security.py": "b5e0fa73943a046501d1015d9b0288272b93e6a8458a47651c1bd9fa3b03a680",
|
||||
"app/server.py": "092af34b790f3103eb715f956c686655e18309cf6bab7073a015222f709d61b4",
|
||||
"app/services.py": "128a6668ad74697c05612fd9ad13107d14ba0ed0c9074338fb2d14437b54c785",
|
||||
"app/storage.py": "8573d43b3993c9f1e0e5ecbf82a9f65f58041e6bd50d4a315875d24b67042bd1",
|
||||
"app/telemetry.py": "232fde35c8445e3a80a067f41a1ac264fd41c662b3ae1ebbc3c7e661200fe9bc",
|
||||
"app/trace_context.py": "4ad1e98c346c6542df0bba808f0449922127ced410289b05ca2a3d94354b9740",
|
||||
"app/worker.py": "8eb05a2c37325a6d1da77102a683a19cc25fc7cb8adeeaa2b1c525a4029c7bbc",
|
||||
"compose.yaml": "a29a6af5228d9c2a1a529818071787690d3108d82cb83d7edc668ea0656fd969",
|
||||
"Dockerfile": "a0029707f843a15501a7d05142c06662b2705e16eb0292954148674cfe739b87",
|
||||
"infra/grafana/dashboards/messenger.json": "826c301adef62987d040a4f0d764d7f3dc89275de0c488979a03737ad73dbce9",
|
||||
"infra/grafana/provisioning/dashboards/dashboards.yaml": "e764b58e795886455758da0c93b1199e75d220bdf9e0c087f859a4ce67f744ef",
|
||||
"infra/grafana/provisioning/datasources/datasources.yaml": "8234d0c4b273127f3f5aad084f942dcc4dc498640f796bcc34feaae3a1eae1cb",
|
||||
"infra/loki/loki.yaml": "4614b904e6297165bebdafb4004e2c4ba93ea6bb278b4f53f196566621740dbd",
|
||||
"infra/otel/collector.yaml": "52fd1266cb78fd1e73bdc2a36057ec913abba6f50e7ce93bb3a31db69e25b8ba",
|
||||
"infra/prometheus/alerts.yaml": "252499c676f9fa9564c6466e7f268a9975c9044df0bdd82de78565f90548bbbe",
|
||||
"infra/prometheus/prometheus.yaml": "362d7f375ef6bb85fbd2e2e1e9c09c1a8507e94a3b115e13b0869ffcbc777d47",
|
||||
"infra/tempo/tempo.yaml": "a87b8a536ee4a42d8f83716c753b0380498ea17893082c1b7d1d21aafc21fc51",
|
||||
"migrations/001_initial_schema.sql": "e5c956278405d4f65be50b2e0b92c37f0870589e9c0d972c70d6733f1ae439e2",
|
||||
"migrations/002_indexes.sql": "6b0a375c09505f5220eb3add3ff3e4115b8426bbe1e1d97556effb8f64e4c7ff",
|
||||
"migrations/003_idempotency_keys.sql": "9554fd815d4b0e3085d2fe725c0e0e8b2a36e212fe3a486345e28afaf38e8b69",
|
||||
"migrations/004_attachments.sql": "2f421cbfb0cbf50d80d4bb8812378cdbc2619a5b8fa7aa5116238e287a8c6c6b",
|
||||
"migrations/005_async_pipeline.sql": "ce928ea8304c95a8bf2ad0ca92e6ffce81070362020de1a3373e381fd3dab247",
|
||||
"migrations/006_trace_context.sql": "a452982ba83fb0c270c54a3796d5c257bd86f74bdd8627474dcb3b022e33ad33",
|
||||
"requirements.txt": "4abdbc52e41c0ac0cfe5d91718558e5e9ee3951e27efe99d8c31a83c736f2924",
|
||||
"scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671",
|
||||
"scripts/migrate.py": "9f095a2181ed2a84584b09fe8cdfbf896c5c26306c7c5c5c262b4bbbb3f28d88",
|
||||
"scripts/observe_lab6.py": "bbfa84f0c360c538db68396afd97d228e272bd9ca68456c72a7a89eb77b91025",
|
||||
"scripts/s3_bootstrap.py": "f7bddb150df0e94e97a21efa2cad22c426e00fc935e710c16e11070f15344ef2",
|
||||
"scripts/test_lab5.py": "957546f3939f1d120d119d61a6df672034c0d234a32d014c413152490bec59ef",
|
||||
"scripts/test_lab6.py": "fd2ee01770b601c3fb0c89ea8e79c50d18a859e20a06c19f873ae38b1b6d06d0",
|
||||
"scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54",
|
||||
"student-tests/image_validation_unit.py": "531a2de31f90c92e094f7e5ef8bea663f2d70ecee41464a461e0e54d9800f795",
|
||||
"student-tests/lab5_helpers.py": "68a8a0cc9fc644d883342cb82642b2a679a32ed2db252f2e5a5b845d9ad2cec1",
|
||||
"student-tests/lab5_images.py": "048cc6dbb674869608c13ae290f20d57a8356da79a510f4517242bf57f5a505d",
|
||||
"student-tests/lab5_limits.py": "c2095797ce002e78a68e2d00afb2b10aa46a85787e8f1a43e6fb2b5a70045336",
|
||||
"student-tests/lab5_recovery.py": "8d7aa7de8dc09e18f206e32cac3001f4a916e9448e3887d0de7154aa832553d9",
|
||||
"student-tests/observability_unit.py": "9759b4ca7e312d656a2a0fb6f886a6dcb19ca50983a3f1733371142b24579d09"
|
||||
},
|
||||
"notes": "Collected before third commit; source hashes normalize CRLF to LF. Synthetic data only."
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools ps --status running --services
|
||||
api
|
||||
broker
|
||||
db
|
||||
grafana
|
||||
lb
|
||||
loki
|
||||
otel-collector
|
||||
prometheus
|
||||
publisher
|
||||
s3
|
||||
tempo
|
||||
valkey
|
||||
worker
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools build tester
|
||||
Image messenger-lab6-tester Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 587B 0.0s done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile.tester
|
||||
#2 transferring dockerfile: 405B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.11-alpine3.22
|
||||
#3 DONE 0.2s
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/docker:28.5.1-cli
|
||||
#4 DONE 0.2s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 286B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [stage-1 1/5] FROM docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
|
||||
#6 resolve docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 0.0s done
|
||||
#6 DONE 0.1s
|
||||
|
||||
#7 [docker-cli 1/1] FROM docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad
|
||||
#7 resolve docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad 0.0s done
|
||||
#7 DONE 0.1s
|
||||
|
||||
#8 [stage-1 2/5] RUN apk add --no-cache make
|
||||
#8 CACHED
|
||||
|
||||
#9 [stage-1 3/5] RUN pip install --no-cache-dir Pillow==11.0.0
|
||||
#9 CACHED
|
||||
|
||||
#10 [stage-1 4/5] COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||
#10 CACHED
|
||||
|
||||
#11 [stage-1 5/5] WORKDIR /work
|
||||
#11 CACHED
|
||||
|
||||
#12 exporting to image
|
||||
#12 exporting layers done
|
||||
#12 exporting manifest sha256:1c93bba11e9ded8b819bbcadafd9217f8b6b4fc5301deeca1e720c49a45f1b2e done
|
||||
#12 exporting config sha256:9ad2daad56b9cfccc6c03e7be0468faf51994e1604e86b502077a03f28844325 done
|
||||
#12 exporting attestation manifest sha256:96fa2816d4a538ed8e6e3ea5c702033990470ee27e842b91c5f2979f0304e95a 0.1s done
|
||||
#12 exporting manifest list sha256:3e586fe0bf4aeb3814ad182a780c6173a9e252da1c33e7e61f20187228c8b23f 0.0s done
|
||||
#12 naming to docker.io/library/messenger-lab6-tester:latest
|
||||
#12 naming to docker.io/library/messenger-lab6-tester:latest done
|
||||
#12 unpacking to docker.io/library/messenger-lab6-tester:latest 0.0s done
|
||||
#12 DONE 0.2s
|
||||
|
||||
#13 resolving provenance for metadata file
|
||||
#13 DONE 0.0s
|
||||
Image messenger-lab6-tester Built
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:42Z
|
||||
|
||||
UTC: 2026-11-21T09:10:08Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
Container messenger-lab6-tester-run-100a83886eb9 Creating
|
||||
Container messenger-lab6-tester-run-100a83886eb9 Created
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:10:09Z
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:46Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/smoke.py
|
||||
Container messenger-lab6-tester-run-42a477b98467 Creating
|
||||
Container messenger-lab6-tester-run-42a477b98467 Created
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:53Z
|
||||
@@ -0,0 +1,17 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 42 minutes ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 42 minutes ago Up 2 minutes (healthy) 5432/tcp
|
||||
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 27 minutes ago Up 26 minutes 127.0.0.1:3000->3000/tcp
|
||||
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 26 minutes ago Up 2 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
|
||||
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 27 minutes ago Up 16 minutes 127.0.0.1:3100->3100/tcp
|
||||
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate About a minute ago Exited (0) About a minute ago
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 29 minutes ago Up 28 minutes 4317/tcp, 55678-55679/tcp
|
||||
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 27 minutes ago Up 26 minutes 127.0.0.1:9090->9090/tcp
|
||||
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher About a minute ago Up About a minute 8080/tcp
|
||||
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 42 minutes ago Up 2 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init About a minute ago Exited (0) About a minute ago
|
||||
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo About a minute ago Up About a minute 127.0.0.1:3200->3200/tcp
|
||||
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 42 minutes ago Up 2 minutes (healthy) 6379/tcp
|
||||
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker About a minute ago Up About a minute 8080/tcp
|
||||
@@ -0,0 +1,44 @@
|
||||
|
||||
UTC: 2026-11-21T09:09:40Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab6-api-1 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
|
||||
messenger-lab6-api-2 messenger-lab6-api:dev "python -m app.server" api 18 minutes ago Up 17 minutes (healthy) 8080/tcp
|
||||
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 34 minutes ago Up 24 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 34 minutes ago Up 10 minutes (healthy) 5432/tcp
|
||||
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 19 minutes ago Up 18 minutes 127.0.0.1:3000->3000/tcp
|
||||
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 17 minutes ago Up 17 minutes (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
|
||||
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 19 minutes ago Up 8 minutes 127.0.0.1:3100->3100/tcp
|
||||
messenger-lab6-migrate-1 messenger-lab6-api:dev "python scripts/migr…" migrate 18 minutes ago Exited (0) 16 minutes ago
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 20 minutes ago Up 20 minutes 4317/tcp, 55678-55679/tcp
|
||||
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 19 minutes ago Up 18 minutes 127.0.0.1:9090->9090/tcp
|
||||
messenger-lab6-publisher-1 messenger-lab6-api:dev "python -m app.jobs" publisher 18 minutes ago Up 17 minutes 8080/tcp
|
||||
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 34 minutes ago Up 13 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab6-s3-init-1 messenger-lab6-api:dev "python scripts/s3_b…" s3-init 18 minutes ago Exited (0) 16 minutes ago
|
||||
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 3 minutes ago Up 3 minutes 127.0.0.1:3200->3200/tcp
|
||||
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 34 minutes ago Up 24 minutes (healthy) 6379/tcp
|
||||
messenger-lab6-worker-1 messenger-lab6-api:dev "python -m app.worker" worker 18 minutes ago Up 5 minutes 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:09:40Z
|
||||
|
||||
UTC: 2026-11-21T09:16:01Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab6-api-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 7 seconds (healthy) 8080/tcp
|
||||
messenger-lab6-api-2 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.server" api 24 minutes ago Up 43 seconds (healthy) 8080/tcp
|
||||
messenger-lab6-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 40 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab6-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 40 minutes ago Up About a minute (healthy) 5432/tcp
|
||||
messenger-lab6-grafana-1 grafana/grafana:11.2.2 "/run.sh" grafana 25 minutes ago Up 24 minutes 127.0.0.1:3000->3000/tcp
|
||||
messenger-lab6-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 24 minutes ago Up 36 seconds (healthy) 127.0.0.1:8086->8080/tcp, 127.0.0.1:8406->8404/tcp
|
||||
messenger-lab6-loki-1 grafana/loki:3.2.1 "/usr/bin/loki -conf…" loki 25 minutes ago Up 14 minutes 127.0.0.1:3100->3100/tcp
|
||||
messenger-lab6-migrate-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/migr…" migrate 24 minutes ago Exited (0) 23 minutes ago
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.111.0 "/otelcol-contrib --…" otel-collector 27 minutes ago Up 26 minutes 4317/tcp, 55678-55679/tcp
|
||||
messenger-lab6-prometheus-1 prom/prometheus:v2.55.0 "/bin/prometheus --c…" prometheus 25 minutes ago Up 24 minutes 127.0.0.1:9090->9090/tcp
|
||||
messenger-lab6-publisher-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.jobs" publisher 24 minutes ago Up 44 seconds 8080/tcp
|
||||
messenger-lab6-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 40 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab6-s3-init-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python scripts/s3_b…" s3-init 24 minutes ago Exited (0) 22 minutes ago
|
||||
messenger-lab6-tempo-1 grafana/tempo:2.6.1 "/tempo -config.file…" tempo 9 minutes ago Up 9 minutes 127.0.0.1:3200->3200/tcp
|
||||
messenger-lab6-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 40 minutes ago Up About a minute (healthy) 6379/tcp
|
||||
messenger-lab6-worker-1 sha256:efa9adc78d01ce2a8b6b72223b3964036998ac290fd83f91c929d417c137e0a4 "python -m app.worker" worker 24 minutes ago Up 44 seconds 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:16:01Z
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.05064601900016896",
|
||||
"event": "http.request.complete",
|
||||
"flags": "1",
|
||||
"method": "POST",
|
||||
"observed_timestamp": "1795251728469179436",
|
||||
"outcome": "success",
|
||||
"request_id": "observe-81adf6fa6db94a2caf9dbdf96d62b259",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "0d13c5814dab",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "0ff19193d60d0a11",
|
||||
"status_code": "202",
|
||||
"trace_id": "167d5623ef64fb0ed646ec1e2fe01ad8"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251728468912896",
|
||||
"http.request.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 87014288,
|
||||
"linesProcessedPerSecond": 188878,
|
||||
"totalBytesProcessed": 1367324,
|
||||
"totalLinesProcessed": 2968,
|
||||
"execTime": 0.015714,
|
||||
"queueTime": 0.000907,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 2,
|
||||
"shards": 2,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 1251756
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 2,
|
||||
"totalChunksMatched": 2,
|
||||
"totalBatches": 3,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 60282,
|
||||
"headChunkLines": 306,
|
||||
"decompressedBytes": 1307042,
|
||||
"decompressedLines": 2662,
|
||||
"compressedBytes": 58630,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 53856,
|
||||
"decompressedStructuredMetadataBytes": 1197900
|
||||
},
|
||||
"chunkRefsFetchTime": 386158,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "streams",
|
||||
"result": [
|
||||
{
|
||||
"stream": {
|
||||
"attachment_id": "567d9b19-91b2-468c-95bb-8317c6ef90f2",
|
||||
"attempt": "1",
|
||||
"code_filepath": "/app/app/telemetry.py",
|
||||
"code_function": "emit",
|
||||
"code_lineno": "180",
|
||||
"deployment_environment_name": "lab",
|
||||
"detected_level": "info",
|
||||
"duration_seconds": "0.4046185140000489",
|
||||
"event": "job.complete",
|
||||
"flags": "1",
|
||||
"job_id": "2c0f93ca-74a0-4191-9b63-85fd07008e86",
|
||||
"job_type": "attachment.process.v1",
|
||||
"observed_timestamp": "1795251729011236193",
|
||||
"outcome": "success",
|
||||
"scope_name": "messenger.telemetry.events",
|
||||
"service_instance_id": "b5902353dfd648c39ac966e0780e90a3",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"severity_number": "9",
|
||||
"severity_text": "INFO",
|
||||
"span_id": "59e8500a32f934ab",
|
||||
"trace_id": "167d5623ef64fb0ed646ec1e2fe01ad8"
|
||||
},
|
||||
"values": [
|
||||
[
|
||||
"1795251729011139328",
|
||||
"job.complete"
|
||||
]
|
||||
]
|
||||
}
|
||||
],
|
||||
"stats": {
|
||||
"summary": {
|
||||
"bytesProcessedPerSecond": 197625,
|
||||
"linesProcessedPerSecond": 1051,
|
||||
"totalBytesProcessed": 1128,
|
||||
"totalLinesProcessed": 6,
|
||||
"execTime": 0.005708,
|
||||
"queueTime": 0.001056,
|
||||
"subqueries": 0,
|
||||
"totalEntriesReturned": 1,
|
||||
"splits": 2,
|
||||
"shards": 2,
|
||||
"totalPostFilterLines": 1,
|
||||
"totalStructuredMetadataBytesProcessed": 1056
|
||||
},
|
||||
"querier": {
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": false,
|
||||
"chunk": {
|
||||
"headChunkBytes": 0,
|
||||
"headChunkLines": 0,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 0,
|
||||
"headChunkStructuredMetadataBytes": 0,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 0,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"ingester": {
|
||||
"totalReached": 2,
|
||||
"totalChunksMatched": 2,
|
||||
"totalBatches": 3,
|
||||
"totalLinesSent": 1,
|
||||
"store": {
|
||||
"totalChunksRef": 0,
|
||||
"totalChunksDownloaded": 0,
|
||||
"chunksDownloadTime": 0,
|
||||
"queryReferencedStructuredMetadata": true,
|
||||
"chunk": {
|
||||
"headChunkBytes": 1128,
|
||||
"headChunkLines": 6,
|
||||
"decompressedBytes": 0,
|
||||
"decompressedLines": 0,
|
||||
"compressedBytes": 0,
|
||||
"totalDuplicates": 0,
|
||||
"postFilterLines": 1,
|
||||
"headChunkStructuredMetadataBytes": 1056,
|
||||
"decompressedStructuredMetadataBytes": 0
|
||||
},
|
||||
"chunkRefsFetchTime": 227539,
|
||||
"congestionControlLatency": 0,
|
||||
"pipelineWrapperFilteredLines": 0
|
||||
}
|
||||
},
|
||||
"cache": {
|
||||
"chunk": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"index": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"result": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"statsResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"volumeResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"seriesResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"labelResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
},
|
||||
"instantMetricResult": {
|
||||
"entriesFound": 0,
|
||||
"entriesRequested": 0,
|
||||
"entriesStored": 0,
|
||||
"bytesReceived": 0,
|
||||
"bytesSent": 0,
|
||||
"requests": 0,
|
||||
"downloadTime": 0,
|
||||
"queryLengthServed": 0
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"totalChunks": 0,
|
||||
"postFilterChunks": 0,
|
||||
"shardsDuration": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,20 @@
|
||||
|
||||
UTC: 2026-11-21T09:03:11Z
|
||||
Command: C:\Users\masha\AppData\Local\Programs\Python\Python313\python.exe -X utf8 -c "import json
|
||||
import scripts.observe_lab6 as m
|
||||
client,stand=m.Business(),m.Stand()
|
||||
try:
|
||||
m.wait_backends();client.bootstrap()
|
||||
result=m.lag_incident(client,stand,'MessengerQueueLag')
|
||||
path=m.EVIDENCE/'observability-incidents.json'
|
||||
data=json.loads(path.read_text(encoding='utf-8'))
|
||||
data['incidents'][0]=result
|
||||
data['captured_at']=m.now()
|
||||
m.save('observability-incidents.json',data)
|
||||
print('PASS: clean queue and inactive alert -> newly firing -> resolved; original bytes verified')
|
||||
finally:
|
||||
stand.restore()
|
||||
"
|
||||
PASS: clean queue and inactive alert -> newly firing -> resolved; original bytes verified
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-21T09:04:11Z
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,42 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: '^trace_id$'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: false
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{$${__tags}} | trace_id="$${__span.traceId}"'
|
||||
tags: [{ key: service.name, value: service_name }]
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,102 @@
|
||||
# HAProxy 3.0 — единственная публичная точка входа (порт 8080).
|
||||
# Backend-адреса берутся из встроенного DNS Docker: имя сервиса `api` возвращает A-запись
|
||||
# каждого работающего контейнера-реплики. Реплики не публикуют порты наружу.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
|
||||
# Встроенный DNS Docker. hold valid 2s — как часто перечитывается список реплик
|
||||
# (новые контейнеры при scale/rolling update появляются в пуле за ≤ 2–3 s).
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold nx 2s
|
||||
hold timeout 2s
|
||||
hold obsolete 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
# Таймауты согласованы с приложением:
|
||||
# connect 1s — недоступная реплика быстро даёт ошибку соединения -> retry на другую;
|
||||
# server 10s — больше statement_timeout БД (5 s) + ожидания пула (3 s);
|
||||
# http-keep-alive 10s < timeout_keep_alive uvicorn (30 s): LB закрывает простой первым.
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s
|
||||
timeout queue 5s
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend http_in
|
||||
bind :8080
|
||||
# X-Request-Id создаётся на входе (если клиент не прислал корректный), передаётся в API
|
||||
# и попадает в лог LB — один ID в логах LB и реплики.
|
||||
unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
|
||||
acl has_valid_rid req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$
|
||||
http-request set-header X-Request-Id %[unique-id] unless has_valid_rid
|
||||
http-request set-var(txn.rid) req.hdr(X-Request-Id)
|
||||
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.rid)] \"%HM %HPO\""
|
||||
default_backend api
|
||||
|
||||
# Ошибки, созданные самим LB (нет живых реплик / timeout), — в формате контракта.
|
||||
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Реплика API не ответила вовремя\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
|
||||
backend api
|
||||
# Обычный round-robin без sticky session: корректность обеспечивает общее состояние в PostgreSQL.
|
||||
balance roundrobin
|
||||
|
||||
# Активная проверка готовности каждые 1 s; 2 неудачи подряд -> DOWN, 2 успеха -> UP.
|
||||
# Максимальное время обнаружения отказа «процесс жив, но не готов»:
|
||||
# inter × fall + timeout check = 1 s × 2 + 1 s ≈ 3 s.
|
||||
# Упавший процесс (connection refused) обнаруживается быстрее: observe layer4 + error-limit 1
|
||||
# переводит сервер в DOWN по первой реальной ошибке соединения, а запрос уходит на другую реплику.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker init-addr none resolve-prefer ipv4 check
|
||||
timeout check 1s
|
||||
|
||||
# Повторы на другую реплику:
|
||||
# - ошибка установления соединения повторяется всегда (запрос не дошёл до приложения);
|
||||
# - L7-повторы (пустой ответ / 503 от дренирующейся реплики) разрешены только для
|
||||
# безопасных GET и для POST сообщения С Idempotency-Key — его повтор не создаст дубль.
|
||||
# POST /users, /chats, /auth/sessions после отправки не повторяются (нет ключа идемпотентности).
|
||||
retry-on conn-failure empty-response 503
|
||||
acl is_message_post path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
acl has_idem_key req.hdr(Idempotency-Key) -m found
|
||||
http-request disable-l7-retry if METH_POST !is_message_post
|
||||
http-request disable-l7-retry if METH_POST !has_idem_key
|
||||
http-request disable-l7-retry if METH_DELETE
|
||||
# Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор
|
||||
# только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся.
|
||||
http-request disable-l7-retry if { path_beg /api/v1/attachments/ }
|
||||
# Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном
|
||||
# 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало.
|
||||
http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ }
|
||||
|
||||
# До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес.
|
||||
server-template api 8 api:8080
|
||||
|
||||
# Служебная страница состояния backend-ов (UP/DOWN, счётчики ошибок/повторов).
|
||||
# В compose.yaml публикуется только на 127.0.0.1 хоста — не часть публичного API.
|
||||
frontend stats
|
||||
bind :8404
|
||||
http-request return status 200 content-type text/plain string "ok" if { path /lb-health }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1,39 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
# Bound cardinality: request IDs / trace IDs remain structured metadata.
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
+16
-4
@@ -1,7 +1,19 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Наблюдаемость lab6
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`API / publisher / worker → OTLP HTTP → Collector → Prometheus / Loki / Tempo → Grafana`.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
* [Collector](collector.yaml): OTLP receivers, memory limiter, bounded batch,
|
||||
sending queues и retry. Collector доступен только во внутренней сети Compose.
|
||||
* [Prometheus](../prometheus/prometheus.yaml) хранит метрики приложения и самого
|
||||
Collector; [alerts](../prometheus/alerts.yaml) оцениваются каждые 5 секунд.
|
||||
* [Loki](../loki/loki.yaml) принимает native OTLP logs; технические ID —
|
||||
structured metadata, не индексные labels.
|
||||
* [Tempo](../tempo/tempo.yaml) хранит реальные API/producer/consumer spans.
|
||||
* [Grafana](../grafana/provisioning/datasources/datasources.yaml) подключает
|
||||
три datasource; [dashboard](../grafana/dashboards/messenger.json) импортируется
|
||||
provisioning автоматически. Backend ports и Grafana опубликованы на loopback.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
Сценарии диагностики и точные SLI/SLO: [RUNBOOK](RUNBOOK.md).
|
||||
Запуск, семантика метрик, результаты и ограничения: [отчёт](../../REPORT.md).
|
||||
Воспроизводимая проверка с evidence: `python -X utf8 scripts/test_lab6.py --incidents`.
|
||||
Подробные сценарии находятся в `scripts/observe_lab6.py`.
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# Диагностика стенда lab6
|
||||
|
||||
Правила предназначены для локального учебного стенда и синтетической нагрузки.
|
||||
Пороги 15 секунд и `for: 10s` сокращены для демонстрации, а не выведены из
|
||||
производственной статистики. Alert оценивается Prometheus каждые 5 секунд.
|
||||
Firing и последующее отсутствие активного alert сохраняет сценарий проверки;
|
||||
внешняя доставка уведомлений не требуется.
|
||||
|
||||
## SLI и SLO
|
||||
|
||||
- Доступность API: доля бизнес-ответов с кодом ниже 500 среди всех
|
||||
бизнес-ответов. `/health/*` исключается, 4xx считаются доступным сервисом.
|
||||
Учебный SLO: не менее 99% в скользящем окне 5 минут.
|
||||
- Готовность вложения: доля завершений `ready` с временем не более 60 секунд
|
||||
от устойчивого принятия до COMMIT `ready`. SLO: не менее 95% в окне 5 минут.
|
||||
Доля вычисляется по `messenger_attachment_time_to_ready_seconds_bucket{le="60"}`
|
||||
и `_count`; незавершённые и failed задачи отдельно видны в backlog/errors.
|
||||
Поэтому этот SLI условен по успешным завершениям и один не доказывает,
|
||||
что 95% всех принятых загрузок достигают ready. При остановленном worker
|
||||
отсутствие завершений не интерпретируется как выполненный SLO.
|
||||
- Percentile считается `histogram_quantile` после `sum by (le)` bucket rate
|
||||
всех нужных реплик. Средние p95 отдельных реплик не усредняются.
|
||||
|
||||
## API errors
|
||||
|
||||
Ссылка alert: `MessengerHighErrorRate`.
|
||||
|
||||
1. Открыть Grafana `http://localhost:3000`, dashboard **Messenger lab6**.
|
||||
Проверить интервал инцидента, RPS и 5xx ratio. Ошибки входа 401 и
|
||||
недоступность PostgreSQL 503 отвечают на разные вопросы.
|
||||
2. Выбрать ошибочный API trace из Tempo и связанный структурный лог по
|
||||
`trace_id`; `request_id` связывает лог с `X-Request-Id` ответа.
|
||||
3. Найти DB span: ошибка соединения или большая длительность DB при
|
||||
коротком handler указывает на зависимость. Сравнить pool used/waiting
|
||||
и DB operation duration. В spans нет текста SQL и параметров.
|
||||
Если DB spans быстрые, искать время между ними/в handler, S3 или
|
||||
ожидание свободного ресурса; одно значение p95 причины не устанавливает.
|
||||
4. Проверить `docker compose ps db api`. В контролируемом сценарии
|
||||
`docker compose stop db` ожидаются отказы синхронного пути и readiness.
|
||||
HAProxy может вернуть собственный 503, не попадающий в счётчик API:
|
||||
сценарий диагностики должен сохранять также прямой ответ API.
|
||||
5. Восстановить `docker compose start db`, дождаться API ready, повторить
|
||||
запросы. В Prometheus `http://localhost:9090/alerts` alert сначала
|
||||
становится inactive после выхода ошибок из окна 1 минуты.
|
||||
|
||||
Сохранять ответ без токена, trace JSON, лог, запросы метрик и состояние alert
|
||||
до воздействия, во время и после. Не считать текст ошибки DB в stdout
|
||||
единственным доказательством причины: нужны сохранённые сигналы приложения.
|
||||
|
||||
## Queue lag
|
||||
|
||||
Ссылка alert: `MessengerQueueLag`.
|
||||
|
||||
1. Сравнить queue depth, oldest age, свежесть наблюдения, busy/capacity worker,
|
||||
успешные/ошибочные jobs и длительность попыток. Backlog измерен по
|
||||
устойчивым незавершённым заданиям PostgreSQL, включая outbox и retry;
|
||||
это не только число сообщений ready в RabbitMQ.
|
||||
2. Если depth > 0 и возраст растёт при отсутствии завершений и busy=0,
|
||||
проверить `docker compose ps worker publisher broker`. Остановка worker
|
||||
не останавливает publisher-наблюдатель.
|
||||
3. Если busy=capacity, искать медленные DB/S3/image spans в worker trace.
|
||||
Если растут retry/errors, найти соответствующий consumer span и событие
|
||||
retry. По traceparent пройти producer → consumer → DB/S3, включая попытки.
|
||||
4. Контролируемое воздействие: `docker compose stop worker`, затем несколько
|
||||
upload стандартного fixture. Ответ 202 означает устойчивое принятие;
|
||||
рост oldest age и firing ожидаются через 15 секунд плюс `for` и интервалы
|
||||
сбора/оценки. Счётчик duration не заменяет возраст ожидающей задачи.
|
||||
5. Восстановить `docker compose start worker`. Дождаться ready вложений,
|
||||
depth=0 и inactive alert. Скачать original и thumbnail, проверить байты
|
||||
и размеры. Уменьшение backlog само по себе ещё не доказывает сохранность.
|
||||
|
||||
Sample age защищает от интерпретации устаревшей очереди как свежей. Если DB
|
||||
недоступна и sample age растёт, сначала расследовать DB; автоматически
|
||||
приписывать это остановленному worker нельзя.
|
||||
|
||||
## Недоступен backend telemetry
|
||||
|
||||
1. Перед воздействием сохранить Collector exporter queue/failed counters,
|
||||
память API/worker/Collector и контрольное сообщение/вложение.
|
||||
2. `docker compose stop tempo loki` отключает хранилища traces/logs.
|
||||
Prometheus остаётся включённым и сохраняет self metrics Collector:
|
||||
очередь, ошибки отправки и память. Отдельно проверить недоступность
|
||||
Collector, если нужно исследовать именно переполнение SDK.
|
||||
3. Продолжать бизнес-запросы и загрузки. Проверить 201 сообщений, 202 → ready
|
||||
вложений и чтение данных. Telemetry отсутствует в readiness; зависимость
|
||||
от неё не должна отбирать все API из балансировки.
|
||||
4. Collector: memory limiter 192 MiB, spike 48 MiB, batch максимум 512,
|
||||
sending queue максимум 256 batches на exporter, 2 consumers; timeout 3s,
|
||||
retry не более 30s на batch. После исчерпания retries данные отбрасываются.
|
||||
SDK также имеет конечные очереди и timeout; экспорт выполняется фоново.
|
||||
Ограничения RAM контейнеров — дополнительный предел, а не замена очередям.
|
||||
5. `docker compose start tempo loki`; дождаться backend ready и повторить
|
||||
контрольный запрос. Новые trace/log должны появиться. Часть старых
|
||||
сигналов может потеряться: сохранённый рост failed/dropped counters и
|
||||
границы измерения описываются в отчёте. Нельзя обещать полное восстановление
|
||||
telemetry после перезапуска Collector: его очереди находятся в памяти.
|
||||
|
||||
Команды `docker compose down` сохраняют volumes. `down -v` удаляет учебные
|
||||
данные и историю, для восстановления после инцидента эта команда не нужна.
|
||||
@@ -0,0 +1,69 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 5m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/loki:
|
||||
# otlphttp appends /v1/logs; Loki 3.2 accepts native OTLP here.
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
address: 0.0.0.0:8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
@@ -0,0 +1,32 @@
|
||||
groups:
|
||||
- name: messenger-lab6
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*",status_class="5xx"}[1m]))
|
||||
/
|
||||
clamp_min(sum(rate(messenger_http_requests_total{service_name="messenger-api",route!~"/health/.*"}[1m])), 1e-9)
|
||||
) > 0.05
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Business API 5xx ratio exceeds 5%"
|
||||
description: "5xx / all business HTTP responses over 1 minute, continuously above 5% for 10 seconds. Health and client 4xx are not failures in this SLI."
|
||||
runbook: "infra/otel/RUNBOOK.md#api-errors"
|
||||
- alert: MessengerQueueLag
|
||||
expr: |
|
||||
max(messenger_queue_oldest_age_seconds{service_name="messenger-publisher"}) > 15
|
||||
and
|
||||
max(messenger_queue_depth{service_name="messenger-publisher"}) > 0
|
||||
and
|
||||
max(messenger_queue_sample_age_seconds{service_name="messenger-publisher"}) < 10
|
||||
for: 10s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Oldest unfinished attachment job exceeds 15 seconds"
|
||||
description: "Durable PostgreSQL backlog is nonempty and oldest job age is above 15 seconds for 10 seconds; the observer sample is less than 10 seconds old. This is queue age, not worker processing duration."
|
||||
runbook: "infra/otel/RUNBOOK.md#queue-lag"
|
||||
@@ -0,0 +1,17 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: [localhost:9090]
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{ "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" }
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{ "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" }
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"Write:messenger-attachments",
|
||||
"List:messenger-attachments"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand.
|
||||
#
|
||||
# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the
|
||||
# container only (-ip.bind): their HTTP API has no authentication and must
|
||||
# not be reachable from the Docker network. Only the S3 gateway (9000)
|
||||
# listens on all interfaces, and it requires SigV4 credentials.
|
||||
# - Identities and rights come from s3.json.template; the secret values are
|
||||
# substituted at start from Docker secrets into a file readable by root
|
||||
# only (never stored in Git or in the image).
|
||||
set -eu
|
||||
umask 077
|
||||
s() { cat "/run/secrets/$1"; }
|
||||
sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \
|
||||
-e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \
|
||||
-e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \
|
||||
-e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \
|
||||
/opt/lab/s3.json.template > /tmp/s3.json
|
||||
|
||||
exec weed -logtostderr=true server \
|
||||
-dir=/data \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.volumeSizeLimitMB=256 -volume.max=0 \
|
||||
-s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json
|
||||
@@ -0,0 +1,25 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
ingester:
|
||||
trace_idle_period: 5s
|
||||
max_block_duration: 1m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,48 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT users_username_format_check
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
|
||||
CONSTRAINT users_display_name_nonblank_check
|
||||
CHECK (char_length(btrim(display_name)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT chats_title_nonblank_check
|
||||
CHECK (char_length(btrim(title)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text VARCHAR(4000) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT messages_text_nonblank_check
|
||||
CHECK (char_length(btrim(text)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash CHAR(64) NOT NULL UNIQUE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE INDEX IF NOT EXISTS messages_keyset_idx
|
||||
ON messages (chat_id, created_at ASC, id ASC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chats_keyset_idx
|
||||
ON chats (created_at ASC, id ASC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
|
||||
ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sessions_active_token_idx
|
||||
ON sessions (token_hash, expires_at)
|
||||
WHERE revoked_at IS NULL;
|
||||
@@ -0,0 +1,18 @@
|
||||
-- Lab3: shared idempotency of POST /chats/{chat_id}/messages.
|
||||
-- Scope: user + method + chat path + key. The key row and the message are
|
||||
-- written in one transaction, so either both exist or neither does.
|
||||
-- Expand-only change: the running lab2/lab3 API keeps working with it.
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
method VARCHAR(8) NOT NULL,
|
||||
key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
message_id UUID REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, method, key)
|
||||
);
|
||||
|
||||
-- For a periodic cleanup of keys older than IDEMPOTENCY_TTL_SECONDS.
|
||||
CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,68 @@
|
||||
-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private
|
||||
-- S3 bucket under a server-generated object_key. Expand-only migration: the
|
||||
-- lab3 API keeps working on this schema.
|
||||
|
||||
-- Text may be empty when the message has attachments; "not blank without
|
||||
-- attachments" is checked by the API (one message + its links are inserted
|
||||
-- in one transaction).
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
-- Display name only; never used as a path or an S3 key.
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
-- Lab4 has only "ready": the row is inserted after the object is stored.
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
error_code VARCHAR(64),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT attachments_content_type_check
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size_check
|
||||
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256_check
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('ready'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
-- A message references 0-10 attachments of its own chat; "position" keeps
|
||||
-- the order of attachment_ids. One attachment may be referenced by several
|
||||
-- messages of the chat.
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
position SMALLINT NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
|
||||
-- Upload intent: committed BEFORE the S3 PUT and deleted in the same
|
||||
-- transaction that inserts the attachment row. An intent older than the
|
||||
-- grace period therefore means "the object may exist in S3, but the request
|
||||
-- never committed" (crash, lost DB, client gone) -> the janitor deletes the
|
||||
-- object and then the intent.
|
||||
CREATE TABLE IF NOT EXISTS upload_intents (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS upload_intents_created_idx
|
||||
ON upload_intents (created_at);
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Lab5: the database owns the work intent; RabbitMQ carries only its IDs.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key TEXT,
|
||||
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
|
||||
ADD COLUMN job_id UUID UNIQUE,
|
||||
ADD COLUMN attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
ADD COLUMN attempts_run INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN lease_token UUID,
|
||||
ADD COLUMN lease_until TIMESTAMPTZ,
|
||||
ADD COLUMN next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
ADD COLUMN staging_deleted BOOLEAN NOT NULL DEFAULT false,
|
||||
ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX outbox_pending_idx ON outbox(available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX attachments_lease_idx ON attachments(lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachments_staging_idx ON attachments(updated_at)
|
||||
WHERE staging_key IS NOT NULL AND NOT staging_deleted;
|
||||
|
||||
-- Failed registry = attachments WHERE status='failed'. This append-only
|
||||
-- journal explains retry, lease recovery and explicit operator replay.
|
||||
CREATE TABLE job_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
event TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_attachment_idx ON job_events(attachment_id, id);
|
||||
|
||||
-- Internal one-shot crash injection, never exposed through public HTTP.
|
||||
CREATE TABLE fault_injections (
|
||||
point TEXT PRIMARY KEY,
|
||||
remaining INTEGER NOT NULL DEFAULT 0 CHECK (remaining >= 0),
|
||||
hits INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- W3C context is part of the durable work intent, not an in-memory cache.
|
||||
-- Retry attempts retain the latest causal span; business payload is unchanged.
|
||||
ALTER TABLE attachments ADD COLUMN traceparent TEXT, ADD COLUMN tracestate TEXT;
|
||||
ALTER TABLE outbox ADD COLUMN traceparent TEXT, ADD COLUMN tracestate TEXT;
|
||||
@@ -0,0 +1,20 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
pydantic==2.9.2
|
||||
SQLAlchemy==2.0.36
|
||||
asyncpg==0.30.0
|
||||
argon2-cffi==23.1.0
|
||||
redis==5.0.8
|
||||
# Lab4: S3 client (async) and image header/JPEG decoding
|
||||
aiobotocore==2.15.2
|
||||
botocore==1.35.36
|
||||
aiohttp==3.10.10
|
||||
Pillow==11.0.0
|
||||
|
||||
# Lab5: durable RabbitMQ messages and publisher confirmations.
|
||||
aio-pika==9.4.3
|
||||
|
||||
# Explicit OTLP/HTTP SDK; semantic conventions bundled as 0.49b2.
|
||||
opentelemetry-api==1.28.2
|
||||
opentelemetry-sdk==1.28.2
|
||||
opentelemetry-exporter-otlp-proto-http==1.28.2
|
||||
@@ -0,0 +1,32 @@
|
||||
# Creates the same local secret files as init-secrets.sh on Windows.
|
||||
# Existing values are preserved and never printed.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$secretDirectory = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../infra/secrets'))
|
||||
[IO.Directory]::CreateDirectory($secretDirectory) | Out-Null
|
||||
function New-RandomHex {
|
||||
$bytes = New-Object byte[] 24
|
||||
$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $randomGenerator.GetBytes($bytes) } finally { $randomGenerator.Dispose() }
|
||||
return ([BitConverter]::ToString($bytes)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
function Write-NewSecret([string]$Name, [string]$Value) {
|
||||
$secretPath = Join-Path $secretDirectory $Name
|
||||
if (Test-Path -LiteralPath $secretPath) { Write-Output "keep infra/secrets/$Name" }
|
||||
else {
|
||||
[IO.File]::WriteAllText($secretPath, $Value, (New-Object Text.UTF8Encoding $false))
|
||||
Write-Output "create infra/secrets/$Name"
|
||||
}
|
||||
}
|
||||
Write-NewSecret 'postgres_password.txt' (New-RandomHex)
|
||||
$databasePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'postgres_password.txt')).Trim()
|
||||
Write-NewSecret 'database_url.txt' "postgresql://messenger:${databasePassword}@db:5432/messenger"
|
||||
Write-NewSecret 'valkey_password.txt' (New-RandomHex)
|
||||
$cachePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'valkey_password.txt')).Trim()
|
||||
Write-NewSecret 'cache_url.txt' "redis://:${cachePassword}@valkey:6379/0"
|
||||
Write-NewSecret 's3_admin_access_key_id.txt' ('admin' + (New-RandomHex).Substring(0,15))
|
||||
Write-NewSecret 's3_admin_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 's3_access_key_id.txt' ('api' + (New-RandomHex).Substring(0,17))
|
||||
Write-NewSecret 's3_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 'broker_password.txt' (New-RandomHex)
|
||||
$queuePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'broker_password.txt')).Trim()
|
||||
Write-NewSecret 'broker_url.txt' "amqp://messenger:${queuePassword}@broker:5672/"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
# Creates local Docker secrets for a synthetic lab stand.
|
||||
# Existing files are never overwritten; values are never printed.
|
||||
set -eu
|
||||
|
||||
dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets"
|
||||
mkdir -p "$dir"
|
||||
# Host access is limited by the directory mode; the files themselves must stay
|
||||
# readable by the non-root users inside containers (bind-mounted secrets).
|
||||
chmod 700 "$dir" 2>/dev/null || true
|
||||
|
||||
random() {
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
|
||||
}
|
||||
|
||||
write_new() {
|
||||
if [ -f "$dir/$1" ]; then
|
||||
echo "keep infra/secrets/$1"
|
||||
else
|
||||
printf '%s' "$2" > "$dir/$1"
|
||||
chmod 644 "$dir/$1" 2>/dev/null || true
|
||||
echo "create infra/secrets/$1"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ ! -f "$dir/postgres_password.txt" ]; then
|
||||
write_new postgres_password.txt "$(random)"
|
||||
fi
|
||||
pg_password="$(cat "$dir/postgres_password.txt")"
|
||||
write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger"
|
||||
|
||||
if [ ! -f "$dir/valkey_password.txt" ]; then
|
||||
write_new valkey_password.txt "$(random)"
|
||||
fi
|
||||
valkey_password="$(cat "$dir/valkey_password.txt")"
|
||||
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
|
||||
|
||||
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
|
||||
# bootstrap; the API gets its own key pair restricted to one bucket.
|
||||
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
|
||||
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
|
||||
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
|
||||
write_new s3_secret_access_key.txt "$(random)$(random)"
|
||||
|
||||
# Lab5: RabbitMQ uses a separate local password. The URL is a Docker secret;
|
||||
# no credentials appear in compose.yaml or the report.
|
||||
if [ ! -f "$dir/broker_password.txt" ]; then
|
||||
write_new broker_password.txt "$(random)"
|
||||
fi
|
||||
broker_password="$(cat "$dir/broker_password.txt")"
|
||||
write_new broker_url.txt "amqp://messenger:${broker_password}@broker:5672/"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown.
|
||||
# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?target API instance alias required}"
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
for id in $(service_ids api); do
|
||||
case "$id" in "$TARGET_INSTANCE_ID"*)
|
||||
docker kill --signal KILL "$id" >/dev/null
|
||||
echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)"
|
||||
exit 0 ;;
|
||||
esac
|
||||
done
|
||||
echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2
|
||||
exit 3
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
|
||||
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
|
||||
PROJECT="${COMPOSE_PROJECT:-messenger-lab6}"
|
||||
|
||||
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
|
||||
service_ids() {
|
||||
# $1 — сервис, $2 — необязательный флаг -a
|
||||
docker ps ${2:-} -q --no-trunc \
|
||||
--filter "label=com.docker.compose.project=$PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1"
|
||||
}
|
||||
|
||||
# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом).
|
||||
# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть
|
||||
# переменные вызывающего цикла.
|
||||
wait_healthy() {
|
||||
_wh_timeout="$1"; shift
|
||||
_wh_deadline=$(( $(date +%s) + _wh_timeout ))
|
||||
for _wh_id in "$@"; do
|
||||
while :; do
|
||||
_wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing)
|
||||
case "$_wh_state" in healthy|running) break ;; esac
|
||||
if [ "$(date +%s)" -ge "$_wh_deadline" ]; then
|
||||
echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
short() { echo "$1" | cut -c1-12; }
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user