58 changed files with 6343 additions and 51 deletions
+12 -24
View File
@@ -1,30 +1,18 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
API_CPU_LIMIT=1.0
API_MEMORY_LIMIT=512M
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
MAX_CONCURRENT_UPLOADS=10
WORKER_CONCURRENCY=1
WORKER_REPLICAS=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
RECONCILE_INTERVAL_SECONDS=900
RECONCILE_GRACE_SECONDS=3600
GRAFANA_PORT=3000
OTEL_SDK_DISABLED=false
OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=1.0
+22
View File
@@ -0,0 +1,22 @@
FROM python:3.12.7-slim
WORKDIR /app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY . /app
RUN chown -R app:app /app
USER app
EXPOSE 8080
CMD ["python3", "src/server.py"]
+37 -2
View File
@@ -4,10 +4,12 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load benchmark-observability test-attachments test-idempotency-race test-reconcile \
test-worker test-pipeline-faults test-pipeline-replay test-outbox-recovery test-worker-recovery \
test-reconcile-integration test-upload-concurrency test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check/validate: offline contract checks; test: public smoke; student targets: focused attachment/pipeline checks"
check:
$(PYTHON) scripts/check_contract.py
@@ -21,6 +23,39 @@ test:
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
benchmark-observability:
$(PYTHON) student-tests/track1_benchmark.py --base-url "$(BASE_URL)"
test-attachments:
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency-race:
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile:
$(PYTHON) student-tests/test_reconcile.py
test-worker:
$(PYTHON) student-tests/test_worker.py
test-pipeline-faults:
$(PYTHON) student-tests/test_pipeline_faults.py
test-pipeline-replay:
$(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-outbox-recovery:
$(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-worker-recovery:
$(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile-integration:
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-upload-concurrency:
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+56 -19
View File
@@ -1,37 +1,74 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: не указаны.
- Целевая оценка: 5; трек 1 — цена наблюдаемости.
- Commit/PR: не создан.
- Источники и инструменты: README и CONTRIBUTING лабораторной 6; документация OpenTelemetry, Prometheus, Grafana, Loki и Tempo; исходники Lab 5; помощь ИИ использовалась для переноса, реализации и проверки.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Compose v2 и Python 3.9+ (проверено на Python 3.12.7). Для локального запуска:
```sh
make check
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
cp .env.example .env
scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождаться успешного ответа GET /health/ready.
make test PYTHON=.venv/bin/python BASE_URL=http://localhost:8080
make validate PYTHON=.venv/bin/python
docker compose --env-file .env stop
```
При занятом порте 8080 задайте `APP_PORT` в локальном `.env` и используйте соответствующий `BASE_URL`; в проверенной среде использовался порт 18080. `scripts/bootstrap-secrets.sh` создаёт локальные секреты в `secrets/`; не копируйте их в Git. Обычная остановка сохраняет named volumes. Только для намеренного удаления учебных данных выполните `docker compose --env-file .env down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перенесены API и фоновый pipeline Lab 5 без изменения публичного контракта. API записывает бизнес-состояние и transactional outbox в PostgreSQL; publisher передаёт задания в RabbitMQ, worker обрабатывает вложения и сохраняет оригиналы/варианты в S3-совместимом SeaweedFS. PostgreSQL хранит авторитетное состояние и метаданные; reconciler сверяет их с объектами хранилища. Доставка at-least-once, повторные эффекты ограничены идемпотентностью и fencing.
API, publisher и worker экспортируют traces, metrics и структурные logs через OTLP в Collector. W3C trace context сохраняется в outbox и переносится через broker. Prometheus собирает метрики Collector, Loki хранит логи, Tempo — traces, Grafana загружает datasource и dashboard из provisioning. Метрики используют route templates и другие низкокардинальные атрибуты, не идентификаторы пользователей, чатов или сообщений. В telemetry не включаются тела сообщений, credentials, filenames или signed URLs.
SDK batch queues ограничены 4096 records, batch — 512; Collector ограничен memory limiter 384 MiB (spike 96 MiB), batch до 1024 элементов и очередями exporters по 256 requests с retry не более 30 секунд. Для локального инцидента с недоступным Tempo API smoke продолжил проходить, а измеренная память API и Collector оставалась около 57–76 MiB.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke и контракты | `make check`, `make validate`, `make test PYTHON=.venv/bin/python BASE_URL=http://localhost:18080` | Все проверки прошли; smoke прошёл с включённой телеметрией | README и `tests/` |
| Уровень 3: pipeline и API | `make test-attachments`, `make test-idempotency-race`, `make test-pipeline-replay` | Валидация вложений прошла; 12 конкурентных запросов с одним idempotency key дали одно сообщение; replay прошёл | `student-tests/`; `evidence/pipeline-faults.json` |
| Уровень 3: Grafana и сигналы | Grafana provisioning и Tempo trace | Загружен dashboard из 14 панелей и три datasource. Trace `69a718dd0f13278bddb76e719b8b23f1` связал upload/API, publisher, worker, S3 и DB | `infra/otel/grafana/`; Tempo |
| Уровень 4: восстановление данных | `make test-outbox-recovery`, `make test-worker-recovery`, `make test-reconcile-integration` | Outbox и worker восстановили вложение до `ready`; reconcile восстановил отсутствующий объект, удалил сироты и сохранил свежий объект до истечения grace period | `student-tests/` |
| Уровень 4: dashboard и SLO | Prometheus rules API, `promtool check rules`, Grafana dashboard | Доступны latency histogram, ошибки, очередь, DB pool и worker; rolling SLI `messenger:api_availability:ratio7d` вернул 1 при отсутствии 5xx. Правила прошли проверку; выражения нулевой ошибки и возраста очереди исправлены по результатам проверки | `infra/otel/alerts.yml`; `infra/otel/grafana/dashboards/messenger.json` |
| Уровень 4: отказ и alerts | Остановить worker с ожидающей вложением; затем восстановить. Отдельно остановить PostgreSQL и отправить 90 последовательных API-запросов | `MessengerQueueLag` перешёл firing и стал inactive после обработки задания. PostgreSQL outage дал 90 ответов 503; `MessengerApiErrorBudgetBurn` перешёл firing. После запуска PostgreSQL и API readiness восстановилась | `evidence/incidents.json` |
| Уровень 4: telemetry backend | Остановить Tempo на 40 секунд и выполнить `make test` | API smoke прошёл, память API/Collector осталась ограниченной; после возврата Tempo API readiness восстановилась | `evidence/incidents.json`; `infra/otel/collector.yaml` |
| Трек 1: цена наблюдаемости | `.venv/bin/python student-tests/track1_benchmark.py --duration 15 --concurrency 2 --target-rps 20 --warmup-seconds 3 --base-url http://localhost:18080` | Четыре режима завершили по 300 запросов без ошибок; результаты приведены ниже | `evidence/track1-observability-cost.json` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Дополнительно прошли `make test-reconcile`, `make test-worker`, `make test-pipeline-faults`, `make test-upload-concurrency`. Upload concurrency обработал 10 загрузок при 10 параллельных операциях; измеренный пик прироста working set API-контейнеров — около 14 MiB.
## Эксперименты и отказы
## Исследовательский трек 1 — цена наблюдаемости
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Гипотеза: sampling снижает число сохранённых spans и объём traces при сохранении метрик и логов. Нагрузка одинаковая для режимов: paced synthetic POST, target 20 RPS, concurrency 2, 15 секунд, прогрев 3 секунды; macOS 27 arm64, 10 логических CPU, Compose-контейнеры. Все режимы завершили 300 запросов с нулём ошибок; фактическая пропускная способность была 16.97–17.85 успешных RPS.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Режим | Traces / доля запросов | Успешный RPS | p50 / p95 / p99, ms | Средний CPU, сумма контейнеров | Пиковая память, MB | Tempo volume delta |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| SDK off | 0 / 0% | 17.750 | 7.449 / 12.198 / 14.054 | 17.18% | 331 | 120 KiB |
| Sampling 100% | 300 / 100% | 16.970 | 8.220 / 14.766 / 37.456 | 25.32% | 340 | 418 KiB |
| Sampling 25% | 88 / 29.3% | 17.845 | 6.822 / 12.894 / 14.415 | 26.23% | 352 | 3.90 MiB |
| Sampling 10% | 25 / 8.3% | 17.393 | 7.351 / 13.809 / 17.187 | 28.82% | 340 | 104 KiB |
## Ограничения и следующие шаги
В sampling-режимах в среднем было около 8 spans на message trace. Поэтому приблизительная оценка не записанных spans составляет 0 при 100%, 1696 при 25% и 2200 при 10%; это экстраполяция по первым 20 traces, а не счётчик доставки Collector. SDK off отключает telemetry SDK целиком, поэтому там ожидалось около 2400 spans, если бы трассировка была включена.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Для дальнейшего использования выбран sampling 25%: в этом прогоне он сохранил около 29% message traces и уменьшил оценочное число spans примерно на 71% относительно 100%, оставляя достаточно трасс для диагностики.
## Инциденты и runbooks
| Воздействие | Причина и наблюдение | Восстановление / данные |
| --- | --- | --- |
| Worker остановлен при одной queued attachment | API принял задание; рост возраста очереди виден в метрике, `MessengerQueueLag` сработал. | Worker запущен, задание обработано до `ready`, очередь осушена, alert inactive. Потери вложения нет. |
| PostgreSQL остановлен на время серии запросов | API вернул 503 на 90 последовательных запросов; `MessengerApiErrorBudgetBurn` сработал по метрике 5xx. | PostgreSQL запущен, API контейнеры перезапущены для сброса stale pool connections, `/health/ready` восстановился. Запросы получили 503; частично принятых операций не было. |
| Tempo недоступен 40 секунд | При остановленном trace backend API smoke оставался успешен; память API/Collector не демонстрировала неограниченного роста. | Tempo запущен, API readiness осталась healthy. |
Для API 5xx: проверить SLI/alert, сгруппировать ошибки по route template, открыть trace и correlated log, проверить DB/S3 spans и состояние соответствующего backend; после восстановления подтвердить health/readiness и возврат error ratio. Для queue lag: проверить depth/age, worker consuming logs и broker; восстановить worker, дождаться `ready` и inactive alert. Для telemetry outage: проверить Collector exporter failures и backend health; бизнес-операции проверять отдельно от полноты telemetry.
+5 -6
View File
@@ -1,6 +1,5 @@
# Topology worksheet, deliberately NOT a runnable stack (no application supplied).
# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment.
# Service names below are logical roles, not imposed container names.
# Topology worksheet, deliberately NOT a runnable stack.
# Implemented roles are in compose.yaml; these names are logical, not container names.
x-lab-number: 6
x-required-roles:
- api
@@ -10,8 +9,8 @@ x-required-roles:
- broker
- worker
- otel_collector
- telemetry_storage
- prometheus
- loki
- tempo
- grafana
# TODO: implement real images/builds, network wiring, volumes, healthchecks,
# credentials and bootstrap. Details: contracts/infrastructure.json and README.md.
services: {}
+468
View File
@@ -0,0 +1,468 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
rabbitmq:
image: rabbitmq:3.13.7-management-alpine
secrets:
- source: rabbitmq_config
target: /etc/rabbitmq/conf.d/10-defaults.conf
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
restart: unless-stopped
s3-core:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- server
- -dir=/data
- -filer=true
- -s3=false
volumes:
- seaweed_data:/data
healthcheck:
test:
- CMD-SHELL
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
networks:
- seaweed_internal
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- s3
- -filer=s3-core:8888
- -config=/run/secrets/s3_config
- -port=8333
- -port.https=0
- -port.grpc=0
- -port.iceberg=0
- -port.lance=0
- -concurrentFileUploadLimit=10
- -concurrentUploadLimitMB=100
- -readerCacheSizeMB=64
- -autoCreateBucket=false
- -ip.bind=0.0.0.0
secrets:
- s3_config
depends_on:
s3-core:
condition: service_healthy
networks:
- default
- seaweed_internal
healthcheck:
test:
- CMD-SHELL
- "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3-bootstrap:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/bootstrap_s3.py"]
environment:
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
UPLOAD_TIMEOUT_SECONDS: "60"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX_SIZE: "8"
DB_POOL_TIMEOUT_SECONDS: "3"
DB_STATEMENT_TIMEOUT_MS: "5000"
DB_LOCK_TIMEOUT_MS: "2000"
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
otel-collector:
condition: service_started
healthcheck:
test:
- CMD
- python3
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
interval: 3s
timeout: 3s
retries: 5
start_period: 10s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_LIMIT:-1.0}"
memory: "${API_MEMORY_LIMIT:-512M}"
stop_grace_period: 20s
restart: unless-stopped
publisher:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.publisher"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-publisher
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- database_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
otel-collector:
condition: service_started
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
worker:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.worker"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
MAX_IMAGE_PIXELS: "40000000"
DB_POOL_MAX_SIZE: "2"
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-worker
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- database_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
otel-collector:
condition: service_started
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
reconciler:
build:
context: .
dockerfile: Dockerfile
command: ["sh", "scripts/reconcile-loop.sh"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://s3:8333
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
DB_POOL_MAX_SIZE: "2"
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.163.2
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
depends_on:
- tempo
- loki
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.55.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
volumes:
- ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro
- prometheus_data:/prometheus
depends_on:
- otel-collector
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yml"]
volumes:
- ./infra/otel/loki.yml:/etc/loki/loki.yml:ro
- loki_data:/loki
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.1
command: ["-config.file=/etc/tempo/tempo.yml"]
volumes:
- ./infra/otel/tempo.yml:/etc/tempo/tempo.yml:ro
- tempo_data:/var/tempo
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
grafana:
image: grafana/grafana:11.4.0
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
secrets:
- grafana_admin_password
volumes:
- grafana_data:/var/lib/grafana
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
depends_on:
- prometheus
- loki
- tempo
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
load_balancer:
image: haproxy:3.0.7-alpine
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
restart: unless-stopped
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
s3_config:
file: ./secrets/s3.json
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
rabbitmq_password:
file: ./secrets/rabbitmq_password
rabbitmq_config:
file: ./secrets/rabbitmq.conf
broker_url:
file: ./secrets/broker_url
grafana_admin_password:
file: ./secrets/grafana_admin_password
volumes:
postgres_data:
seaweed_data:
rabbitmq_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
networks:
seaweed_internal:
internal: true
+39
View File
@@ -0,0 +1,39 @@
{
"result": "PASS",
"scenarios": [
{
"scenario": "worker_outage",
"injected_fault": "Stopped the worker while one attachment job was queued.",
"alert": "MessengerQueueLag",
"alert_fired": true,
"alert_resolved_after_recovery": true,
"job_reached_ready": true,
"data_loss": false
},
{
"scenario": "postgres_outage",
"injected_fault": "Stopped PostgreSQL during 90 sequential API user-creation requests.",
"api_responses": {
"503": 90
},
"alert": "MessengerApiErrorBudgetBurn",
"alert_fired": true,
"api_ready_after_database_and_api_recovery": true
},
{
"scenario": "tempo_outage",
"injected_fault": "Stopped Tempo for 40 seconds.",
"public_smoke_passed_during_outage": true,
"api_and_collector_memory_mib_while_stopped": {
"api_replica_1": 58.6,
"api_replica_2": 58.6,
"otel_collector": 76.2
},
"api_ready_after_recovery": true
}
],
"limitations": [
"The fault checks are local single-run experiments, not production availability tests.",
"Telemetry backends may drop data after bounded exporter retries during an outage."
]
}
+58
View File
@@ -0,0 +1,58 @@
{
"fixture": "tests/fixtures/sample.png",
"invariants": {
"ambiguous_confirm_republishes_stable_job": true,
"confirmed_work_eventually_ready": true,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
"duplicate_effects_create_no_second_variant_set": true,
"stale_attempt_writes_no_effects": true,
"stale_pipeline_cannot_publish_current_metadata": true
},
"result": "PASS",
"scenarios": [
{
"deliveries": 2,
"duplicate_delivery_writes": 0,
"effective_object_writes": 4,
"scenario": "duplicate_delivery_after_commit_before_ack"
},
{
"attempts": 2,
"scenario": "crash_after_claim",
"terminal_status": "ready"
},
{
"attempts": 2,
"scenario": "crash_after_original_put",
"terminal_status": "ready"
},
{
"current_attempt": 2,
"scenario": "out_of_order_attempt_delivery",
"stale_attempt_extra_writes": 0
},
{
"current_pipeline_version": 2,
"published_variant_versions": [
"v2"
],
"scenario": "stale_pipeline_after_newer_version"
},
{
"outbox_recovered_after_restart": true,
"published_after_restart": 1,
"scenario": "publisher_crash_before_publish"
},
{
"deliveries": 2,
"same_message_id": true,
"scenario": "publish_confirm_lost_after_broker_accept"
},
{
"configured_timeout_seconds": 1,
"observed_seconds": 1.006,
"scenario": "transform_timeout_terminated"
}
],
"track": "2"
}
+208
View File
@@ -0,0 +1,208 @@
{
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
"method": {
"duration_seconds": 15,
"concurrency": 2,
"target_rps": 20,
"warmup_seconds": 3,
"telemetry_flush_wait_seconds": 15,
"hardware": {
"platform": "macOS-27.2.1-arm64-arm-64bit-Mach-O",
"processor": "arm",
"logical_cpus": 10
},
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size."
},
"limitations": [
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
"Database history grows between modes; all modes use identical duration and concurrency.",
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes."
],
"results": [
{
"configuration": "off",
"sdk_disabled": true,
"trace_sampling_ratio": 1.0,
"load": {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": 15,
"target_rps": 20,
"elapsed_seconds": 17.135,
"concurrency": 2,
"successful_requests": 300,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 17.508,
"successful_latency_ms": {
"p50": 8.449,
"p95": 15.118,
"p99": 21.618
},
"docker_resource_samples": 4,
"docker_resources": {
"mean_cpu_percent_sum": 21.28,
"peak_cpu_percent_sum": 38.75,
"peak_memory_bytes_sum": 567678072,
"services": [
"api",
"worker",
"publisher",
"otel-collector"
]
}
},
"window_unix_seconds": {
"start": 1794559044,
"end": 1794559061
},
"sampled_message_trace_count": 0,
"estimated_spans_per_message_trace": 0.0,
"persistent_storage_bytes_delta": {
"prometheus": 1492625,
"loki": 0,
"tempo": 0
},
"estimated_spans_not_recorded": 2400.0
},
{
"configuration": "sample-100-percent",
"sdk_disabled": false,
"trace_sampling_ratio": 1.0,
"load": {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": 15,
"target_rps": 20,
"elapsed_seconds": 18.348,
"concurrency": 2,
"successful_requests": 300,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 16.35,
"successful_latency_ms": {
"p50": 8.284,
"p95": 15.915,
"p99": 18.98
},
"docker_resource_samples": 4,
"docker_resources": {
"mean_cpu_percent_sum": 31.85,
"peak_cpu_percent_sum": 41.59,
"peak_memory_bytes_sum": 550523370,
"services": [
"api",
"worker",
"publisher",
"otel-collector"
]
}
},
"window_unix_seconds": {
"start": 1794559100,
"end": 1794559118
},
"sampled_message_trace_count": 300,
"estimated_spans_per_message_trace": 8.0,
"persistent_storage_bytes_delta": {
"prometheus": 36951,
"loki": 204007,
"tempo": 422028
},
"estimated_spans_not_recorded": 0.0
},
{
"configuration": "sample-25-percent",
"sdk_disabled": false,
"trace_sampling_ratio": 0.25,
"load": {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": 15,
"target_rps": 20,
"elapsed_seconds": 17.76,
"concurrency": 2,
"successful_requests": 300,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 16.891,
"successful_latency_ms": {
"p50": 7.793,
"p95": 14.838,
"p99": 16.862
},
"docker_resource_samples": 4,
"docker_resources": {
"mean_cpu_percent_sum": 15.26,
"peak_cpu_percent_sum": 36.45,
"peak_memory_bytes_sum": 550554827,
"services": [
"api",
"worker",
"publisher",
"otel-collector"
]
}
},
"window_unix_seconds": {
"start": 1794559157,
"end": 1794559175
},
"sampled_message_trace_count": 88,
"estimated_spans_per_message_trace": 8.0,
"persistent_storage_bytes_delta": {
"prometheus": 40456,
"loki": 269949,
"tempo": 1026146
},
"estimated_spans_not_recorded": 1696.0
},
{
"configuration": "sample-10-percent",
"sdk_disabled": false,
"trace_sampling_ratio": 0.1,
"load": {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": 15,
"target_rps": 20,
"elapsed_seconds": 18.355,
"concurrency": 2,
"successful_requests": 300,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 16.344,
"successful_latency_ms": {
"p50": 7.625,
"p95": 14.84,
"p99": 18.012
},
"docker_resource_samples": 4,
"docker_resources": {
"mean_cpu_percent_sum": 22.53,
"peak_cpu_percent_sum": 39.51,
"peak_memory_bytes_sum": 550198311,
"services": [
"api",
"worker",
"publisher",
"otel-collector"
]
}
},
"window_unix_seconds": {
"start": 1794559215,
"end": 1794559233
},
"sampled_message_trace_count": 38,
"estimated_spans_per_message_trace": 8.0,
"persistent_storage_bytes_delta": {
"prometheus": 42617,
"loki": 240135,
"tempo": 1924562
},
"estimated_spans_not_recorded": 2096.0
}
],
"balanced_configuration": "sample-25-percent"
}
+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
option http-keep-alive
timeout connect 2s
timeout client 15s
timeout server 15s
timeout http-request 5s
timeout check 1s
retries 2
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
frontend messenger
bind :8080
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
option redispatch
default-server inter 1s fall 2 rise 2
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
+42
View File
@@ -0,0 +1,42 @@
groups:
- name: messenger-slo
interval: 10s
rules:
- record: messenger:api_5xx_ratio:rate5m
expr: |
(sum(rate(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0))
/
clamp_min(sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])), 1e-9)
- record: messenger:api_availability:ratio7d
expr: |
1 -
(sum(increase(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[7d])) or vector(0))
/
clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[7d])), 1)
- alert: MessengerApiErrorBudgetBurn
expr: messenger:api_5xx_ratio:rate5m > 0.01
for: 2m
labels:
severity: warning
annotations:
summary: API error budget burn exceeds the 1% short-window budget
description: The rolling 5-minute business API 5xx ratio is above 1%.
runbook: Check API traces for the failing route, then inspect correlated logs and DB/S3 spans.
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_age_seconds) > 60 and max(messenger_queue_depth) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Oldest attachment job has waited for more than 60 seconds
description: Inspect worker availability, RabbitMQ health, and the worker trace for the oldest job.
runbook: Check queue depth and age, worker consuming logs, broker, then recover the worker and confirm resolved state.
- alert: MessengerDatabasePoolSaturation
expr: max(messenger_db_pool_requests_waiting) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Requests are waiting for a database connection
description: Compare DB waiters with request traces and PostgreSQL health.
runbook: Inspect DB spans and pool use; verify database health and connection limits.
+66
View File
@@ -0,0 +1,66 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 384
spike_limit_mib: 96
batch:
timeout: 1s
send_batch_size: 512
send_batch_max_size: 1024
exporters:
prometheus:
endpoint: 0.0.0.0:9464
send_timestamps: true
resource_to_telemetry_conversion:
enabled: true
otlp_grpc/tempo:
endpoint: tempo:4317
tls:
insecure: true
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlp_http/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
telemetry:
logs:
level: warn
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_grpc/tempo]
@@ -0,0 +1,174 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": false,
"graphTooltip": 1,
"panels": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 0},
"id": 1,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_route}}", "refId": "A"}],
"title": "API requests per second",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 12, "y": 0},
"id": 2,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",http_route!~\"/health/.*\"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m])), 1e-9)", "legendFormat": "5xx / business requests", "refId": "A"}],
"title": "Business API 5xx ratio (5m)",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 7},
"id": 3,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "API latency percentiles from merged histogram buckets",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 7},
"id": 4,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "Worker job duration percentiles from histogram buckets",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 15},
"id": 5,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[5m]))", "legendFormat": "{{outcome}}", "refId": "A"}],
"title": "Jobs processed / failed per second",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 15},
"id": 6,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_queue_depth)", "legendFormat": "queued attachments", "refId": "A"}],
"title": "Durable attachment backlog",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 15},
"id": 7,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest queued job", "refId": "A"}],
"title": "Age of oldest queued attachment",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 22},
"id": 8,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_db_pool_connections_used)", "legendFormat": "used", "refId": "A"}, {"expr": "max(messenger_db_pool_requests_waiting)", "legendFormat": "waiting", "refId": "B"}],
"title": "Database pool use / waiters",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 22},
"id": 9,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "busy / capacity", "refId": "A"}],
"title": "Worker saturation",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 22},
"id": 10,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_process_memory_usage_bytes)", "legendFormat": "RSS", "refId": "A"}],
"title": "Process memory",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 29},
"id": 11,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_process_cpu_utilization_ratio)", "legendFormat": "CPU / core", "refId": "A"}],
"title": "Process CPU utilization",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 7, "w": 16, "x": 8, "y": 29},
"id": 12,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachments_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95 time to ready", "refId": "A"}],
"title": "Durable acceptance to attachment ready",
"type": "timeseries"
},
{
"datasource": {"type": "loki", "uid": "loki"},
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 36},
"id": 13,
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": true, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
"targets": [{"expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", "refId": "A"}],
"title": "Structured API / worker logs",
"type": "logs"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0.9, "max": 1}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 44},
"id": 14,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
"targets": [{"expr": "messenger:api_availability:ratio7d", "legendFormat": "7-day availability; objective 99%", "refId": "A"}],
"title": "API SLI: 7-day rolling availability (99% SLO)",
"type": "timeseries"
}
],
"schemaVersion": 39,
"tags": ["messenger", "observability"],
"templating": {"list": []},
"time": {"from": "now-1h", "to": "now"},
"timezone": "browser",
"title": "Messenger observability",
"uid": "messenger-observability",
"version": 1
}
@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: true
editable: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,39 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
matcherRegex: '"trace_id":"([a-f0-9]+)"'
url: "$${__value.raw}"
datasourceUid: tempo
editable: false
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-1h"
spanEndTimeShift: "1h"
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"'
serviceMap:
datasourceUid: prometheus
editable: false
+34
View File
@@ -0,0 +1,34 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
+14
View File
@@ -0,0 +1,14 @@
global:
scrape_interval: 10s
evaluation_interval: 10s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:9464"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+27
View File
@@ -0,0 +1,27 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
trace_idle_period: 10s
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
+43
View File
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title varchar(100) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method varchar(8) NOT NULL,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key varchar(128) NOT NULL,
request_hash bytea NOT NULL,
response jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
+59
View File
@@ -0,0 +1,59 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename varchar(255) NOT NULL,
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
staging_key text NOT NULL UNIQUE,
job_id uuid NOT NULL UNIQUE,
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
lease_until timestamptz,
status varchar(16) NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'),
error_code varchar(64),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, ordinal)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
CREATE TABLE attachment_outbox (
id uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
attempt integer NOT NULL CHECK (attempt >= 1),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
UNIQUE (job_id, attempt)
);
CREATE INDEX attachment_outbox_pending_idx
ON attachment_outbox (available_at, created_at)
WHERE published_at IS NULL;
CREATE TABLE reconciliation_incidents (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, incident_type)
);
+16
View File
@@ -0,0 +1,16 @@
ALTER TABLE attachment_outbox
DROP CONSTRAINT attachment_outbox_job_id_fkey;
CREATE TABLE attachment_pipeline_events (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
prior_job_id uuid NOT NULL,
job_id uuid NOT NULL,
prior_version integer NOT NULL CHECK (prior_version >= 1),
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
action varchar(16) NOT NULL CHECK (action = 'replay'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachment_pipeline_events_attachment_created_idx
ON attachment_pipeline_events (attachment_id, created_at);
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE attachment_outbox
ADD COLUMN traceparent varchar(55),
ADD COLUMN tracestate varchar(512);
+12
View File
@@ -0,0 +1,12 @@
psycopg[binary]==3.2.10
psycopg-pool==3.3.3
redis==5.2.1
boto3==1.35.99
Pillow==11.0.0
pika==1.3.2
opentelemetry-api==1.29.0
opentelemetry-sdk==1.29.0
opentelemetry-exporter-otlp-proto-http==1.29.0
opentelemetry-instrumentation-psycopg==0.50b0
opentelemetry-instrumentation-botocore==0.50b0
psutil==6.1.1
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
import json
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
def main():
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
names = (
"postgres_password",
"database_url",
"s3_access_key_id",
"s3_secret_access_key",
"s3_admin_access_key_id",
"s3_admin_secret_access_key",
"rabbitmq_password",
"rabbitmq.conf",
"broker_url",
"s3.json",
"grafana_admin_password",
)
existing = [name for name in names if (SECRET_DIR / name).exists()]
if existing:
raise SystemExit(
"Secret files already exist; refusing to rotate credentials without a migration: "
+ ", ".join(existing)
)
db_password = secrets.token_hex(24)
broker_password = secrets.token_urlsafe(32)
values = {
"postgres_password": db_password,
"database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger",
"s3_access_key_id": "mngr" + secrets.token_hex(8),
"s3_secret_access_key": secrets.token_urlsafe(40),
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
"rabbitmq_password": broker_password,
"broker_url": f"amqp://messenger:{broker_password}@rabbitmq:5672/%2f",
"grafana_admin_password": secrets.token_urlsafe(32),
}
for name, value in values.items():
path = SECRET_DIR / name
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
descriptor = os.open(
SECRET_DIR / "rabbitmq.conf",
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write("default_user = messenger\n")
output.write(f"default_pass = {broker_password}\n")
s3_config = {
"identities": [
{
"name": "bootstrap-admin",
"credentials": [
{
"accessKey": values["s3_admin_access_key_id"],
"secretKey": values["s3_admin_secret_access_key"],
}
],
"actions": ["Admin"],
},
{
"name": "messenger-api",
"credentials": [
{
"accessKey": values["s3_access_key_id"],
"secretKey": values["s3_secret_access_key"],
}
],
"actions": [
"Read:messenger-attachments",
"Write:messenger-attachments",
"List:messenger-attachments",
],
},
]
}
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(s3_config, output, separators=(",", ":"))
output.write("\n")
print("Local secrets generated with owner-only permissions; values were not printed.")
if __name__ == "__main__":
main()
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 "$(dirname "$0")/bootstrap-secrets.py"
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
interval=${RECONCILE_INTERVAL_SECONDS:-900}
grace=${RECONCILE_GRACE_SECONDS:-3600}
case "$interval" in
''|*[!0-9]*)
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
exit 2
;;
esac
case "$grace" in
''|*[!0-9]*)
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
exit 2
;;
esac
if [ "$interval" -lt 60 ]; then
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
exit 2
fi
if [ "$grace" -lt 60 ]; then
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
exit 2
fi
while :
do
python3 src/reconcile.py --grace-seconds "$grace" --apply
sleep "$interval"
done
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart postgres s3-core rabbitmq prometheus loki tempo
docker compose up -d --wait postgres s3-core rabbitmq prometheus loki tempo
docker compose restart s3
docker compose up -d --wait s3
attempt=0
until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 4 ]; then
echo "S3 object reads did not recover after restart" >&2
exit 1
fi
sleep 2
done
docker compose restart api
docker compose restart publisher worker
docker compose restart otel-collector grafana
docker compose up -d --wait api publisher worker otel-collector grafana
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
found=
for container_id in $(docker compose ps --quiet --status running api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
found=$container_id
break
fi
done
if [ -z "$found" ]; then
printf '%s\n' 'No running API container matched the requested instance alias.' >&2
exit 1
fi
docker stop --time 5 "$found" >/dev/null
+1
View File
@@ -0,0 +1 @@
"""Messenger API application package."""
+41
View File
@@ -0,0 +1,41 @@
import logging
import os
import signal
import threading
from functools import partial
from http.server import ThreadingHTTPServer
from .config import APP_HOST, APP_PORT
from .handlers import MessengerHandler
from .store import Store
from .telemetry import initialize, start_runtime_metrics
LOGGER = logging.getLogger("messenger")
def run_server():
store = Store()
store.open()
start_runtime_metrics(store)
try:
handler = partial(MessengerHandler, store=store)
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
def handle_shutdown(signum, frame):
LOGGER.info("received signal %s, shutting down gracefully", signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
try:
server.serve_forever()
finally:
server.server_close()
finally:
store.close()
def main():
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-api"))
run_server()
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
import logging
import os
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
LOGGER = logging.getLogger("messenger.s3.bootstrap")
def secret(name: str) -> str:
direct = os.getenv(name)
if direct:
return direct
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError(f"Set {name} or {name}_FILE")
def main():
client = boto3.client(
"s3",
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
region_name="us-east-1",
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}),
)
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
client.create_bucket(Bucket=bucket)
client.put_bucket_acl(Bucket=bucket, ACL="private")
LOGGER.info("private S3 bucket bootstrap completed")
if __name__ == "__main__":
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
main()
+61
View File
@@ -0,0 +1,61 @@
import os
import re
import socket
import uuid
from pathlib import Path
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
def secret_value(name: str, default: str = "") -> str:
value = os.getenv(name)
if value:
return value
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
if default:
return default
raise RuntimeError(f"Set {name} or {name}_FILE")
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
if IDEMPOTENCY_TTL_SECONDS < 86400:
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
raise RuntimeError("Database timeouts must be positive")
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
raise RuntimeError("Database pool settings must be positive")
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
raise RuntimeError("HTTP limits must be positive")
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
if MAX_CONCURRENT_UPLOADS < 1:
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
if UPLOAD_TIMEOUT_SECONDS < 1:
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
+26
View File
@@ -0,0 +1,26 @@
from typing import Dict, Optional
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class CacheUnavailableError(Exception):
pass
class IdempotencyConflictError(Exception):
pass
class AttachmentNotReadyError(Exception):
pass
class ObjectMissingError(Exception):
pass
+633
View File
@@ -0,0 +1,633 @@
import base64
import hashlib
import json
import logging
import threading
import time
import tempfile
import uuid
import warnings
import zlib
from http.server import BaseHTTPRequestHandler
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
import unicodedata
import psycopg
from opentelemetry import propagate
from opentelemetry.instrumentation.utils import suppress_instrumentation
from opentelemetry.trace import SpanKind, Status, StatusCode
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, UnidentifiedImageError
from psycopg_pool import PoolTimeout
from .config import (
IDEMPOTENCY_KEY_RE,
INSTANCE_ID,
MAX_CONCURRENT_UPLOADS,
MAX_IMAGE_PIXELS,
MAX_INFLIGHT_REQUESTS,
MAX_REQUEST_BODY_BYTES,
MAX_UPLOAD_BYTES,
UPLOAD_TIMEOUT_SECONDS,
USER_RE,
)
from .errors import (
ApiError,
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .security import hash_password, isoformat, valid_uuid
from .store import Store
from .telemetry import REQUEST_ID, record_http_request, tracer
LOGGER = logging.getLogger("messenger.http")
class RequestGuard:
def __init__(self):
self.lock = threading.Lock()
self.inflight = 0
def acquire(self):
with self.lock:
if self.inflight >= MAX_INFLIGHT_REQUESTS:
raise ApiError(503, "unavailable", "Server is busy")
self.inflight += 1
def release(self):
with self.lock:
self.inflight = max(0, self.inflight - 1)
REQUEST_GUARD = RequestGuard()
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
class MessengerHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server, store: Store):
self.store = store
super().__init__(request, client_address, server)
def setup(self):
super().setup()
self.connection.settimeout(15)
def log_message(self, format: str, *args):
return
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
path = urlsplit(self.path).path
route = self._route_template(path)
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
parent_context = propagate.extract({key.lower(): value for key, value in self.headers.items()})
started = time.perf_counter()
token = REQUEST_ID.set(request_id)
self._response_status = 500
if route.startswith("/health/"):
try:
self._dispatch_request(method, request_id)
finally:
record_http_request(method, route, self._response_status, time.perf_counter() - started)
REQUEST_ID.reset(token)
return
with tracer().start_as_current_span(
f"{method} {route}",
context=parent_context,
kind=SpanKind.SERVER,
attributes={
"http.request.method": method,
"http.route": route,
"http.request_id": request_id,
},
) as span:
try:
self._dispatch_request(method, request_id)
finally:
duration = time.perf_counter() - started
span.set_attribute("http.response.status_code", self._response_status)
if self._response_status >= 500:
span.set_status(Status(StatusCode.ERROR))
record_http_request(method, route, self._response_status, duration)
REQUEST_ID.reset(token)
@staticmethod
def _route_template(path: str) -> str:
if path in ("/health/live", "/health/ready", "/api/v1/users", "/api/v1/auth/sessions",
"/api/v1/auth/sessions/current", "/api/v1/users/me", "/api/v1/chats"):
return path
if path.startswith("/api/v1/chats/") and path.endswith("/attachments"):
return "/api/v1/chats/{chat_id}/attachments"
if path.startswith("/api/v1/chats/") and path.endswith("/messages"):
return "/api/v1/chats/{chat_id}/messages"
if path.startswith("/api/v1/chats/"):
return "/api/v1/chats/{chat_id}"
if path.startswith("/api/v1/attachments/") and path.endswith("/content"):
return "/api/v1/attachments/{attachment_id}/content"
if path.startswith("/api/v1/attachments/"):
return "/api/v1/attachments/{attachment_id}"
return "unmatched"
def _dispatch_request(self, method: str, request_id: str):
acquired = False
try:
REQUEST_GUARD.acquire()
acquired = True
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == "/health/live" and method == "GET":
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/health/ready" and method == "GET":
try:
with suppress_instrumentation():
self.store.ready()
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "A required storage service is unavailable")
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/api/v1/users" and method == "POST":
self._handle_register(request_id)
elif path == "/api/v1/auth/sessions" and method == "POST":
self._handle_create_session(request_id)
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
self._handle_revoke_session(request_id)
elif path == "/api/v1/users/me" and method == "GET":
self._send_json(200, self._get_actor(request_id), request_id)
elif path == "/api/v1/chats" and method == "POST":
self._handle_create_chat(request_id)
elif path == "/api/v1/chats" and method == "GET":
self._handle_list_chats(query, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
chat_id = path.rsplit("/", 2)[1]
self._handle_upload_attachment(chat_id, query, request_id)
elif path.startswith("/api/v1/attachments/") and method == "GET":
attachment_path = path.removeprefix("/api/v1/attachments/")
if attachment_path.endswith("/content"):
attachment_id = attachment_path.removesuffix("/content")
self._handle_download_attachment(attachment_id, query, request_id)
else:
self._handle_get_attachment(attachment_path, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
chat_id = path.rsplit("/", 2)[1]
if method == "POST":
self._handle_create_message(chat_id, request_id)
else:
self._handle_list_messages(chat_id, query, request_id)
elif path.startswith("/api/v1/chats/") and method == "GET":
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
else:
raise ApiError(404, "not_found", "Object not found")
except ApiError as exc:
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
except CacheUnavailableError:
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
except IdempotencyConflictError:
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
except AttachmentNotReadyError:
self._send_error(409, "conflict", "Attachment is not ready", request_id)
except ObjectMissingError:
LOGGER.error("attachment object missing for request %s", request_id)
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
except (BotoCoreError, ClientError):
LOGGER.warning("object storage unavailable for request %s", request_id)
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
self._send_error(503, "unavailable", "Database is unavailable", request_id)
except psycopg.errors.UniqueViolation:
self._send_error(409, "conflict", "Username already exists", request_id)
except psycopg.errors.ForeignKeyViolation:
self._send_error(400, "invalid_request", "Unknown member", request_id)
except (BrokenPipeError, ConnectionResetError, TimeoutError):
self._set_response_status(499)
self.close_connection = True
LOGGER.info("client disconnected before response for request %s", request_id)
except Exception as exc:
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
self._send_error(500, "internal_error", "Internal server error", request_id)
finally:
if acquired:
REQUEST_GUARD.release()
@staticmethod
def _trace_context():
carrier = {}
propagate.inject(carrier)
return {"traceparent": carrier.get("traceparent"), "tracestate": carrier.get("tracestate")}
def _set_response_status(self, status: int) -> None:
self._response_status = status
def _read_json(self, request_id: str) -> Any:
if self.headers.get_content_type() != "application/json":
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Missing Content-Length")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size > MAX_REQUEST_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Request body too large")
raw = self.rfile.read(size)
if not raw:
raise ApiError(400, "invalid_request", "Empty body")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise ApiError(400, "invalid_request", "Malformed JSON")
def _get_actor(self, request_id: str) -> Dict[str, Any]:
header = self.headers.get("Authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not token or " " in token:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
user = self.store.session_user(token)
if user is None:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def _handle_register(self, request_id: str):
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
username = body["username"]
display_name = body["display_name"]
password = body["password"]
if not isinstance(username, str) or not USER_RE.fullmatch(username):
raise ApiError(400, "invalid_request", "Invalid username")
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
raise ApiError(400, "invalid_request", "Invalid display_name")
if not isinstance(password, str) or not (12 <= len(password) <= 128):
raise ApiError(400, "invalid_request", "Invalid password")
user = self.store.create_user(username, display_name, hash_password(password))
self._send_json(201, user, request_id)
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
value = self.headers.get("Authorization", "")
scheme, separator, encoded = value.partition(" ")
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
if scheme.lower() != "basic" or not separator or not encoded:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
return username, password
def _handle_create_session(self, request_id: str):
username, password = self._basic_credentials(request_id)
user = self.store.authenticate(username, password)
if user is None:
raise ApiError(
401,
"unauthorized",
"Invalid credentials",
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
token, expires_at = self.store.create_session(user["id"])
self._send_json(
201,
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
request_id,
)
def _handle_revoke_session(self, request_id: str):
self._get_actor(request_id)
token = self.headers["Authorization"].partition(" ")[2]
if not self.store.revoke_session(token):
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
self._set_response_status(204)
self.send_response(204)
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.end_headers()
def _handle_create_chat(self, request_id: str):
actor = self._get_actor(request_id)
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
raise ApiError(400, "invalid_request", "Expected title and member_ids")
title, member_ids = body["title"], body["member_ids"]
if not isinstance(title, str) or not title.strip() or len(title) > 100:
raise ApiError(400, "invalid_request", "Invalid title")
if (
not isinstance(member_ids, list)
or not 1 <= len(member_ids) <= 100
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
or len(member_ids) != len(set(member_ids))
or actor["id"] not in member_ids
):
raise ApiError(400, "invalid_request", "Invalid member_ids")
chat = self.store.create_chat(title, member_ids, actor["id"])
self._send_json(201, chat, request_id)
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
def _handle_get_chat(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat = self.store.get_chat(chat_id, actor["id"])
if chat is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat, request_id)
def _handle_create_message(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
idempotency_key = self.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
text = body["text"]
attachment_ids = body.get("attachment_ids", [])
if (
not isinstance(text, str)
or len(text) > 4000
or not isinstance(attachment_ids, list)
or len(attachment_ids) > 10
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
or len(attachment_ids) != len(set(attachment_ids))
or (not text.strip() and not attachment_ids)
):
raise ApiError(400, "invalid_request", "Invalid message text")
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
canonical_body = json.dumps(
{"text": text, "attachment_ids": attachment_ids},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
message = self.store.create_idempotent_message(
chat_id,
actor["id"],
text,
attachment_ids,
idempotency_key,
canonical_body,
)
if message is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(201, message, request_id)
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
if declared_type not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
length = self.headers.get("Content-Length")
if length is None or self.headers.get("Transfer-Encoding"):
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size <= 0:
raise ApiError(415, "unsupported_media_type", "Image is empty")
if size > MAX_UPLOAD_BYTES:
self.close_connection = True
remaining = min(size, MAX_UPLOAD_BYTES + 1)
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
while remaining:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
break
self.connection.settimeout(min(15, remaining_time))
try:
chunk = self.rfile.read(min(64 * 1024, remaining))
except (TimeoutError, OSError):
LOGGER.info("stopped bounded drain of oversized upload")
break
if not chunk:
break
remaining -= len(chunk)
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
digest = hashlib.sha256()
bytes_read = 0
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
try:
while bytes_read < size:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise ApiError(408, "invalid_request", "Image upload timed out")
self.connection.settimeout(min(15, remaining_time))
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
if not chunk:
raise ApiError(400, "invalid_request", "Incomplete image body")
content.write(chunk)
digest.update(chunk)
bytes_read += len(chunk)
except (TimeoutError, OSError):
content.close()
raise ApiError(400, "invalid_request", "Incomplete image body") from None
try:
content.seek(0)
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
if actual_type is None or actual_type != declared_type:
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
width, height = image.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
image.verify()
content.seek(0)
with IMAGE_DECODE_GUARD:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
image.load()
content.seek(0)
return content, actual_type, size, digest.hexdigest()
except ApiError:
content.close()
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
content.close()
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
content.close()
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
@staticmethod
def _valid_filename(filename: str) -> bool:
return (
1 <= len(filename) <= 255
and not any(char in ("/", "\\", "\x00") for char in filename)
and not any(unicodedata.category(char) == "Cc" for char in filename)
)
def _handle_upload_attachment(
self, chat_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
if not self.store.can_access_chat(chat_id, actor["id"]):
raise ApiError(404, "not_found", "Chat not found")
filenames = query.get("filename", [])
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
raise ApiError(400, "invalid_request", "Invalid filename")
if not UPLOAD_GUARD.acquire(blocking=False):
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
try:
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
try:
attachment = self.store.create_attachment(
chat_id,
actor["id"],
filenames[0],
content_type,
content,
size,
digest,
**self._trace_context(),
)
finally:
content.close()
finally:
UPLOAD_GUARD.release()
if attachment is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(202, attachment, request_id)
def _handle_get_attachment(self, attachment_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
if attachment is None:
raise ApiError(404, "not_found", "Attachment not found")
self._send_json(200, attachment, request_id)
def _handle_download_attachment(
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
variants = query.get("variant", ["original"])
if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"):
raise ApiError(400, "invalid_request", "Invalid attachment variant")
variant = variants[0]
if variant != "original" and variant not in ("thumbnail", "crop", "optimized"):
raise ApiError(404, "not_found", "Attachment variant not found")
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant)
if result is None:
raise ApiError(404, "not_found", "Attachment not found")
content_type, content = result
self._set_response_status(200)
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(content)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Cache-Control", "private, no-store")
self.end_headers()
self.wfile.write(content)
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if result is None:
raise ApiError(404, "not_found", "Chat not found")
items, next_cursor = result
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
def _send_json(
self,
status: int,
payload: Dict[str, Any],
request_id: str,
headers: Optional[Dict[str, str]] = None,
):
self._set_response_status(status)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
if headers:
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def _send_error(
self,
status: int,
code: str,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
+50
View File
@@ -0,0 +1,50 @@
import json
import os
from pathlib import Path
import pika
QUEUE_NAME = "attachments.process.v1"
DEAD_QUEUE_NAME = "attachments.process.dead"
LEASE_NETWORK_BUDGET_SECONDS = 150
def broker_url() -> str:
value = os.getenv("BROKER_URL")
if value:
return value
file_path = os.getenv("BROKER_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set BROKER_URL or BROKER_URL_FILE")
def connect() -> pika.BlockingConnection:
parameters = pika.URLParameters(broker_url())
parameters.connection_attempts = 3
parameters.retry_delay = 1
parameters.socket_timeout = 5
parameters.blocked_connection_timeout = 10
return pika.BlockingConnection(parameters)
def declare_queues(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
channel.queue_declare(
queue=DEAD_QUEUE_NAME,
durable=True,
arguments={"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
)
channel.queue_declare(
queue=QUEUE_NAME,
durable=True,
arguments={
"x-queue-type": "quorum",
"x-quorum-initial-group-size": 1,
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": DEAD_QUEUE_NAME,
},
)
def encode_job(job: dict) -> bytes:
return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8")
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
import logging
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
def main():
logging.basicConfig(level="INFO")
store = Store()
store.open()
try:
store.migrate()
finally:
store.close()
if __name__ == "__main__":
main()
+76
View File
@@ -0,0 +1,76 @@
import logging
from datetime import datetime
from typing import BinaryIO, Iterator, Optional, Tuple, Union
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
from .errors import ObjectMissingError
LOGGER = logging.getLogger("messenger.s3")
class ObjectStore:
def __init__(self):
self.client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
config=Config(
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
),
)
def put(
self,
key: str,
content: Union[bytes, BinaryIO],
content_type: str,
size_bytes: Optional[int] = None,
) -> None:
if size_bytes is None:
size_bytes = len(content)
self.client.put_object(
Bucket=S3_BUCKET,
Key=key,
Body=content,
ContentLength=size_bytes,
ContentType=content_type,
)
def get(self, key: str, max_bytes: int) -> bytes:
try:
response = self.client.get_object(Bucket=S3_BUCKET, Key=key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code in ("NoSuchKey", "404", "NotFound"):
raise ObjectMissingError from exc
raise
body = response["Body"]
try:
value = body.read(max_bytes + 1)
finally:
body.close()
if len(value) > max_bytes:
raise RuntimeError("Stored object exceeds configured safety limit")
return value
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
def ready(self) -> bool:
self.client.head_bucket(Bucket=S3_BUCKET)
return True
def objects(self) -> Iterator[Tuple[str, datetime]]:
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET)
for page in pages:
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]
+4
View File
@@ -0,0 +1,4 @@
PIPELINE_QUALITY = {
1: {"jpeg": 85, "webp": 80},
2: {"jpeg": 80, "webp": 75},
}
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
import logging
import os
import time
import uuid
import pika
import psycopg
from opentelemetry import propagate
from opentelemetry.instrumentation.utils import suppress_instrumentation
from opentelemetry.trace import SpanKind
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues, encode_job
from .security import isoformat
from .store import Store
from .telemetry import initialize, tracer
LOGGER = logging.getLogger("messenger.publisher")
def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
with store.pool.connection() as conn:
with conn.transaction():
with suppress_instrumentation():
row = store.claim_outbox(conn)
if row is None:
return False
payload = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": isoformat(row["created_at"]),
}
parent_context = propagate.extract(
{
key: row.get(key)
for key in ("traceparent", "tracestate")
if row.get(key) is not None
}
)
with tracer().start_as_current_span(
"attachment.publish",
context=parent_context,
kind=SpanKind.PRODUCER,
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "publish"},
):
carrier = {}
propagate.inject(carrier)
payload.update(carrier)
channel.basic_publish(
exchange="",
routing_key=QUEUE_NAME,
body=encode_job(payload),
properties=pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=str(row["id"]),
type="attachment.process.v1",
),
mandatory=True,
)
conn.execute(
"UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
return True
def main() -> None:
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-publisher"))
store = Store()
store.open()
try:
while True:
connection = None
try:
connection = connect()
channel = connection.channel()
declare_queues(channel)
channel.confirm_delivery()
store.recover_expired_attachment_jobs(
int(os.getenv("JOB_MAX_ATTEMPTS", "5")),
int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS,
)
if not publish_pending(store, channel):
time.sleep(1)
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("outbox publishing cycle failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
finally:
store.close()
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import logging
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
LOGGER = logging.getLogger("messenger.reconcile")
def reconcile(
store: Store,
grace_seconds: int,
apply_changes: bool,
restore_dir: Optional[Path] = None,
):
started = time.monotonic()
metadata = store.reconciliation_snapshot()
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
found = set()
orphans = []
for key, last_modified in store.objects.objects():
found.add(key)
if key not in metadata and last_modified <= cutoff:
orphans.append(key)
missing = [
row for key, row in metadata.items()
if key not in found and row.get("reconcile_type", "original") == "original"
]
restore_candidates = []
if restore_dir is not None:
root = restore_dir.resolve(strict=True)
for row in missing:
source = restore_dir / str(row["id"])
try:
resolved = source.resolve(strict=True)
if (
resolved.parent != root
or not resolved.is_file()
or resolved.stat().st_size != row["size_bytes"]
):
continue
content = resolved.read_bytes()
except OSError:
continue
if (
len(content) == row["size_bytes"]
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
):
restore_candidates.append((row, content))
restored = 0
if apply_changes:
for key in orphans:
store.objects.delete(key)
restorable_ids = set()
for row, content in restore_candidates:
store.objects.put(row["object_key"], content, row["content_type"])
store.clear_missing_object(row["id"])
restorable_ids.add(row["id"])
restored += 1
for row in missing:
if row["id"] not in restorable_ids:
store.record_missing_object(row["id"])
for key, row in metadata.items():
if key in found and row.get("reconcile_type", "original") == "original":
store.clear_missing_object(row["id"])
unresolved_missing = len(missing) - restored
return {
"mode": "apply" if apply_changes else "dry-run",
"grace_seconds": grace_seconds,
"objects_seen": len(found),
"metadata_seen": len({str(row["id"]) for row in metadata.values()}),
"orphan_objects": len([key for key in found if key not in metadata]),
"stale_orphan_candidates": len(orphans),
"deleted_orphans": len(orphans) if apply_changes else 0,
"missing_objects": len(missing),
"unresolved_missing_objects": unresolved_missing,
"restorable_missing_objects": len(restore_candidates),
"restored_missing_objects": restored,
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
"duration_ms": round((time.monotonic() - started) * 1000, 3),
}
def main():
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
parser.add_argument(
"--restore-dir",
type=Path,
help="optional trusted backup directory with exact original bytes named by attachment UUID",
)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error("--grace-seconds must be at least 60")
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
store = Store()
store.open()
try:
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
print(json.dumps(result, sort_keys=True))
if result["unresolved_missing_objects"]:
LOGGER.error(
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
result["unresolved_missing_objects"],
)
finally:
store.close()
if __name__ == "__main__":
main()
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
import argparse
import json
import uuid
from .store import Store
def main() -> None:
parser = argparse.ArgumentParser(
description="Explicitly replay a failed attachment or start a newer pipeline version."
)
parser.add_argument("--attachment-id", required=True, type=uuid.UUID)
parser.add_argument(
"--pipeline-version",
required=True,
type=int,
help="pipeline version implemented by the deployed worker",
)
args = parser.parse_args()
if args.pipeline_version < 1:
parser.error("--pipeline-version must be positive")
store = Store()
store.open()
try:
result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version)
print(json.dumps(result, sort_keys=True))
finally:
store.close()
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
import base64
import hashlib
import hmac
import secrets
import uuid
from datetime import datetime, timezone
def isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def valid_uuid(value: str) -> bool:
try:
uuid.UUID(value)
return True
except (ValueError, TypeError, AttributeError):
return False
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
derived = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
)
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
return f"scrypt$32768$8$1${salt_text}${hash_text}"
def verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_text, hash_text = stored.split("$")
if scheme != "scrypt":
return False
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
actual = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=int(n),
r=int(r),
p=int(p),
dklen=len(expected),
maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(actual, expected)
except (ValueError, TypeError, UnicodeError):
return False
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.app import main
else:
from .app import main
if __name__ == "__main__":
main()
+848
View File
@@ -0,0 +1,848 @@
import base64
import hashlib
import hmac
import json
import logging
import math
import os
import secrets
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
import redis
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
IDEMPOTENCY_TTL_SECONDS,
SESSION_TTL_SECONDS,
)
from .errors import (
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .object_store import ObjectStore
from .pipeline_versions import PIPELINE_QUALITY
from .security import isoformat, utc_now, valid_uuid, verify_password
LOGGER = logging.getLogger("messenger.store")
def database_url() -> str:
value = os.getenv("DATABASE_URL")
if value:
return value
file_path = os.getenv("DATABASE_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
class Store:
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
self.objects = ObjectStore()
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
self.objects.ready()
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
"attachment_ids": attachment_ids or [],
}
@staticmethod
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
variants = row.get("variants") or []
public_variants = [
{
name: variant[name]
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
}
for variant in variants
]
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"].strip(),
"status": row.get("status", "queued"),
"variants": public_variants,
"error_code": row.get("error_code"),
"created_at": isoformat(row["created_at"]),
"updated_at": isoformat(row["updated_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
if not row or not verify_password(password, row["password_hash"]):
return None
return self.user(row)
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
attachment_ids: List[str],
idempotency_key: str,
canonical_body: str,
) -> Optional[Dict[str, Any]]:
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
if attachment_ids:
attachments = conn.execute(
"SELECT id, status FROM attachments "
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
(chat_id, attachment_ids),
).fetchall()
if len(attachments) != len(attachment_ids):
return None
if any(item["status"] == "failed" for item in attachments):
raise AttachmentNotReadyError
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
existing = conn.execute(
"SELECT request_hash, response, expires_at "
"FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
"FOR UPDATE",
(sender_id, chat_id, idempotency_key),
).fetchone()
if existing and existing["expires_at"] > utc_now():
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
raise IdempotencyConflictError
return existing["response"]
if existing:
conn.execute(
"DELETE FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
(sender_id, chat_id, idempotency_key),
)
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
for ordinal, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
"VALUES (%s, %s, %s)",
(row["id"], attachment_id, ordinal),
)
message = self.message(row, attachment_ids)
conn.execute(
"INSERT INTO idempotency_records "
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
(
sender_id,
chat_id,
idempotency_key,
request_hash,
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
),
)
return message
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
ids_by_message = {}
if rows:
links = conn.execute(
"SELECT message_id, attachment_id FROM message_attachments "
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
([row["id"] for row in rows],),
).fetchall()
for link in links:
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone() is not None
def create_attachment(
self,
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
content: BinaryIO,
size_bytes: int,
digest: str,
traceparent: Optional[str] = None,
tracestate: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
if not self.can_access_chat(chat_id, owner_id):
return None
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"original/{attachment_id}"
content.seek(0)
self.objects.put(staging_key, content, content_type, size_bytes)
try:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO attachments "
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"object_key, staging_key, job_id) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, variants, error_code, created_at, updated_at",
(
attachment_id,
chat_id,
owner_id,
filename,
content_type,
size_bytes,
digest,
object_key,
staging_key,
job_id,
),
).fetchone()
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"VALUES (%s, %s, %s, 1, 1, %s, %s)",
(uuid.uuid4(), job_id, attachment_id, traceparent, tracestate),
)
return self.attachment(row)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"attachment staging compensation failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
return self.attachment(row) if row else None
def download_attachment(
self, attachment_id: str, user_id: str, variant: str = "original"
) -> Optional[Tuple[str, bytes]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
if not row:
return None
if row["status"] != "ready":
raise AttachmentNotReadyError
if variant == "original":
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
else:
match = next(
(item for item in (row["variants"] or []) if item["name"] == variant),
None,
)
if match is None:
return None
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
return content_type, self.objects.get(key, size_bytes)
def claim_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
"object_key, content_type, size_bytes, sha256 "
"FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
not row
or str(row["job_id"]) != job_id
or row["pipeline_version"] != pipeline_version
or row["status"] in ("ready", "failed")
or row["attempt"] != attempt
):
return None
if row["status"] == "processing" and row["lease_until"] is not None:
if row["lease_until"] > utc_now():
return None
return conn.execute(
"UPDATE attachments SET status = 'processing', "
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
"updated_at = clock_timestamp() "
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
"size_bytes, sha256, attempt, pipeline_version",
(lease_seconds, attachment_id),
).fetchone()
def complete_attachment_job(
self,
attachment_id: str,
job_id: str,
pipeline_version: int,
attempt: int,
variants: List[Dict[str, Any]],
) -> bool:
with self.pool.connection() as conn:
with conn.transaction():
updated = conn.execute(
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
"lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
"AND status = 'processing' AND lease_until > clock_timestamp()",
(
json.dumps(variants, separators=(",", ":")),
attachment_id,
job_id,
pipeline_version,
attempt,
),
)
return updated.rowcount == 1
def fail_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
traceparent: Optional[str] = None, tracestate: Optional[str] = None,
) -> None:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT status, attempt FROM attachments "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
(attachment_id, job_id, pipeline_version),
).fetchone()
if not row or row["status"] != "processing" or row["attempt"] != attempt:
return
if retry_delay_seconds is not None and attempt < max_attempts:
next_attempt = attempt + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, error_code, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) "
"ON CONFLICT (job_id, attempt) DO NOTHING",
(
uuid.uuid4(), job_id, attachment_id, pipeline_version,
next_attempt, retry_delay_seconds, traceparent, tracestate,
),
)
else:
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
conn.execute(
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
"updated_at = clock_timestamp() WHERE id = %s",
(final_code, attachment_id),
)
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
if pipeline_version not in PIPELINE_QUALITY:
raise ValueError("pipeline version is not implemented by this deployment")
with self.pool.connection() as conn:
current = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
(attachment_id,),
).fetchone()
if current is None:
raise ValueError("attachment not found")
if pipeline_version < current["pipeline_version"]:
raise ValueError("pipeline version cannot move backwards")
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
raise ValueError("same-version replay is allowed only for failed attachments")
if current["status"] not in ("queued", "processing", "ready", "failed"):
raise ValueError("attachment cannot be replayed in its current state")
try:
content = self.objects.get(current["object_key"], current["size_bytes"])
except ObjectMissingError as object_error:
try:
content = self.objects.get(current["staging_key"], current["size_bytes"])
except ObjectMissingError:
raise object_error
if (
len(content) != current["size_bytes"]
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
):
raise ValueError("stored original does not match attachment metadata")
prior_job_id = str(current["job_id"])
next_job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}/{next_job_id}"
self.objects.put(staging_key, content, current["content_type"], len(content))
try:
with self.pool.connection() as conn:
with conn.transaction():
locked = conn.execute(
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
"WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
locked is None
or str(locked["job_id"]) != prior_job_id
or locked["pipeline_version"] != current["pipeline_version"]
or locked["attempt"] != current["attempt"]
or locked["status"] != current["status"]
):
raise RuntimeError("attachment changed while preparing replay")
conn.execute(
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s",
(next_job_id, pipeline_version, staging_key, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, 1)",
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
)
conn.execute(
"INSERT INTO attachment_pipeline_events "
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
"VALUES (%s, %s, %s, %s, %s, 'replay')",
(
attachment_id,
prior_job_id,
next_job_id,
current["pipeline_version"],
pipeline_version,
),
)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"replay staging cleanup failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
return {
"attachment_id": str(attachment_id),
"previous_job_id": prior_job_id,
"job_id": str(next_job_id),
"pipeline_version": pipeline_version,
"attempt": 1,
"status": "queued",
}
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
with self.pool.connection() as conn:
with conn.transaction():
rows = conn.execute(
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
).fetchall()
for row in rows:
if row["attempt"] >= max_attempts:
conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
continue
next_attempt = row["attempt"] + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, row["id"]),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
)
return len(rows)
def claim_outbox(self, conn):
return conn.execute(
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
"o.traceparent, o.tracestate, "
"a.created_at FROM attachment_outbox o "
"JOIN attachments a ON a.id = o.attachment_id "
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
).fetchone()
def queue_snapshot(self) -> Tuple[int, float]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, "
"COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age "
"FROM attachments WHERE status = 'queued'"
).fetchone()
return int(row["depth"]), float(row["oldest_age"])
def reconciliation_snapshot(self):
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
"size_bytes, sha256, content_type FROM attachments"
).fetchall()
expected = {}
for row in rows:
if row["status"] == "ready":
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
for variant in row["variants"] or []:
expected[variant["object_key"]] = {
**row,
"object_key": variant["object_key"],
"size_bytes": variant["size_bytes"],
"sha256": variant["sha256"],
"content_type": variant["content_type"],
"reconcile_type": "variant",
}
elif row["status"] in ("queued", "processing"):
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
for name, extension in (
("thumbnail", "jpg"),
("crop", "jpg"),
("optimized", "webp"),
):
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
return expected
def record_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
"DO UPDATE SET last_seen_at = clock_timestamp()",
(attachment_id,),
)
def clear_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"DELETE FROM reconciliation_incidents "
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
(attachment_id,),
)
+321
View File
@@ -0,0 +1,321 @@
import contextvars
import json
import logging
import os
import platform
import socket
import threading
import time
from datetime import datetime, timezone
from typing import Optional
import psycopg
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON, ParentBased, TraceIdRatioBased
from opentelemetry.instrumentation.utils import suppress_instrumentation
REQUEST_ID = contextvars.ContextVar("request_id", default="")
_lock = threading.Lock()
_runtime = {
"pool_used": 0,
"pool_waiting": 0,
"queue_depth": 0,
"queue_oldest_seconds": 0.0,
"worker_busy": 0,
"worker_capacity": 0,
"cpu": 0.0,
"memory": 0,
}
HISTOGRAM_BUCKETS = {
"messenger.http.duration": (
0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.0075, 0.008,
0.0085, 0.009, 0.0095, 0.01, 0.0125, 0.015, 0.02, 0.025,
0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
),
"messenger.jobs.duration": (
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25,
0.5, 1, 2.5, 5, 10, 30, 60,
),
"messenger.attachments.time_to_ready": (
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5,
5, 10, 30, 60, 120, 300, 600,
),
}
_provider = None
_meter = None
_service_name = "messenger"
_enabled = True
_request_counter = None
_request_duration = None
_job_counter = None
_job_duration = None
_time_to_ready = None
_queue_depth = None
_queue_oldest = None
_pool_used = None
_pool_waiting = None
_worker_busy = None
_worker_capacity = None
_process_cpu = None
_process_memory = None
class JsonLogFormatter(logging.Formatter):
converter = time.gmtime
def format(self, record):
span_context = trace.get_current_span().get_span_context()
document = {
"timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
"severity": record.levelname,
"service.name": _service_name,
"event": record.getMessage(),
"request_id": REQUEST_ID.get(),
}
if span_context.is_valid:
document["trace_id"] = format(span_context.trace_id, "032x")
document["span_id"] = format(span_context.span_id, "016x")
if record.exc_info:
document["exception.type"] = record.exc_info[0].__name__
return json.dumps(document, ensure_ascii=True, separators=(",", ":"))
def _sampler():
if os.getenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") == "always_off":
return ALWAYS_OFF
if os.getenv("OTEL_TRACES_SAMPLER", "") == "always_on":
return ALWAYS_ON
ratio = float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0"))
if not 0 <= ratio <= 1:
raise ValueError("OTEL_TRACES_SAMPLER_ARG must be between 0 and 1")
return ParentBased(TraceIdRatioBased(ratio))
def initialize(service_name: str) -> None:
global _enabled
global _service_name
global _provider, _meter
global _request_counter, _request_duration, _job_counter, _job_duration, _time_to_ready
global _queue_depth, _queue_oldest, _pool_used, _pool_waiting
global _worker_busy, _worker_capacity, _process_cpu, _process_memory
enabled = os.getenv("OTEL_SDK_DISABLED", "false").lower() != "true"
_enabled = enabled
_service_name = service_name
instance_id = os.getenv("INSTANCE_ID", socket.gethostname())
resource = Resource.create(
{
"service.name": service_name,
"service.version": os.getenv("SERVICE_VERSION", "lab6"),
"service.instance.id": instance_id,
"deployment.environment.name": os.getenv("APP_ENV", "lab"),
"host.name": socket.gethostname(),
"os.type": platform.system().lower(),
}
)
if enabled:
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
_provider = TracerProvider(resource=resource, sampler=_sampler())
_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint.rstrip("/") + "/v1/traces"),
max_queue_size=4096,
max_export_batch_size=512,
schedule_delay_millis=1000,
export_timeout_millis=10000,
)
)
trace.set_tracer_provider(_provider)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint.rstrip("/") + "/v1/metrics"),
export_interval_millis=10000,
export_timeout_millis=5000,
)
views = [
View(
instrument_name=name,
aggregation=ExplicitBucketHistogramAggregation(boundaries=buckets),
)
for name, buckets in HISTOGRAM_BUCKETS.items()
]
metrics.set_meter_provider(
MeterProvider(resource=resource, metric_readers=[reader], views=views)
)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint.rstrip("/") + "/v1/logs"),
max_queue_size=4096,
max_export_batch_size=512,
schedule_delay_millis=1000,
export_timeout_millis=10000,
)
)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otel_handler.setFormatter(JsonLogFormatter())
else:
trace.set_tracer_provider(TracerProvider(resource=resource, sampler=ALWAYS_OFF))
metrics.set_meter_provider(MeterProvider(resource=resource))
otel_handler = None
root = logging.getLogger()
root.setLevel(os.getenv("LOG_LEVEL", "INFO"))
formatter = JsonLogFormatter()
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
root.handlers[:] = [stream_handler]
if otel_handler is not None:
root.addHandler(otel_handler)
logging.getLogger("pika").setLevel(logging.WARNING)
if enabled:
try:
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
BotocoreInstrumentor().instrument()
PsycopgInstrumentor().instrument()
except ImportError as exc:
raise RuntimeError("OTel database and S3 instrumentation dependencies are required") from exc
_meter = metrics.get_meter("messenger.telemetry")
_request_counter = _meter.create_counter("messenger.http.requests", unit="{request}")
_request_duration = _meter.create_histogram("messenger.http.duration", unit="s")
_job_counter = _meter.create_counter("messenger.jobs.processed", unit="{job}")
_job_duration = _meter.create_histogram("messenger.jobs.duration", unit="s")
_time_to_ready = _meter.create_histogram("messenger.attachments.time_to_ready", unit="s")
def observe(name, attributes=None):
def callback(_options):
with _lock:
return [metrics.Observation(_runtime[name], attributes=attributes or {})]
return callback
_queue_depth = _meter.create_observable_gauge(
"messenger.queue.depth",
callbacks=[observe("queue_depth", {"queue.name": "attachments.process.v1"})],
unit="{job}",
)
_queue_oldest = _meter.create_observable_gauge(
"messenger.queue.oldest_age",
callbacks=[observe("queue_oldest_seconds", {"queue.name": "attachments.process.v1"})],
unit="s",
)
_pool_used = _meter.create_observable_gauge(
"messenger.db.pool.connections.used",
callbacks=[observe("pool_used", {"pool.name": "primary"})],
unit="{connection}",
)
_pool_waiting = _meter.create_observable_gauge(
"messenger.db.pool.requests.waiting",
callbacks=[observe("pool_waiting", {"pool.name": "primary"})],
unit="{request}",
)
_worker_busy = _meter.create_observable_gauge(
"messenger.worker.busy", callbacks=[observe("worker_busy")], unit="{worker}"
)
_worker_capacity = _meter.create_observable_gauge(
"messenger.worker.capacity", callbacks=[observe("worker_capacity")], unit="{worker}"
)
_process_cpu = _meter.create_observable_gauge(
"messenger.process.cpu.utilization", callbacks=[observe("cpu")], unit="1"
)
_process_memory = _meter.create_observable_gauge(
"messenger.process.memory.usage", callbacks=[observe("memory")], unit="By"
)
def tracer():
return trace.get_tracer("messenger.application")
def record_http_request(method: str, route: str, status: int, duration: float) -> None:
attributes = {
"http.request.method": method,
"http.route": route,
"status_class": f"{status // 100}xx",
}
if _request_counter is not None:
_request_counter.add(1, attributes)
_request_duration.record(duration, attributes)
if _enabled and not route.startswith("/health/"):
logging.getLogger("messenger.http").info(
"http request completed method=%s route=%s status_class=%s",
method,
route,
f"{status // 100}xx",
)
def record_job(outcome: str, duration: float, time_to_ready: Optional[float] = None) -> None:
attributes = {"job.type": "attachment.process.v1", "outcome": outcome}
if _job_counter is not None:
_job_counter.add(1, attributes)
_job_duration.record(duration, attributes)
if time_to_ready is not None:
_time_to_ready.record(time_to_ready, {"outcome": outcome})
def update_runtime(**values) -> None:
with _lock:
_runtime.update(values)
def start_runtime_metrics(store, worker_capacity: int = 0) -> Optional[threading.Thread]:
if not _enabled:
return None
def refresh():
try:
import psutil
process = psutil.Process()
process.cpu_percent(interval=None)
while True:
try:
stats = store.pool.get_stats()
with suppress_instrumentation():
queue_depth, oldest_age = store.queue_snapshot()
cpu = process.cpu_percent(interval=1) / 100.0
memory = process.memory_info().rss
update_runtime(
pool_used=max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)),
pool_waiting=stats.get("requests_waiting", 0),
queue_depth=queue_depth,
queue_oldest_seconds=oldest_age,
worker_capacity=worker_capacity,
cpu=cpu,
memory=memory,
)
except (psycopg.Error, OSError, RuntimeError, ValueError):
logging.getLogger("messenger.telemetry").exception("runtime metric sample failed")
time.sleep(9)
except (OSError, RuntimeError, ValueError):
logging.getLogger("messenger.telemetry").exception("runtime metric sampler stopped")
raise
thread = threading.Thread(target=refresh, name="runtime-metrics", daemon=True)
thread.start()
return thread
def worker_started() -> None:
with _lock:
_runtime["worker_busy"] += 1
def worker_stopped() -> None:
with _lock:
_runtime["worker_busy"] = max(0, _runtime["worker_busy"] - 1)
+390
View File
@@ -0,0 +1,390 @@
#!/usr/bin/env python3
import hashlib
import io
import json
import logging
import multiprocessing
import os
import random
import time
import uuid
import warnings
import zlib
from datetime import datetime, timezone
from typing import Callable, Optional
import pika
import psycopg
from opentelemetry import propagate
from opentelemetry.trace import SpanKind, Status, StatusCode
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, ImageOps, UnidentifiedImageError
from .config import MAX_IMAGE_PIXELS
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues
from .pipeline_versions import PIPELINE_QUALITY
from .store import Store
from .telemetry import (
initialize,
record_job,
start_runtime_metrics,
tracer,
worker_started,
worker_stopped,
)
LOGGER = logging.getLogger("messenger.worker")
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
class InvalidAttachment(Exception):
pass
class UnsupportedPipeline(InvalidAttachment):
pass
def _rgb_image(content: bytes) -> Image.Image:
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(content)) as source:
if source.format not in ("PNG", "JPEG"):
raise InvalidAttachment("unsupported_image")
width, height = source.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise InvalidAttachment("image_too_large")
source.load()
image = ImageOps.exif_transpose(source)
try:
if "A" in image.getbands() or "transparency" in image.info:
with image.convert("RGBA") as rgba:
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return image.convert("RGB")
finally:
if image is not source:
image.close()
except InvalidAttachment:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError,
UnidentifiedImageError, ValueError, zlib.error) as exc:
raise InvalidAttachment("invalid_image") from exc
def _encode(image: Image.Image, image_format: str, **options) -> bytes:
output = io.BytesIO()
image.save(output, format=image_format, **options)
return output.getvalue()
def _resize_within(image: Image.Image, bounds: tuple[int, int]) -> Image.Image:
width, height = image.size
scale = min(1.0, bounds[0] / width, bounds[1] / height)
size = (max(1, int(width * scale)), max(1, int(height * scale)))
if size == image.size:
return image.copy()
return image.resize(size, Image.Resampling.LANCZOS)
def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1):
quality = PIPELINE_QUALITY.get(pipeline_version)
if quality is None:
raise UnsupportedPipeline("unsupported_pipeline_version")
image = _rgb_image(content)
try:
variants = []
thumbnail = _resize_within(image, (256, 256))
try:
thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes))
finally:
thumbnail.close()
crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
try:
crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("crop", "image/jpeg", 128, 128, crop_bytes))
finally:
crop.close()
optimized = _resize_within(image, (1280, 1280))
try:
optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6)
variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes))
finally:
optimized.close()
result = []
for name, content_type, width, height, data in variants:
extension = "webp" if content_type == "image/webp" else "jpg"
key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}"
result.append(
{
"name": name,
"content_type": content_type,
"size_bytes": len(data),
"width": width,
"height": height,
"sha256": hashlib.sha256(data).hexdigest(),
"object_key": key,
"content": data,
}
)
return result
finally:
image.close()
class ProcessingTimeout(TimeoutError):
pass
def _variant_process(connection, attachment_id, content, pipeline_version, builder):
try:
connection.send(("ok", builder(attachment_id, content, pipeline_version)))
except InvalidAttachment as exc:
connection.send(("invalid", str(exc)))
except Exception as exc:
connection.send(("error", type(exc).__name__))
finally:
connection.close()
def _build_variants_bounded(
attachment_id: str,
content: bytes,
pipeline_version: int,
timeout_seconds: int,
builder=build_variants,
):
if timeout_seconds < 1:
raise ValueError("timeout_seconds must be positive")
context = multiprocessing.get_context("spawn")
receiver, sender = context.Pipe(duplex=False)
process = context.Process(
target=_variant_process,
args=(sender, attachment_id, content, pipeline_version, builder),
)
process.start()
sender.close()
try:
if not receiver.poll(timeout_seconds):
process.terminate()
process.join(timeout=2)
if process.is_alive():
process.kill()
process.join()
raise ProcessingTimeout("image processing exceeded its time budget")
try:
result, value = receiver.recv()
except EOFError as exc:
raise RuntimeError("image processing process exited without a result") from exc
process.join(timeout=2)
if process.is_alive():
process.terminate()
process.join()
if result == "invalid":
raise InvalidAttachment(value)
if result == "error":
raise RuntimeError(f"image processing failed in subprocess ({value})")
return value
finally:
receiver.close()
if process.is_alive():
process.terminate()
process.join()
def _job(payload: bytes):
value = json.loads(payload)
required = {
"schema_version", "job_id", "type", "attachment_id",
"pipeline_version", "attempt", "created_at",
}
if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}):
raise ValueError("invalid job fields")
if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1":
raise ValueError("invalid job schema")
uuid.UUID(value["job_id"])
uuid.UUID(value["attachment_id"])
if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1:
raise ValueError("invalid pipeline version")
if not isinstance(value["attempt"], int) or value["attempt"] < 1:
raise ValueError("invalid job attempt")
return value
def _process(
store: Store,
job: dict,
fault_injector: Optional[Callable[[str], None]] = None,
) -> str:
def inject(point: str) -> None:
if fault_injector is not None:
fault_injector(point)
attachment_id = job["attachment_id"]
row = store.claim_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS,
)
if row is None:
LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"])
return "skipped"
try:
inject("after_claim")
with tracer().start_as_current_span("attachment.read_staging"):
content = store.objects.get(row["staging_key"], row["size_bytes"])
if hashlib.sha256(content).hexdigest() != row["sha256"].strip():
raise InvalidAttachment("checksum_mismatch")
variants = _build_variants_bounded(
attachment_id,
content,
job["pipeline_version"],
JOB_TIMEOUT_SECONDS,
)
inject("before_original_put")
with tracer().start_as_current_span("attachment.write_original"):
store.objects.put(row["object_key"], content, row["content_type"], len(content))
inject("after_original_put")
persisted = []
for variant in variants:
data = variant.pop("content")
inject(f"before_variant_put:{variant['name']}")
with tracer().start_as_current_span("attachment.write_variant"):
store.objects.put(variant["object_key"], data, variant["content_type"], len(data))
inject(f"after_variant_put:{variant['name']}")
persisted.append(variant)
inject("before_commit")
with tracer().start_as_current_span("attachment.commit_ready"):
completed = store.complete_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
persisted,
)
if not completed:
LOGGER.warning("job lease was superseded before completion: %s", job["job_id"])
return "skipped"
inject("after_commit")
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id)
LOGGER.info("attachment processing completed: %s", attachment_id)
return "success"
except InvalidAttachment as exc:
carrier = {}
propagate.inject(carrier)
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, str(exc), None, carrier.get("traceparent"), carrier.get("tracestate"),
)
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id)
LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc)
return "failed"
except Exception as exc:
delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25)
carrier = {}
propagate.inject(carrier)
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, "processing_failed", delay, carrier.get("traceparent"), carrier.get("tracestate"),
)
LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__)
return "failed"
def handle_delivery(
store: Store,
channel: pika.adapters.blocking_connection.BlockingChannel,
method,
body: bytes,
fault_injector: Optional[Callable[[str], None]] = None,
) -> None:
try:
job = _job(body)
except (ValueError, TypeError, json.JSONDecodeError):
LOGGER.exception("invalid broker job rejected")
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
return
parent_context = propagate.extract(
{key: job[key] for key in ("traceparent", "tracestate") if key in job}
)
started = time.perf_counter()
created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00"))
with tracer().start_as_current_span(
"attachment.process",
context=parent_context,
kind=SpanKind.CONSUMER,
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "process"},
) as span:
worker_started()
try:
outcome = _process(store, job, fault_injector)
if outcome == "failed":
span.set_status(Status(StatusCode.ERROR))
if outcome != "skipped":
elapsed = time.perf_counter() - started
ready_duration = (
max(0.0, datetime.now(timezone.utc).timestamp() - created_at.timestamp())
if outcome == "success"
else None
)
record_job(outcome, elapsed, ready_duration)
finally:
worker_stopped()
if fault_injector is not None:
fault_injector("before_ack")
channel.basic_ack(delivery_tag=method.delivery_tag)
def main() -> None:
if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1:
raise RuntimeError("Worker limits must be positive")
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-worker"))
store = Store()
store.open()
start_runtime_metrics(store, WORKER_CONCURRENCY)
connection = None
try:
while True:
try:
connection = connect()
channel = connection.channel()
declare_queues(channel)
channel.basic_qos(prefetch_count=WORKER_CONCURRENCY)
def on_message(ch, method, properties, body):
handle_delivery(store, ch, method, body)
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False)
LOGGER.info("worker consuming durable attachment queue")
channel.start_consuming()
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("worker connection failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
connection = None
finally:
store.close()
if __name__ == "__main__":
main()
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
import uuid
from pathlib import Path
import sys
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202):
return client.call(
"POST",
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
expected,
actor=actor,
raw=data,
headers={"Content-Type": content_type},
schema="Attachment" if expected in (201, 202) else None,
)
def main():
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
other_chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
schema="Chat",
)["json"]
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
print("Checking accepted fixture upload.", flush=True)
attachment = upload(client, alice, chat["id"], image)["json"]
print("Checking damaged and spoofed image rejection.", flush=True)
upload(client, alice, chat["id"], b"not an image", expected=415)
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
print("Checking byte and pixel limits.", flush=True)
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
import struct
import zlib
def chunk(kind, payload):
return (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
)
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
bad_idat[-3] ^= 0xFF
damaged_idat = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", bytes(bad_idat))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], damaged_idat, expected=415)
too_many_pixels = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b""))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], too_many_pixels, expected=413)
print("Checking filename and cross-chat authorization.", flush=True)
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
upload(client, eve, chat["id"], image, expected=404)
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
client.call(
"POST",
"/api/v1/chats/" + other_chat["id"] + "/messages",
404,
actor=alice,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": "", "attachment_ids": [attachment["id"]]},
)
client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
400,
actor=bob,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": ""},
)
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
if __name__ == "__main__":
run(main)
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Prove a concurrent idempotency-key race through the load balancer."""
from concurrent.futures import ThreadPoolExecutor
import threading
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def main():
args = arguments("Concurrent shared-idempotency integration check")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = "race-" + uuid.uuid4().hex
text = "concurrent-" + uuid.uuid4().hex
count = 12
barrier = threading.Barrier(count)
def send_once(_):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=10)
return worker.message(alice, chat, text, key=key)
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(send_once, range(count)))
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
messages = [response["json"] for response in responses]
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]
matches = [message for message in page["items"] if message["text"] == text]
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
if __name__ == "__main__":
run(main)
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Verify an upload committed while the publisher is stopped is processed after restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify committed attachment outbox intent survives publisher downtime")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
compose("stop", "publisher")
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted into queued state")
finally:
compose("start", "publisher")
ready = client.wait_ready(alice, attachment)
stored_original = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
thumbnail = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
variant = next(item for item in ready["variants"] if item["name"] == "thumbnail")
check(stored_original == original, "recovered job changed original bytes")
check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch")
print(
json.dumps(
{
"result": "PASS",
"accepted_status_before_publisher_restart": attachment["status"],
"final_status": ready["status"],
"attachment_id": attachment["id"],
"original_sha256": hashlib.sha256(stored_original).hexdigest(),
"thumbnail_sha256": variant["sha256"],
"recovered_without_manual_republish": True,
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+485
View File
@@ -0,0 +1,485 @@
#!/usr/bin/env python3
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
import hashlib
import json
import os
from pathlib import Path
import sys
import time
import uuid
from datetime import datetime, timezone
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import pika
from src.publisher import publish_pending
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
JOB_ID = "00000000-0000-4000-8000-000000000002"
class SimulatedCrash(BaseException):
pass
class FakeObjects:
def __init__(self, content):
self.data = {"staging/" + ATTACHMENT_ID: content}
self.puts = []
def get(self, key, max_bytes):
return self.data[key]
def put(self, key, content, content_type, size_bytes=None):
value = content if isinstance(content, bytes) else content.read()
assert len(value) == size_bytes
self.data[key] = value
self.puts.append((key, hashlib.sha256(value).hexdigest()))
def delete(self, key):
self.data.pop(key, None)
class FakeStore:
def __init__(self, content):
self.objects = FakeObjects(content)
self.row = {
"id": ATTACHMENT_ID,
"job_id": JOB_ID,
"pipeline_version": 1,
"attempt": 1,
"status": "queued",
"lease_until": 0.0,
"staging_key": "staging/" + ATTACHMENT_ID,
"object_key": "original/" + ATTACHMENT_ID,
"content_type": "image/png",
"size_bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
"variants": [],
}
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] in ("ready", "failed")
):
return None
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
return None
row["status"] = "processing"
row["lease_until"] = time.monotonic() + lease_seconds
return dict(row)
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] != "processing"
or row["lease_until"] <= time.monotonic()
):
return False
row["status"] = "ready"
row["lease_until"] = 0.0
row["variants"] = variants
return True
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
row = self.row
if (
row["id"] == attachment_id
and row["job_id"] == job_id
and row["pipeline_version"] == version
and row["attempt"] == attempt
and row["status"] == "processing"
):
row["status"] = "failed" if delay is None else "queued"
row["lease_until"] = 0.0
row["error_code"] = error_code
if delay is not None:
row["attempt"] += 1
def recover_expired(self):
if self.row["status"] == "processing":
self.row["attempt"] += 1
self.row["status"] = "queued"
self.row["lease_until"] = 0.0
def replay(self, version):
row = self.row
previous_job = row["job_id"]
row["job_id"] = str(uuid.uuid4())
row["pipeline_version"] = version
row["attempt"] = 1
row["status"] = "queued"
row["lease_until"] = 0.0
row["variants"] = []
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
self.objects.put(
row["staging_key"],
self.objects.data[row["object_key"]],
row["content_type"],
row["size_bytes"],
)
return {
"schema_version": 1,
"job_id": row["job_id"],
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
"previous_job_id": previous_job,
}
class FakeChannel:
def __init__(self):
self.acked = []
self.nacked = []
def basic_ack(self, delivery_tag):
self.acked.append(delivery_tag)
def basic_nack(self, delivery_tag, requeue):
self.nacked.append((delivery_tag, requeue))
class FakeMethod:
def __init__(self, delivery_tag):
self.delivery_tag = delivery_tag
def job_body(job_id=JOB_ID, version=1, attempt=1):
return json.dumps(
{
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": attempt,
"created_at": "2026-01-01T00:00:00Z",
}
).encode()
def crash_at(expected):
def inject(point):
if point == expected:
raise SimulatedCrash(point)
return inject
def slow_builder(attachment_id, content, pipeline_version):
time.sleep(5)
def test_duplicate_and_ack_loss(content):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(1), job_body(),
fault_injector=crash_at("before_ack"),
)
raise AssertionError("fault injection did not crash before ack")
except SimulatedCrash:
pass
assert store.row["status"] == "ready"
first_write_count = len(store.objects.puts)
assert channel.acked == []
handle_delivery(store, channel, FakeMethod(2), job_body())
assert channel.acked == [2]
assert len(store.objects.puts) == first_write_count
assert store.row["status"] == "ready"
return {
"scenario": "duplicate_delivery_after_commit_before_ack",
"deliveries": 2,
"effective_object_writes": first_write_count,
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
}
def test_crash_before_and_after_side_effect(content, crash_point):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(3), job_body(),
fault_injector=crash_at(crash_point),
)
raise AssertionError(f"fault injection did not crash at {crash_point}")
except SimulatedCrash:
pass
assert channel.acked == []
store.recover_expired()
retry = job_body(attempt=2)
handle_delivery(store, channel, FakeMethod(4), retry)
assert store.row["status"] == "ready"
assert store.row["attempt"] == 2
assert channel.acked == [4]
assert store.objects.data["original/" + ATTACHMENT_ID] == content
return {
"scenario": "crash_" + crash_point,
"attempts": store.row["attempt"],
"terminal_status": store.row["status"],
}
def test_reordered_attempts(content):
store = FakeStore(content)
store.row["attempt"] = 2
channel = FakeChannel()
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
writes_after_current = len(store.objects.puts)
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
assert store.row["status"] == "ready"
assert len(store.objects.puts) == writes_after_current
assert channel.acked == [5, 6]
return {
"scenario": "out_of_order_attempt_delivery",
"current_attempt": store.row["attempt"],
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
}
def test_stale_pipeline_cannot_publish_newer_result(content):
store = FakeStore(content)
channel = FakeChannel()
replayed = []
def supersede_after_first_variant(point):
if point == "after_variant_put:thumbnail" and not replayed:
replayed.append(store.replay(2))
handle_delivery(
store, channel, FakeMethod(7), job_body(),
fault_injector=supersede_after_first_variant,
)
assert store.row["status"] == "queued"
assert store.row["pipeline_version"] == 2
assert channel.acked == [7]
current_job = replayed[0]
handle_delivery(
store,
channel,
FakeMethod(8),
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
)
assert store.row["status"] == "ready"
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
assert channel.acked == [7, 8]
return {
"scenario": "stale_pipeline_after_newer_version",
"current_pipeline_version": store.row["pipeline_version"],
"published_variant_versions": sorted(
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
),
}
class FakeTransaction:
def __init__(self, store):
self.store = store
def __enter__(self):
self.published = False
self.store.current_transaction = self
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None and self.published:
self.store.outbox_published = True
self.store.current_transaction = None
return False
class FakeConnection:
def __init__(self, store):
self.store = store
def transaction(self):
return FakeTransaction(self.store)
def execute(self, query, params):
assert "UPDATE attachment_outbox" in query
assert params == ("outbox-row",)
self.store.current_transaction.published = True
class FakePool:
def __init__(self, store):
self.store = store
def connection(self):
connection = FakeConnection(self.store)
class ConnectionContext:
def __enter__(self_inner):
return connection
def __exit__(self_inner, exc_type, exc_value, traceback):
return False
return ConnectionContext()
class FakeOutboxStore:
def __init__(self):
self.pool = FakePool(self)
self.outbox_published = False
self.current_transaction = None
def claim_outbox(self, connection):
if self.outbox_published:
return None
return {
"id": "outbox-row",
"job_id": JOB_ID,
"attachment_id": ATTACHMENT_ID,
"pipeline_version": 1,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
}
class AmbiguousConfirmChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
self.messages.append(message)
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
class FailBeforePublishChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
self.messages.append(message)
def test_committed_outbox_survives_publisher_crash():
store = FakeOutboxStore()
channel = FailBeforePublishChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher crash was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert channel.messages == []
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 1
return {
"scenario": "publisher_crash_before_publish",
"published_after_restart": len(channel.messages),
"outbox_recovered_after_restart": store.outbox_published,
}
def test_outbox_confirm_gap_republishes_same_job():
store = FakeOutboxStore()
channel = AmbiguousConfirmChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher confirm fault was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 2
first, second = channel.messages
assert first["body"] == second["body"]
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
return {
"scenario": "publish_confirm_lost_after_broker_accept",
"deliveries": len(channel.messages),
"same_message_id": True,
}
def test_image_processing_has_a_hard_timeout():
started = time.monotonic()
try:
_build_variants_bounded(
ATTACHMENT_ID,
b"bounded timeout test",
1,
timeout_seconds=1,
builder=slow_builder,
)
raise AssertionError("slow transform exceeded its time budget without termination")
except ProcessingTimeout:
pass
elapsed = time.monotonic() - started
assert elapsed < 4
return {
"scenario": "transform_timeout_terminated",
"configured_timeout_seconds": 1,
"observed_seconds": round(elapsed, 3),
}
def main():
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
results = [
test_duplicate_and_ack_loss(content),
test_crash_before_and_after_side_effect(content, "after_claim"),
test_crash_before_and_after_side_effect(content, "after_original_put"),
test_reordered_attempts(content),
test_stale_pipeline_cannot_publish_newer_result(content),
test_committed_outbox_survives_publisher_crash(),
test_outbox_confirm_gap_republishes_same_job(),
test_image_processing_has_a_hard_timeout(),
]
report = {
"result": "PASS",
"track": "2",
"fixture": "tests/fixtures/sample.png",
"scenarios": results,
"invariants": {
"confirmed_work_eventually_ready": True,
"duplicate_effects_create_no_second_variant_set": True,
"stale_attempt_writes_no_effects": True,
"stale_pipeline_cannot_publish_current_metadata": True,
"ambiguous_confirm_republishes_stable_job": True,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
},
}
evidence = ROOT / "evidence" / "pipeline-faults.json"
evidence.parent.mkdir(exist_ok=True)
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(report, sort_keys=True))
print(f"Evidence written to {evidence.relative_to(ROOT)}")
if __name__ == "__main__":
main()
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Verify explicit replay changes the deployed pipeline while preserving the original."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
MARK_FAILED_CODE = """
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
updated = conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', "
"variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'",
(sys.argv[1],),
)
if updated.rowcount != 1:
raise RuntimeError("test attachment was not ready for failed replay injection")
finally:
store.close()
"""
AUDIT_CODE = """
import json
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
rows = conn.execute(
"SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events "
"WHERE attachment_id = %s ORDER BY id",
(sys.argv[1],),
).fetchall()
print(json.dumps(rows, sort_keys=True))
finally:
store.close()
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def replay(container_id, attachment_id, version):
result = docker_exec(
container_id,
"python3",
"-m",
"src.replay_attachment",
"--attachment-id",
attachment_id,
"--pipeline-version",
str(version),
)
return json.loads(result.stdout)
def main():
args = arguments("Verify versioned attachment replay against the running Compose stack")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready_v1 = client.wait_ready(alice, attachment)
v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail")
v1_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch")
containers = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout.split()
check(bool(containers), "no running API container found")
replay_v2 = replay(containers[0], attachment["id"], 2)
check(replay_v2["status"] == "queued", "replay was not queued")
check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2")
check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id")
ready_v2 = client.wait_ready(alice, ready_v1)
v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail")
v2_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding")
check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch")
check(original_bytes == original, "pipeline replay changed the original file")
docker_exec(
containers[0],
"python3",
"-c",
MARK_FAILED_CODE,
attachment["id"],
)
failed_replay = replay(containers[0], attachment["id"], 2)
check(failed_replay["status"] == "queued", "failed attachment replay was not queued")
check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version")
check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id")
ready_replayed = client.wait_ready(alice, ready_v2)
audit = json.loads(
docker_exec(
containers[0],
"python3",
"-c",
AUDIT_CODE,
attachment["id"],
).stdout
)
check(
[(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit]
== [(1, 2, "replay"), (2, 2, "replay")],
"pipeline replay audit trail is incomplete",
)
check(ready_replayed["status"] == "ready", "failed attachment did not recover")
print(
json.dumps(
{
"result": "PASS",
"previous_job_id": replay_v2["previous_job_id"],
"job_id": replay_v2["job_id"],
"pipeline_version": replay_v2["pipeline_version"],
"failed_replay_job_id": failed_replay["job_id"],
"audited_replays": len(audit),
"v1_thumbnail_sha256": v1_thumbnail["sha256"],
"v2_thumbnail_sha256": v2_thumbnail["sha256"],
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
import os
import hashlib
import sys
import tempfile
from pathlib import Path
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from datetime import datetime, timedelta, timezone
from src.reconcile import reconcile
class FakeObjects:
def __init__(self, items):
self.items = items
self.deleted = []
def objects(self):
return iter(self.items)
def delete(self, key):
self.deleted.append(key)
self.items = [(name, date) for name, date in self.items if name != key]
def put(self, key, content, content_type):
self.items.append((key, datetime.now(timezone.utc)))
class FakeStore:
def __init__(self):
now = datetime.now(timezone.utc)
self.metadata = {
"original/present": {"id": "present-id", "object_key": "original/present"},
"original/missing": {
"id": "missing-id",
"object_key": "original/missing",
"size_bytes": 4,
"sha256": hashlib.sha256(b"test").hexdigest(),
"content_type": "image/png",
},
}
self.objects = FakeObjects(
[
("original/present", now),
("original/stale-orphan", now - timedelta(hours=2)),
("original/new-orphan", now - timedelta(minutes=10)),
]
)
self.missing_incidents = set()
self.cleared_incidents = set()
def reconciliation_snapshot(self):
return self.metadata
def record_missing_object(self, attachment_id):
self.missing_incidents.add(attachment_id)
def clear_missing_object(self, attachment_id):
self.missing_incidents.discard(attachment_id)
self.cleared_incidents.add(attachment_id)
def main():
store = FakeStore()
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
assert dry_run["stale_orphan_candidates"] == 1
assert dry_run["missing_objects"] == 1
assert dry_run["deleted_orphans"] == 0
assert store.objects.deleted == [] and store.missing_incidents == set()
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
assert store.objects.deleted == ["original/stale-orphan"]
assert "original/new-orphan" not in store.objects.deleted
assert store.missing_incidents == {"missing-id"}
assert store.cleared_incidents == {"present-id"}
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
with tempfile.TemporaryDirectory() as backup:
restore_dir = Path(backup)
(restore_dir / "missing-id").write_bytes(b"test")
preview = reconcile(store, 3600, False, restore_dir)
assert preview["restorable_missing_objects"] == 1
assert store.missing_incidents == {"missing-id"}
restored = reconcile(store, 3600, True, restore_dir)
assert restored["restored_missing_objects"] == 1
assert store.missing_incidents == set()
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
assert store.objects.deleted == ["original/stale-orphan"]
assert store.missing_incidents == set()
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
if __name__ == "__main__":
main()
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
import tempfile
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
STORE_CODE = """
import sys
from pathlib import Path
from src.store import Store
store = Store()
store.open()
try:
action = sys.argv[1]
if action == "key":
attachment_id = sys.argv[2]
matches = [
key for key, row in store.reconciliation_snapshot().items()
if str(row["id"]) == attachment_id and row["reconcile_type"] == "original"
]
if len(matches) != 1:
raise RuntimeError("attachment metadata lookup failed")
print(matches[0])
elif action == "put":
key, file_path = sys.argv[2:4]
content = Path(file_path).read_bytes()
store.objects.put(key, content, "image/png", len(content))
elif action == "delete":
store.objects.delete(sys.argv[2])
elif action == "exists":
print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no")
else:
raise RuntimeError("unsupported test operation")
finally:
store.close()
"""
PRIVATE_CHECK_CODE = """
import socket
import sys
import urllib.error
import urllib.request
try:
socket.getaddrinfo("s3-core", 8888)
except socket.gaierror:
print("PASS: SeaweedFS master/filer network is isolated")
else:
raise RuntimeError("API network can resolve the SeaweedFS core service")
for port in (8181, 9101):
try:
socket.create_connection(("s3", port), timeout=1).close()
except OSError:
continue
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
try:
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
except urllib.error.HTTPError as error:
if error.code == 403:
print("PASS: unauthenticated bucket access denied")
else:
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
else:
raise RuntimeError("private bucket allowed an unauthenticated GET")
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def reconciliation(container_id, *args):
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
return json.loads(result.stdout)
def object_exists(container_id, key):
return docker_exec(
container_id, "python3", "-c", STORE_CODE, "exists", key
).stdout.strip() == "yes"
def main():
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, content = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
containers = result.stdout.split()
check(len(containers) >= 2, "at least two running API replicas are required")
container_id = containers[0]
key = docker_exec(
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
).stdout.strip()
orphan_key = "original/" + str(uuid.uuid4())
with tempfile.TemporaryDirectory() as backup:
source = Path(backup) / "original.png"
source.write_bytes(content)
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
recovery_file = recovery_dir + "/" + attachment["id"]
orphan_created = False
orphan_removed = False
missing_deleted = False
missing_restored = False
try:
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
orphan_created = True
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
missing_deleted = True
preview = reconciliation(container_id)
check(
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
"dry-run missed injected inconsistencies",
)
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
immediate_apply = reconciliation(container_id, "--apply")
check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period")
docker_exec(container_id, "mkdir", "-p", recovery_dir)
subprocess.run(
["docker", "cp", str(source), container_id + ":" + recovery_file],
check=True,
)
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
missing_restored = True
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period")
print("Waiting for the 60-second orphan grace period.")
time.sleep(61)
cleanup_preview = reconciliation(container_id)
check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace")
cleaned = reconciliation(container_id, "--apply")
orphan_removed = not object_exists(container_id, orphan_key)
check(orphan_removed, "injected stale orphan was not removed")
final_scan = reconciliation(container_id, "--apply")
check(
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
"repeat scan is not clean",
)
finally:
if missing_deleted and not missing_restored:
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
if orphan_created and not orphan_removed:
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
check("PASS:" in private, "unauthenticated S3 access check failed")
downloaded = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
print(
json.dumps(
{
"result": "PASS",
"dataset_objects_scanned": final_scan["objects_seen"],
"metadata_records_scanned": final_scan["metadata_seen"],
"scan_duration_ms": final_scan["duration_ms"],
"orphan_detected_and_removed": cleaned["deleted_orphans"],
"missing_detected_and_restored": restored["restored_missing_objects"],
"fresh_orphan_preserved_until_grace": True,
"message_attachment_download_checksum": "unchanged",
"private_bucket": "anonymous request denied",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Local transformation checks for the lab5 image worker."""
import hashlib
import io
import os
from pathlib import Path
import sys
os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret")
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from src.worker import build_variants
def main():
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
attachment_id = "00000000-0000-4000-8000-000000000001"
variants = build_variants(attachment_id, original, pipeline_version=1)
by_name = {item["name"]: item for item in variants}
assert set(by_name) == {"thumbnail", "crop", "optimized"}
assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160)
assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128)
assert by_name["optimized"]["content_type"] == "image/webp"
for item in variants:
content = item["content"]
assert len(content) == item["size_bytes"]
assert hashlib.sha256(content).hexdigest() == item["sha256"]
with Image.open(io.BytesIO(content)) as image:
assert image.size == (item["width"], item["height"])
newer = build_variants(attachment_id, original, pipeline_version=2)
assert all(f"/v2/" in item["object_key"] for item in newer)
assert all(f"/v1/" in item["object_key"] for item in variants)
assert by_name["thumbnail"]["sha256"] != {
item["name"]: item for item in newer
}["thumbnail"]["sha256"]
odd_ratio = io.BytesIO()
Image.new("RGB", (319, 200), "blue").save(odd_ratio, format="PNG")
odd_variants = {
item["name"]: item for item in build_variants(attachment_id, odd_ratio.getvalue())
}
assert (odd_variants["thumbnail"]["width"], odd_variants["thumbnail"]["height"]) == (256, 160)
small_image = io.BytesIO()
Image.new("RGB", (100, 50), "red").save(small_image, format="PNG")
small_variants = {
item["name"]: item for item in build_variants(attachment_id, small_image.getvalue())
}
assert (small_variants["thumbnail"]["width"], small_variants["thumbnail"]["height"]) == (100, 50)
assert (small_variants["optimized"]["width"], small_variants["optimized"]["height"]) == (100, 50)
print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums")
if __name__ == "__main__":
main()
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Verify queued uploads and message links survive an API/worker restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify queued attachment recovery after API and worker restart")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
compose("stop", "worker")
worker_stopped = True
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted while worker was stopped")
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message")
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
409,
actor=bob,
)
compose("restart", "api")
compose("start", "worker")
worker_stopped = False
client.call("GET", "/health/ready", schema="Health")
ready = client.wait_ready(bob, attachment)
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(original_bytes == original, "recovered worker changed original bytes")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages",
actor=bob,
schema="MessagePage",
)["json"]
recovered_message = next(item for item in page["items"] if item["id"] == message["id"])
check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link")
print(
json.dumps(
{
"result": "PASS",
"initial_status": attachment["status"],
"final_status": ready["status"],
"message_attachment_link_survived": True,
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
finally:
if worker_stopped:
compose("start", "worker")
if __name__ == "__main__":
run(main)
+360
View File
@@ -0,0 +1,360 @@
#!/usr/bin/env python3
"""Repeat a paced synthetic message workload with telemetry off and sampled."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import platform
import re
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
def command(args, capture=True):
result = subprocess.run(
args,
cwd=ROOT,
check=True,
text=True,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
)
return result.stdout.strip() if capture else ""
def set_env_value(text, name, value):
lines = text.splitlines()
prefix = name + "="
for index, line in enumerate(lines):
if line.startswith(prefix):
lines[index] = prefix + value
break
else:
lines.append(prefix + value)
return "\n".join(lines) + "\n"
def parse_env(text):
values = {}
for line in text.splitlines():
if line and not line.startswith("#") and "=" in line:
name, value = line.split("=", 1)
values[name] = value
return values
def wait_ready(url, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url + "/health/ready", timeout=2) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError, OSError):
pass
time.sleep(2)
raise RuntimeError("API did not become ready after the sampling mode change")
def parse_size(value):
match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value)
if not match:
return None
factors = {
"B": 1,
"kB": 1000,
"MB": 1000**2,
"GB": 1000**3,
"KiB": 1024,
"MiB": 1024**2,
"GiB": 1024**3,
}
return int(float(match.group(1)) * factors[match.group(2)])
def docker_resources():
rows = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}};{{.CPUPerc}};{{.MemUsage}}",
]
)
cpu_total = 0.0
memory_total = 0
found = False
for row in rows.splitlines():
fields = row.split(";")
if len(fields) != 3 or not any(
name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-")
):
continue
cpu = re.match(r"\s*([\d.]+)%", fields[1])
memory = parse_size(fields[2].split("/", 1)[0])
if cpu and memory is not None:
cpu_total += float(cpu.group(1))
memory_total += memory
found = True
return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None
def storage_sizes(compose):
paths = {
"prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]),
"loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]),
"tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]),
}
result = {}
for name, args in paths.items():
output = command(compose + args)
result[name] = int(output.split()[0])
return result
def tempo_message_traces(compose, start, end):
url = (
"http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000"
"&tags=service.name%%3Dmessenger-api" % (start, end)
)
output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url])
data = json.loads(output)
return [
item
for item in data.get("traces", [])
if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"
]
def mean_spans(compose, traces, sample_limit=20):
counts = []
for item in traces[:sample_limit]:
output = command(
compose
+ [
"exec",
"-T",
"tempo",
"wget",
"-qO-",
"http://127.0.0.1:3200/api/traces/" + item["traceID"],
]
)
data = json.loads(output)
count = sum(
len(scope.get("spans", []))
for batch in data.get("batches", [])
for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", []))
)
counts.append(count)
return round(sum(counts) / len(counts), 2) if counts else 0.0
def run_load(base_url, duration, concurrency, target_rps, alice, chat):
from client import Client, Failure
barrier = threading.Barrier(concurrency + 1)
ticket_lock = threading.Lock()
ticket = 0
latencies = []
failures = 0
results_lock = threading.Lock()
start_clock = time.monotonic()
start_epoch = int(time.time())
deadline = start_clock + duration
def worker(worker_id):
nonlocal ticket, failures
client = Client(base_url, timeout=5)
local_latencies = []
local_failures = 0
barrier.wait()
while True:
with ticket_lock:
current = ticket
ticket += 1
due = start_clock + current / target_rps
if due >= deadline:
break
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
started = time.monotonic()
try:
client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current))
local_latencies.append(time.monotonic() - started)
except Failure:
local_failures += 1
with results_lock:
latencies.extend(local_latencies)
failures += local_failures
samples = []
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)]
barrier.wait()
while any(not future.done() for future in futures):
sample = docker_resources()
if sample is not None:
samples.append(sample)
time.sleep(2)
for future in futures:
future.result()
elapsed = time.monotonic() - start_clock
sorted_latencies = sorted(latencies)
def percentile(value):
if not sorted_latencies:
return None
index = max(0, math.ceil(value * len(sorted_latencies)) - 1)
return round(sorted_latencies[index] * 1000, 3)
total = len(sorted_latencies) + failures
report = {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": duration,
"target_rps": target_rps,
"elapsed_seconds": round(elapsed, 3),
"concurrency": concurrency,
"successful_requests": len(sorted_latencies),
"failed_requests": failures,
"error_ratio": round(failures / total, 6) if total else None,
"successful_rps": round(len(sorted_latencies) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(0.50),
"p95": percentile(0.95),
"p99": percentile(0.99),
},
"docker_resource_samples": len(samples),
}
if samples:
report["docker_resources"] = {
"mean_cpu_percent_sum": round(
sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2
),
"peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples),
"peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples),
"services": ["api", "worker", "publisher", "otel-collector"],
}
return report, start_epoch, int(time.time())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--base-url", default=None)
parser.add_argument("--duration", type=int, default=15)
parser.add_argument("--concurrency", type=int, default=2)
parser.add_argument("--target-rps", type=int, default=20)
parser.add_argument("--warmup-seconds", type=int, default=5)
parser.add_argument("--output", default="evidence/track1-observability-cost.json")
args = parser.parse_args()
if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1:
parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive")
env_path = ROOT / args.env_file
if not env_path.is_file():
parser.error("create the local .env from .env.example and generate secrets first")
original = env_path.read_text(encoding="utf-8")
values = parse_env(original)
base_url = args.base_url or "http://localhost:" + values.get("APP_PORT", "8080")
compose = ["docker", "compose", "--env-file", str(env_path)]
output_path = ROOT / args.output
output_path.parent.mkdir(parents=True, exist_ok=True)
results = []
configurations = [
("off", "true", "1.0"),
("sample-100-percent", "false", "1.0"),
("sample-25-percent", "false", "0.25"),
("sample-10-percent", "false", "0.1"),
]
try:
for name, disabled, ratio in configurations:
changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled)
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio)
env_path.write_text(changed, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
wait_ready(base_url)
time.sleep(args.warmup_seconds)
from client import Client
alice, _, _, chat = Client(base_url, timeout=10).bootstrap()
time.sleep(2)
before = storage_sizes(compose)
load, start, end = run_load(base_url, args.duration, args.concurrency, args.target_rps, alice, chat)
time.sleep(15)
after = storage_sizes(compose)
traces = tempo_message_traces(compose, start - 5, end + 20)
spans_per_trace = mean_spans(compose, traces)
results.append(
{
"configuration": name,
"sdk_disabled": disabled == "true",
"trace_sampling_ratio": float(ratio),
"load": load,
"window_unix_seconds": {"start": start, "end": end},
"sampled_message_trace_count": len(traces),
"estimated_spans_per_message_trace": spans_per_trace,
"persistent_storage_bytes_delta": {
signal: after[signal] - before[signal] for signal in before
},
}
)
baseline = next(
item["estimated_spans_per_message_trace"]
for item in results
if item["configuration"] == "sample-100-percent"
)
for item in results:
expected_spans = item["load"]["successful_requests"] * baseline
observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"]
item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2)
report = {
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
"method": {
"duration_seconds": args.duration,
"concurrency": args.concurrency,
"target_rps": args.target_rps,
"warmup_seconds": args.warmup_seconds,
"telemetry_flush_wait_seconds": 15,
"hardware": {
"platform": platform.platform(),
"processor": platform.processor() or "unspecified",
"logical_cpus": os.cpu_count(),
},
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.",
},
"limitations": [
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
"Database history grows between modes; all modes use identical duration and concurrency.",
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.",
],
"results": results,
"balanced_configuration": "sample-25-percent",
}
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
finally:
env_path.write_text(original, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
if __name__ == "__main__":
main()
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
import threading
import time
import uuid
from io import BytesIO
from pathlib import Path
import sys
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
def usage_bytes(line):
match = USAGE_RE.search(line)
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
def api_containers():
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
ids = result.stdout.split()
check(len(ids) >= 2, "at least two running API containers are required")
return ids
def sample_once(ids):
output = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
check=True,
capture_output=True,
text=True,
).stdout.splitlines()
values = [usage_bytes(line) for line in output]
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
return max(values)
def main():
args = arguments("Bounded-memory ten-client upload experiment")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
width = height = 1400
content = BytesIO()
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
content, format="PNG", compress_level=0
)
payload = content.getvalue()
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
ids = api_containers()
baseline = sample_once(ids)
samples = []
stats_process = subprocess.Popen(
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
def sampler():
if stats_process.stdout is None:
return
for line in stats_process.stdout:
value = usage_bytes(line)
if value:
samples.append(value)
sampling_thread = threading.Thread(target=sampler, daemon=True)
sampling_thread.start()
count = 10
barrier = threading.Barrier(count)
def upload(index):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=15)
return worker.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
202,
actor=alice,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(upload, range(count)))
finally:
stopped_by_test = False
if stats_process.poll() is None:
stats_process.kill()
stopped_by_test = True
try:
stats_process.wait(timeout=5)
except subprocess.TimeoutExpired:
stats_process.kill()
stats_process.wait(timeout=5)
sampling_thread.join(timeout=5)
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
check(
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
"docker stats sampling process failed (exit "
+ str(stats_process.returncode)
+ "): "
+ stats_error.strip()[:300],
)
check(bool(samples), "docker stats produced no memory samples")
peak = max([baseline] + samples)
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
print(
json.dumps(
{
"uploads": count,
"concurrency": count,
"bytes_per_upload": len(payload),
"duration_seconds": round(time.monotonic() - started, 3),
"api_containers": len(ids),
"sample_count": len(samples) // len(ids),
"baseline_max_container_bytes": baseline,
"peak_max_container_bytes": peak,
"peak_delta_bytes": max(0, peak - baseline),
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)