feat(observability): add bounded OTLP signals, causal jobs and SLO rules

This commit is contained in:
2026-11-20 15:41:53 +03:00
parent 470d9d6f1e
commit 8f3ff33b2d
76 changed files with 9995 additions and 33 deletions
+17
View File
@@ -18,3 +18,20 @@ __pycache__
node_modules
evidence
.DS_Store
**/__pycache__
**/*.py[cod]
.work
.pytest_cache
.ruff_cache
tests
student-tests
research
scripts
contracts
*.md
Makefile
compose.yaml
.work*
student-unit
+14 -9
View File
@@ -13,18 +13,23 @@ API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_BUCKET=messenger
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
PUBLIC_PORT=8086
API_CPUS=0.5
API_MEMORY=768m
DB_POOL_SIZE=10
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=23
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
OTEL_ENABLED=1
GRAFANA_PORT=3006
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.work/
.work-*.txt
.pytest_cache/
.ruff_cache/
+22
View File
@@ -0,0 +1,22 @@
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+23
View File
@@ -28,3 +28,26 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
BASE_URL = http://127.0.0.1:8086
.PHONY: bootstrap up down lint test-student verify
bootstrap:
$(PYTHON) scripts/bootstrap.py
up: bootstrap
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
down:
docker compose down
lint:
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
.PHONY: unit reconcile
unit:
PYTHONPATH=. $(UV) run --locked pytest student-unit -q
reconcile:
docker compose run --rm --no-deps reconciler python -m src.reconcile
+59 -24
View File
@@ -1,37 +1,72 @@
# Отчёт — лабораторная 6
# Лабораторная6 — Маслов, БВТ2301
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Цель:5, трек2 — SLO и обнаружение инцидентов. Собственная ЛР5 перенесена
поверх текущего skeleton с независимой историей; contracts/tests/README
и исходные check/validate сохранены. Эта версия — начало runtime-проверок:
результаты исследования, clean clone и публикация пока не подтверждены.
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Desktop с Compose v2, Git и uv0.13.7. Python3.14.7;
SDK OpenTelemetry1.45.2, Collector0.165.0, Prometheus3.16.0, Loki3.7.10,
Tempo3.1.1, Grafana13.3.1. Lock и digest в Git; MinIO/mc собираются из
точных upstream commits собственной ЛР5 с проверкой GoSumDB.
## Архитектура и решения
```sh
uv sync --locked
make bootstrap
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
make check validate lint unit PYTHON=.venv/bin/python
NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test PYTHON=.venv/bin/python
make test-student
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
API http://127.0.0.1:8086, Grafana http://127.0.0.1:3006, user admin,
локальный пароль secrets/grafana_password. Bootstrap сохраняет прежние файлы
и не печатает значения. Datasources/dashboard импортируются автоматически.
Остальные telemetry ports внутренние. docker compose down сохраняет volumes.
Fault hooks включаются только явным compose.test.yaml.
## Доказательства критериев
## Архитектура и измерения
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
API/worker/relay/observer → OTLP/HTTP → Collector → Prometheus/Loki/Tempo;
Grafana — интерфейс запросов, а не хранилище трёх сигналов. История backend
сохраняется в отдельных томах. Сессии/ACL/идемпотентность/S3/outbox/worker
сохранены. Migration005 добавляет W3C context и однократные SLI-решения jobs.
Контекст идёт от server span в принимающую транзакцию и outbox, через relay
producer span к consumer и дочернему процессу с S3/DB spans, включая retry.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Логи имеют allowlist полей. Тела/пароли/Authorization/SQL/rawURL не экспортируются;
exception messages/stack traces не добавляются к spans. Метрики используют
route template, ограниченный HTTP method и status class, job type/outcome.
Технические IDs допустимы только в логах/трассах синтетических данных.
## Эксперименты и отказы
Dashboard содержит18панелей. p50/p95/p99 вычисляются из общих histogram buckets;
SDK HTTP duration измеряет ASGI response. API availability SLI берётся с
HAProxy business backend, включая ingress503; health идёт отдельным backend.
Ready SLI включает и незавершённые после10s задания. SDK queues512records,
timeout0.75s; Collector memory limiter192MiB/batch≤256/queues32/finite retry10s.
Политика SLO99%/30m, учебный профиль, окна burn rate и шаги диагностики:
[RUNBOOK](infra/otel/RUNBOOK.md), [rules](infra/otel/alerts.yaml).
Фактические имена/единицы панелей будут сверены перед финальным отчётом.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Результаты начальной проверки
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Offline/OpenAPI/lint и25unit PASS. Новые unit проверяют перенос causal context
между завершёнными producer/consumer spans и отсутствие чувствительных данных.
Сначала исправлены импорт Observation из API SDK и сериализация enum severity
в тестовом приёмнике; privacy assertions сохранены. Runtime evidence появится
только после реального запуска; конфигурация не считается результатом.
## Ограничения и следующие шаги
## Источники и границы
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Текущее задание, собственная ЛР5, pinned libraries и официальные материалы:
[Python OTLP](https://opentelemetry.io/docs/languages/python/exporters/),
[Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
[Tempo3.1.1](https://github.com/grafana/tempo/blob/v3.1.1/example/docker-compose/single-binary/tempo.yaml),
[Collector exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/v0.165.0/exporter/otlphttpexporter),
[Prometheus alerts](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/).
Помощь ИИ использована при проектировании, реализации, проверках и отчёте.
Чужие студенческие решения/истории/измерения не использовались.
Один host/PG/S3/broker/LB остаются SPOF; нет защиты от потери диска. HTTP только
локальный с синтетическими данными. TLS/mTLS вводятся независимо в ЛР7.
+30
View File
@@ -0,0 +1,30 @@
# Explicit test-only fault instrumentation, never enabled by default.
services:
api:
environment:
PIPELINE_FAULTS: "1"
relay:
environment:
PIPELINE_FAULTS: "1"
worker:
environment:
PIPELINE_FAULTS: "1"
JOB_TIMEOUT_SECONDS: "12"
JOB_LEASE_SECONDS: "15"
controller:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command: [python, -c, "import time; time.sleep(86400)"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
cpus: 0.5
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
+497
View File
@@ -0,0 +1,497 @@
name: mtusi-lab6-maslov
services:
postgres:
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
PGDATA: /var/lib/postgresql/data/pgdata
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test:
- CMD-SHELL
- pg_isready -U messenger -d messenger
interval: 2s
timeout: 2s
retries: 20
cpus: 1
mem_limit: 512m
memswap_limit: 512m
cache:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command:
- valkey-server
- /run/secrets/valkey.conf
secrets:
- cache_password
- source: valkey_config
target: valkey.conf
healthcheck:
test:
- CMD-SHELL
- REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.migrate
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
cpus: 0.5
mem_limit: 256m
read_only: true
api:
image: ${API_IMAGE:-mtusi-messenger:lab6}
build:
context: .
args:
VCS_REF: ${VCS_REF:-unknown}
scale: ${API_REPLICAS:-2}
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- cache_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
cache:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
cpus: ${API_CPUS:-0.5}
mem_limit: ${API_MEMORY:-384m}
memswap_limit: ${API_MEMORY:-384m}
pids_limit: 128
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
stop_grace_period: 10s
s3:
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
build:
context: .
dockerfile: infra/s3.Dockerfile
target: server
secrets:
- s3_root_user
- s3_root_password
volumes:
- s3_data:/data
cpus: 1
mem_limit: 1024m
memswap_limit: 1024m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=32m
environment:
HOME: /tmp
security_opt:
- no-new-privileges:true
s3-bootstrap:
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
build:
context: .
dockerfile: infra/s3.Dockerfile
target: bootstrap
secrets:
- s3_root_user
- s3_root_password
- s3_access_key
- s3_secret_key
depends_on:
s3:
condition: service_healthy
cpus: 0.25
mem_limit: 128m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=16m
reconciler:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.reconcile
- --apply
- --loop
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets:
- database_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
cpus: 0.25
mem_limit: 256m
memswap_limit: 256m
read_only: true
healthcheck:
disable: true
broker:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command:
- valkey-server
- /run/secrets/broker.conf
secrets:
- broker_password
- source: broker_config
target: broker.conf
volumes:
- broker_data:/data
healthcheck:
test:
- CMD-SHELL
- REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping
interval: 2s
timeout: 2s
retries: 20
cpus: 0.5
mem_limit: 256m
memswap_limit: 256m
relay:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.pipeline
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- broker_url
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
worker:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.worker
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- broker_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
pids_limit: 128
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
stop_grace_period: 25s
lb:
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
ports:
- 127.0.0.1:${PUBLIC_PORT:-8086}:8080
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=4m
security_opt:
- no-new-privileges:true
stop_grace_period: 10s
observer:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.observer
environment:
DATABASE_URL_FILE: /run/secrets/database_url
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
depends_on:
migrate:
condition: service_completed_successfully
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
otel:
image: otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea
user: 10001:10001
volumes:
- ./infra/otel/collector.yaml:/etc/otel/config.yaml:ro
cpus: 0.5
mem_limit: 256m
memswap_limit: 256m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- --config=/etc/otel/config.yaml
prometheus:
image: prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06
user: 65534:65534
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=24h
- --storage.tsdb.retention.size=256MB
depends_on:
telemetry-init:
condition: service_completed_successfully
loki:
image: grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7
user: 10001:10001
volumes:
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
- loki_data:/loki
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- -config.file=/etc/loki/config.yaml
depends_on:
telemetry-init:
condition: service_completed_successfully
tempo:
image: grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044
user: 10001:10001
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
- tempo_data:/var/tempo
cpus: 0.5
mem_limit: 512m
memswap_limit: 512m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- -config.file=/etc/tempo/config.yaml
depends_on:
telemetry-init:
condition: service_completed_successfully
grafana:
image: grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf
user: 472:472
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_AUTH_ANONYMOUS_ENABLED: 'false'
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: 'false'
GF_PLUGINS_PREINSTALL_DISABLED: 'true'
GF_LOG_LEVEL: warn
secrets:
- grafana_password
ports:
- 127.0.0.1:${GRAFANA_PORT:-3006}:3000
depends_on:
telemetry-init:
condition: service_completed_successfully
telemetry-init:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
user: 0:0
command:
- python
- -c
- import os; [(os.makedirs(p,exist_ok=True),os.chown(p,u,u)) for p,u in [('/data/prometheus',65534),('/data/loki',10001),('/data/tempo',10001),('/data/grafana',472)]]
volumes:
- prometheus_data:/data/prometheus
- loki_data:/data/loki
- tempo_data:/data/tempo
- grafana_data:/data/grafana
read_only: true
cap_drop:
- ALL
cap_add:
- CHOWN
security_opt:
- no-new-privileges:true
cpus: 0.1
mem_limit: 64m
healthcheck:
disable: true
secrets:
broker_password:
file: ./secrets/broker_password
broker_url:
file: ./secrets/broker_url
broker_config:
file: ./secrets/broker.conf
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_password:
file: ./secrets/cache_password
cache_url:
file: ./secrets/cache_url
valkey_config:
file: ./secrets/valkey.conf
name: valkey.conf
grafana_password:
file: ./secrets/grafana_password
volumes:
broker_data: null
s3_data: null
postgres_data: null
prometheus_data: null
loki_data: null
tempo_data: null
grafana_data: null
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+789
View File
@@ -0,0 +1,789 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер: диагностика и SLO",
"schemaVersion": 41,
"version": 1,
"editable": false,
"timezone": "utc",
"refresh": "5s",
"time": {
"from": "now-30m",
"to": "now"
},
"tags": [
"lab6",
"slo"
],
"panels": [
{
"id": 1,
"title": "Запросы через балансировщик, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m]))",
"refId": "A",
"legendFormat": "все бизнес-запросы",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Health requests use a separate backend and are excluded. Includes ingress 503 with no API available.",
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "Доля ответов 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[1m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m])),0.001)",
"refId": "A",
"legendFormat": "ошибки 5xx",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "API: p50 / p95 / p99, секунды",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Quantiles from combined histogram buckets; complete ASGI response, no average of replica quantiles.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"title": "Очередь: ожидающие задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_queue_depth)",
"refId": "A",
"legendFormat": "ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Logical PostgreSQL jobs in queued state, not count of duplicate stream notifications.",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "Возраст старейшего незавершённого задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_oldest_job_age_seconds)",
"refId": "A",
"legendFormat": "возраст",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Обработчик: занято / доступно",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(messenger_worker_busy)",
"refId": "A",
"legendFormat": "занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(messenger_worker_capacity)",
"refId": "B",
"legendFormat": "всего мест",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "Соединения PostgreSQL",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name) (messenger_db_pool_used)",
"refId": "A",
"legendFormat": "{{service_name}} занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum by(service_name) (messenger_db_pool_waiting)",
"refId": "B",
"legendFormat": "{{service_name}} ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Попытки обработки по результату",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(outcome) (rate(messenger_job_attempts_total[2m]))",
"refId": "A",
"legendFormat": "{{outcome}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "Длительность попытки обработки: p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "От приёма до ready: p50 / p95 / p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Only completed publications have a duration. Deadline SLI below also counts unfinished tasks.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "SLO API за 30 минут: цель 99%",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "1 - sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[30m])) / clamp_min(sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\"}[30m])),1)",
"refId": "A",
"legendFormat": "доступность",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "SLO ready за 30 минут: 99% не позже 10 секунд",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(increase(messenger_slo_decisions_total{outcome=\"good\"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)",
"refId": "A",
"legendFormat": "доля вовремя",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Durable one-time decisions per non-obsolete logical job; unfinished jobs become bad at ten seconds. Synthetic 320x200 profile.",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "CPU процессов, ядра",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name)(rate(messenger_process_cpu_seconds_total[1m]))",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Память процессов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max by(service_name)(messenger_process_memory_bytes)",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Буфер экспорта Collector",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "otelcol_exporter_queue_size",
"refId": "A",
"legendFormat": "{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "otelcol_exporter_queue_capacity",
"refId": "B",
"legendFormat": "{{exporter}} capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Ошибки отправки трасс и логов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
"refId": "A",
"legendFormat": "spans/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))",
"refId": "B",
"legendFormat": "logs/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Состояния локальных alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "ALERTS{alertstate=\"firing\"}",
"refId": "A",
"legendFormat": "{{alertname}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Связанные логи API и обработчика",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"queryType": "range",
"datasource": {
"type": "loki",
"uid": "loki"
}
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending"
},
"description": "Filter structured metadata trace_id in Explore; open the same ID in the Traces datasource."
}
],
"annotations": {
"list": []
}
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Laboratory 6
type: file
disableDeletion: true
editable: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,37 @@
apiVersion: 1
datasources:
- name: Metrics
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
timeInterval: 5s
- name: Logs
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
url: $${__value.raw}
datasourceUid: tempo
- name: Traces
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
spanStartTimeShift: -1m
spanEndTimeShift: 1m
+50
View File
@@ -0,0 +1,50 @@
global
log stdout format raw local0
maxconn 512
stats socket /tmp/admin.sock mode 600 level admin
defaults
mode http
log global
log-format "http status=%ST duration_ms=%Ta"
timeout connect 1s
timeout client 15s
timeout server 8s
timeout http-request 10s
timeout http-keep-alive 5s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
use_backend health if { path_beg /health/ }
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
server-template api 1-8 api:8080
backend health
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
server-template api 1-8 api:8080
frontend internal_metrics
bind :8404
http-request use-service prometheus-exporter if { path /metrics }
http-request deny
+50
View File
@@ -0,0 +1,50 @@
# Диагностика по сохранённой телеметрии
Открой dashboard «Мессенджер: диагностика и SLO», временная зона UTC.
Сопоставляй время оповещения с метриками, трассами и связанными логами.
## API errors
Признак: ApiHighErrorRate, ApiFastBurn или ApiSlowBurn. Числитель доступности:
`sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m]))`.
Знаменатель — тот же backend со всеми code. Health использует другой backend.
Сравни API histogram, DB pool used/waiting и worker busy. В Traces найди
ошибочный server span и вложенный db.postgresql или s3.*. Длинная DB-операция
с ошибкой и занятым пулом при нормальной очереди указывает на БД. Отсутствие
API span при ingress503 может означать отсутствие готовых API. Один5xx не
доказывает причину. Через trace_id найди соответствующий Logs record.
После исправления проверь новые сообщения, спад5xx, resolved и прежние данные.
## Queue age
Признак: QueueLagGrowing, ReadyFastBurn или ReadySlowBurn. Сравни логические
queued jobs PG, oldest job age и worker busy/capacity. Upload202 при росте
ожидающей очереди и отсутствии новых worker spans указывает на обработчик.
Если попытки идут и содержат DB/S3 ошибки, причина может быть в зависимости.
Найди путь upload→producer→consumer→S3/DB. Histogram ready содержит только
завершения, поэтому отдельно проверь deadline SLI для незавершённых jobs.
После восстановления проверь тот же attachment_id, original, спад и resolved.
## Telemetry backend
Проверь backend health, Collector queue и accepted/sent/failed счётчики.
Failed attempts не равны числу уникально потерянных spans: пакет повторяется.
После слива очередей сравни изменение accepted-minus-sent и исходное состояние.
SDK queues traces/logs512records, timeout0.75s; Collector memory limiter192MiB,
batch≤256, очереди logs/traces по32пакета, один sender и finite retry≤10s.
При отказе telemetry может теряться; измеряй память и настоящие бизнес-запросы.
После восстановления найди новую трассу и лог, а не только старый screenshot.
## SLO и окна
Учебный профиль: синтетические PNG320×200, upload не чаще одного каждые3s,
закреплённые CPU/RAM. API цель99%ответов без5xx за30m на ingress. Jobs цель99%
ready≤10s от PG created_at принимающей транзакции до committed_at публикации.
Начальная граница консервативно включает завершение принимающей транзакции.
Однократные durable good/bad решения учитывают failed и незавершённые после10s.
Ручной replay не стирает прежний bad; устаревшие до решения jobs исключены.
Burn rate = доля плохих событий/0.01. Быстрое правило: burn>14.4 в30s и2m,
непрерывно45s. Длительное: burn>6 в2m и5m, непрерывно60s. Короткие окна выбраны
для воспроизводимого стенда, а не универсальной production paging policy.
Измеряются краткий всплеск/длительная деградация/firing/resolved/ложные события.
Точность времени ограничена polling2s, scrape/evaluation5s и экспортом.
+53
View File
@@ -0,0 +1,53 @@
groups:
- name: messenger-slo
interval: 5s
rules:
- alert: ApiHighErrorRate
expr: sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) > 0.05
for: 20s
labels: {severity: warning, signal: api}
annotations:
summary: More than 5 percent business responses are 5xx
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: QueueLagGrowing
expr: max(messenger_oldest_job_age_seconds) > 20 and max(messenger_queue_depth) > 0
for: 10s
labels: {severity: warning, signal: ready}
annotations:
summary: Oldest unfinished attachment exceeds 20 seconds
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ApiFastBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[30s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[30s])),0.001) / 0.01 > 14.4) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: api, burn: fast}
annotations:
summary: API burns the one-percent error budget in both fast windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ApiSlowBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 6) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: api, burn: slow}
annotations:
summary: API burns the error budget across two longer windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ReadyFastBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[30s])) / clamp_min(sum(rate(messenger_slo_decisions_total[30s])),0.001) / 0.01 > 14.4) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: ready, burn: fast}
annotations:
summary: Attachment deadline SLO burns the error budget
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ReadySlowBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 6) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[5m])) / clamp_min(sum(rate(messenger_slo_decisions_total[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: ready, burn: slow}
annotations:
summary: Sustained misses of the attachment ten-second deadline
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: CollectorUnavailable
expr: up{job="collector"} == 0
for: 15s
labels: {severity: warning, signal: telemetry}
annotations:
summary: Collector self metrics are unavailable
runbook: infra/otel/RUNBOOK.md#telemetry-backend
+71
View File
@@ -0,0 +1,71 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
metric_expiration: 5m
otlp_http/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
otlp_http/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/tempo]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
+38
View File
@@ -0,0 +1,38 @@
auth_enabled: false
server:
http_listen_port: 3100
log_level: warn
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2026-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
ingestion_rate_mb: 4
ingestion_burst_size_mb: 8
volume_enabled: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name]
- action: structured_metadata
attributes: [service.instance.id, service.version, deployment.environment.name]
analytics:
reporting_enabled: false
+23
View File
@@ -0,0 +1,23 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: application_otlp
static_configs:
- targets: [otel:8889]
- job_name: collector
static_configs:
- targets: [otel:8888]
- job_name: ingress
static_configs:
- targets: [lb:8404]
- job_name: prometheus
static_configs:
- targets: [localhost:9090]
- job_name: tempo
static_configs:
- targets: [tempo:3200]
- job_name: loki
static_configs:
- targets: [loki:3100]
+20
View File
@@ -0,0 +1,20 @@
target: all
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+7
View File
@@ -0,0 +1,7 @@
{
"otel": "otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea",
"prometheus": "prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
"loki": "grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
"tempo": "grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044",
"grafana": "grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
}
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_key=$(cat /run/secrets/s3_access_key)
api_secret=$(cat /run/secrets/s3_secret_key)
# Credentials are passed only inside this isolated bootstrap container.
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
mc mb --ignore-existing local/messenger >/dev/null 2>&1
mc anonymous set none local/messenger >/dev/null 2>&1
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
printf '%s\n' 'Private bucket and scoped application policy ready'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger/objects/*"]
}
]
}
+49
View File
@@ -0,0 +1,49 @@
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
for attempt in 1 2 3 4; do \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
test "$attempt" -lt 4 || exit 1; sleep 1; \
done && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
for attempt in 1 2 3; do \
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
test "$attempt" -lt 3 || exit 1; sleep 1; \
done
FROM toolchain AS mc-build
WORKDIR /source
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
RUN mkdir /data && chown 10001:10001 /data
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
COPY infra/s3-policy.json /policy.json
USER 10001:10001
ENV MC_CONFIG_DIR=/tmp/mc
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
+32
View File
@@ -0,0 +1,32 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD UNIQUE(chat_id,id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code text,
uploader_instance text NOT NULL,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(chat_id,id),
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position),
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+54
View File
@@ -0,0 +1,54 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
ALTER TABLE attachments ADD COLUMN published_version integer;
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
);
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
CREATE TABLE outbox (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz
);
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
size_bytes bigint NOT NULL, sha256 text NOT NULL,
width integer, height integer,
PRIMARY KEY(attachment_id,pipeline_version,name)
);
CREATE TABLE effect_publications (
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(attachment_id,pipeline_version)
);
CREATE TABLE pipeline_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
operator text NOT NULL, reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
CREATE TABLE pipeline_faults (
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
);
CREATE TABLE pipeline_events (
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+10
View File
@@ -0,0 +1,10 @@
ALTER TABLE jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
ALTER TABLE jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
CREATE TABLE job_slo_observations (
job_id uuid PRIMARY KEY REFERENCES jobs(id),
outcome text NOT NULL CHECK(outcome IN ('good','bad')),
decided_at timestamptz NOT NULL DEFAULT clock_timestamp(),
histogram_epoch uuid
);
CREATE INDEX job_slo_epoch_idx ON job_slo_observations(histogram_epoch);
CREATE INDEX effect_publications_time_idx ON effect_publications(committed_at,job_id);
+58
View File
@@ -0,0 +1,58 @@
[project]
name = "mtusi-messenger-lab6"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3",
"pydantic==2.14.1",
"uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6",
"argon2-cffi>=25,<26",
"redis==8.1.0",
"minio==7.2.20",
"pillow==13.0.0",
"opentelemetry-api==1.45.2",
"opentelemetry-sdk==1.45.2",
"opentelemetry-exporter-otlp-proto-http==1.45.2",
"psutil==7.1.3",
]
[dependency-groups]
dev = [
"pytest>=8,<10",
"httpx>=0.28,<1",
"ruff>=0.14,<1",
"openapi-spec-validator==0.7.2",
"psutil>=7,<8",
"matplotlib>=3.10,<4",
"pyyaml==6.0.3",
]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
"src/observer.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
+128
View File
@@ -0,0 +1,128 @@
"""Summarize measured fault invariants and child resource samples."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def effects(case):
for key in ["effect", "v1", "v2"]:
if key in case:
yield key, case[key]
for i, item in enumerate(case.get("effects", [])):
yield str(i + 1), item
def main():
p = argparse.ArgumentParser()
p.add_argument("input", type=Path)
p.add_argument("--output", required=True, type=Path)
a = p.parse_args()
d = json.loads(a.input.read_text())
assert d["complete"] and d.get("scope", "all") == "all"
rows = []
for case in d["cases"]:
for role, e in effects(case):
j = e["job"]
rows.append(
dict(
case=case["case"],
role=role,
job_id=j["id"],
version=j["pipeline_version"],
attempt=j["attempt"],
cycle=j["cycle"],
fence=j["fence"],
state=j["state"],
publications=e["publications"],
variant_rows=len(e["variants"]),
physical_keys=len(e["physical_keys"]),
)
)
a.output.mkdir(parents=True, exist_ok=True)
with (a.output / "effects.csv").open("w") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
ready = [r for r in rows if r["state"] == "ready"]
assert all(
r["publications"] == 1
and r["variant_rows"] == 4
and r["physical_keys"] == 4
for r in ready
)
stale = [r for r in rows if r["state"] == "obsolete"]
assert all(r["publications"] == 0 for r in stale)
job_ids = {r["job_id"] for r in rows}
samples = [
x["detail"]
for x in d["worker_metrics"]
if str(x["job_id"]) in job_ids
and x["detail"].get("result") == "ready"
and "rss_peak_mib" in x["detail"]
]
stats = {
"source_commit": d["source_commit"],
"cases": len(d["cases"]),
"observed_ready_effects": len(ready),
"observed_obsolete_effects": len(stale),
"all_ready_have_one_publication_and_four_variants": True,
"all_obsolete_have_zero_publications": True,
"successful_child_samples": len(samples),
"child_elapsed_range_seconds": [
min(x["seconds"] for x in samples),
max(x["seconds"] for x in samples),
],
"child_cpu_range_seconds": [
min(x["cpu_seconds"] for x in samples),
max(x["cpu_seconds"] for x in samples),
],
"child_peak_rss_range_mib": [
min(x["rss_peak_mib"] for x in samples),
max(x["rss_peak_mib"] for x in samples),
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
}
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
axes[0].bar(
range(1, len(rows) + 1),
[r["publications"] for r in rows],
color=[
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
],
)
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
axes[0].set(
xlabel="Effect observation (CSV row)",
ylabel="Committed publications",
title="One publication per ready version; stale v1 = 0",
yticks=[0, 1],
)
axes[1].scatter(
[x["seconds"] for x in samples],
[x["rss_peak_mib"] for x in samples],
color="#246b8e",
s=35,
alpha=0.8,
)
axes[1].set(
xlabel="Child elapsed, seconds",
ylabel="Child peak RSS, MiB",
title=f"{len(samples)} completed child samples (synthetic320×200)",
)
for ax in axes:
ax.grid(axis="y", alpha=0.2)
fig.savefig(a.output / "pipeline-results.png", dpi=150)
plt.close(fig)
print(json.dumps(stats, indent=2))
if __name__ == "__main__":
main()
+64
View File
@@ -0,0 +1,64 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "grafana_password", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
docker compose restart api postgres cache s3 broker relay worker lb
docker compose up -d --no-build --no-recreate --wait
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py rolling
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab6-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py stop
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose stop --timeout 1 worker
+254
View File
@@ -0,0 +1,254 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab6-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
clone = Path(temporary) / "lab6"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 6
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt[
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab6-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+223
View File
@@ -0,0 +1,223 @@
"""Real S3 retention test: seed is separate from verification output."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.verify_pipeline import inside, operator, wait
def seed(path):
if path.exists():
raise ValueError("Choose a new seed file")
ready()
compose("stop", "--timeout", "1", "reconciler")
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
user, _, session = actor(c)
headers = auth(session)
result = c.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Cleanup synthetic seed",
"member_ids": [user["id"]],
},
)
assert result.status_code == 201
chat = result.json()["id"]
r = c.post(
f"/api/v1/chats/{chat}/attachments",
headers={**headers, "Content-Type": "image/png"},
params={"filename": "retained.png"},
content=raw,
)
assert r.status_code == 202
attachment = r.json()["id"]
wait(
lambda: (
c.get(
"/api/v1/attachments/" + attachment, headers=headers
).json()["status"]
== "ready"
)
)
job = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
data = {
"seed_id": uuid4().hex,
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"ready_attachment": attachment,
"ready_job": job,
"chat": chat,
"owner": user["id"],
}
# Real old v1 objects will become obsolete after v2 is ready.
second = operator(job, version=2)
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
)
)
data["version2_job"] = second
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
data["failed_id"] = str(uuid4())
data["active_id"] = str(uuid4())
data["reservation_id"] = str(uuid4())
data["expired_reservation_id"] = str(uuid4())
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
data["active_source"] = f"objects/staging/{data['active_id']}/original"
data["reservation_key"] = (
f"objects/staging/{data['reservation_id']}/original"
)
data["expired_reservation_key"] = (
f"objects/staging/{data['expired_reservation_id']}/original"
)
inside(f"""from pathlib import Path
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
with db.connection() as c, c.transaction():
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
key='objects/staging/'+identity+'/original'
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
print('true')""")
data["seeded_unix"] = time.time()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: retention seed created; real S3 age must reach300s", flush=True
)
def verify(seed_path, output):
if output.exists():
raise ValueError("Choose a new output path")
data = json.loads(seed_path.read_text())
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"seed": seed_path.name,
"complete": False,
}
try:
compose("stop", "--timeout", "1", "reconciler")
def age():
return inside(
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
)
while age() < 302:
time.sleep(min(5, 303 - age()))
report["real_s3_age_seconds"] = age()
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
inside(
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
)
oldkeys = [
data["orphan"],
data["failed_source"],
data["expired_reservation_key"],
] + [
f"objects/{data['ready_attachment']}/v1/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
protected = [data["active_source"], data["reservation_key"], fresh] + [
f"objects/{data['ready_attachment']}/v2/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
before = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(before.values())
dry = inside(
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
)
candidates = {a["key"] for a in dry["actions"]}
assert set(oldkeys) <= candidates and not set(protected) & candidates
afterdry = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert afterdry == before
applied = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
repeat = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
assert not repeat["actions"]
after = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(not after[k] for k in oldkeys) and all(
after[k] for k in protected
)
failed = inside(
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
)
assert failed["error_code"] == "source_expired"
try:
operator(failed["id"])
except RuntimeError:
replay_refused = True
else:
raise AssertionError("Expired source replay must fail")
# Expired reservation was removed, so a delayed API cannot commit its handoff.
fenced = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
)
assert fenced
report.update(
dry_run=dry,
apply=applied,
repeat=repeat,
before=before,
after=after,
dry_run_unchanged=True,
active_current_and_fresh_protected=True,
expired_source_replay_refused=replay_refused,
expired_writer_fenced=fenced,
complete=True,
)
finally:
# Remove only this fixture's artificial active reservation/job; never real jobs.
inside(
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
)
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
flush=True,
)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["seed", "verify"])
p.add_argument("--seed", required=True, type=Path)
p.add_argument("--output", type=Path)
a = p.parse_args()
if a.action == "seed":
seed(a.seed)
else:
if a.output is None:
p.error("output required")
verify(a.seed, a.output)
if __name__ == "__main__":
main()
+347
View File
@@ -0,0 +1,347 @@
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while (
time.monotonic() - started < seconds
or not finished.is_set()
or time.monotonic() - started
< action_result.get("finished", 0) + 2
):
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()
+687
View File
@@ -0,0 +1,687 @@
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def inspect_effect(job, *, pending=False):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
pending_entries=[]
if {pending!r}:
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
entry=item['message_id']
messages=queue.xrange(STREAM,min=entry,max=entry)
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument(
"--start-at", choices=["all", "versions"], default="all"
)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"scope": args.start_at,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
if args.start_at == "all":
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
stop("broker")
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
"broker_stopped_during_upload": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
stop("relay")
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids, {"before": ids, "after": restored}
start("relay")
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in [
"before_effect",
"after_effect",
"after_commit",
]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(
j, pending=phase == "after_commit"
)
assert at_crash["job"]["state"] == (
"ready"
if phase == "after_commit"
else "processing"
)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
if phase == "after_commit":
assert at_crash["pending_stream_entries"], (
"Result must be committed while notification is still unacknowledged"
)
assert not any(
e["event"] == "child_finished"
for e in at_crash["events"]
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
if phase == "after_commit":
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
and e["detail"]["state"] == "ready"
for e in inspect_effect(j)["events"]
)
)
wait(
lambda j=j: inside(
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
)
)
effect = inspect_effect(j)
else:
assert effect["job"]["attempt"] >= 2
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
marker = inject(j, "before_effect", "hang")
start("worker")
wait(lambda: hit(marker))
limits = inside(
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
)
assert limits["address_space_bytes"] == [805306368, 805306368]
assert limits["memory.max"] == "1610612736"
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save(
"hung_child_killed_and_retried",
{"effect": effect, "observed_limits": limits},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert (
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
)
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
# Keep one controller process hot throughout the narrow stale-writer window.
code = f"""from src.pipeline import event
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
end=time.monotonic()+45
while True:
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
if hit: break
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
time.sleep(.05)
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
end=time.monotonic()+10
while True:
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
if ready: break
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
time.sleep(.05)
with db.connection() as c:
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
end=time.monotonic()+10
while True:
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
if stale: break
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
time.sleep(.05)
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
"""
with ThreadPoolExecutor(max_workers=1) as hot:
future = hot.submit(inside, code)
wait(
lambda j=j: inside(
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
)
)
start("worker")
race = future.result(timeout=45)
k = race["job_id"]
hashes = race["hashes"]
confirm(a, k)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
if __name__ == "__main__":
main()
View File
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import os
import signal
import sys
def early_stop(signum, _frame):
print(
json.dumps({"event": "stopped_before_startup", "signal": signum}),
flush=True,
)
sys.exit(0)
signal.signal(signal.SIGTERM, early_stop)
signal.signal(signal.SIGINT, early_stop)
import uvicorn # noqa: E402
from src.logging_config import JsonFormatter # noqa: E402
def main() -> None:
host = os.environ.get("APP_HOST", "127.0.0.1")
port = int(os.environ.get("APP_PORT", "8080"))
if not 1 <= port <= 65535:
raise ValueError("APP_PORT must be between 1 and 65535")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
uvicorn.run(
"src.api:app",
host=host,
port=port,
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
log_config=None,
)
if __name__ == "__main__":
main()
+412
View File
@@ -0,0 +1,412 @@
import hashlib
import json
import logging
import tempfile
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime
from functools import partial
from urllib.parse import quote
from uuid import UUID, uuid4
from anyio import Semaphore, fail_after, to_thread
from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from starlette.requests import ClientDisconnect
from starlette.responses import JSONResponse, Response, StreamingResponse
from src import images, telemetry
from src.attachments import Attachments
from src.cache import UserCache
from src.cursor import parse_uuid
from src.database import Database
from src.domain import timestamp
from src.errors import DomainError, invalid
from src.object_store import ObjectStore
from src.password import Passwords
from src.repository import PostgresRepository
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
from src.service import MessengerService
from src.settings import Settings
logger = logging.getLogger("messenger")
INSTANCE_ID = uuid4().hex
def response(data: object, *, status: int = 200) -> JSONResponse:
return JSONResponse(
jsonable_encoder(
data, custom_encoder={datetime: timestamp, UUID: str}
),
status_code=status,
)
async def body(request: Request, model: type[Input]) -> Input:
try:
data = await request.body()
return model.model_validate(
json.loads(
data.decode("utf-8"),
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
)
)
except ValueError, ValidationError, UnicodeError, RecursionError:
raise invalid() from None
def pagination(request: Request) -> tuple[int, str | None]:
limits = request.query_params.getlist("limit")
cursors = request.query_params.getlist("cursor")
if len(limits) > 1 or len(cursors) > 1:
raise invalid()
raw = limits[0] if limits else "50"
if not raw.isascii() or not raw.isdecimal():
raise invalid()
raw = raw.lstrip("0") or "0"
if len(raw) > 3:
raise invalid()
limit = int(raw)
if not 1 <= limit <= 100:
raise invalid()
return limit, cursors[0] if cursors else None
def service(request: Request) -> MessengerService:
return request.app.state.service
async def run_io(function, **kwargs):
return await to_thread.run_sync(partial(function, **kwargs))
async def actor(request: Request):
return await run_io(
service(request).identify, value=request.headers.get("authorization")
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
telemetry.configure("messenger-api", instance=INSTANCE_ID)
settings = Settings.load()
database = Database(url=settings.database_url, size=settings.pool_size)
cache = UserCache(url=settings.cache_url)
store = ObjectStore()
try:
await run_io(database.open)
passwords = await run_io(Passwords)
app.state.database = database
app.state.store = store
app.state.upload_slots = Semaphore(2)
app.state.attachments = Attachments(
database=database, store=store, instance=INSTANCE_ID
)
app.state.service = MessengerService(
repository=PostgresRepository(database=database, cache=cache),
passwords=passwords,
session_ttl=settings.session_ttl,
)
app.state.ready = True
yield
finally:
app.state.ready = False
await run_io(store.close)
await run_io(cache.close)
await run_io(database.close)
await run_io(telemetry.stop)
app = FastAPI(
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
)
app.add_middleware(telemetry.TelemetryMiddleware)
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid4())
request.state.request_id = request_id
try:
result = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "internal_error", "request_id": request_id})
)
result = response(
{
"error": {
"code": "internal_error",
"message": "Internal error",
"request_id": request_id,
}
},
status=500,
)
result.headers["X-Request-Id"] = request_id
result.headers["X-Instance-Id"] = INSTANCE_ID
return result
@app.exception_handler(DomainError)
async def domain_error(request: Request, error: DomainError):
result = response(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request.state.request_id,
}
},
status=error.status,
)
result.headers.update(error.headers)
return result
@app.exception_handler(HTTPException)
async def http_error(request: Request, error: HTTPException):
code = "not_found" if error.status_code == 404 else "invalid_request"
return response(
{
"error": {
"code": code,
"message": "Resource not found"
if error.status_code == 404
else "Invalid request",
"request_id": request.state.request_id,
}
},
status=error.status_code,
)
@app.get("/health/live")
async def live():
return response({"status": "ok"})
@app.get("/health/ready")
async def ready():
if not getattr(app.state, "ready", False):
raise DomainError(503, "unavailable", "Not ready")
await run_io(app.state.database.ping)
await run_io(app.state.store.ping)
return response({"status": "ok"})
@app.post("/api/v1/users")
async def register(request: Request):
data = await body(request, CreateUser)
return response(
await run_io(service(request).register, **data.model_dump()),
status=201,
)
@app.get("/api/v1/users/me")
async def current(request: Request):
return response(await actor(request))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
user = await actor(request)
data = await body(request, CreateChat)
return response(
await run_io(
service(request).create_chat, actor_id=user.id, **data.model_dump()
),
status=201,
)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).chats,
actor_id=user.id,
limit=limit,
cursor=cursor,
)
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await actor(request)
return response(
await run_io(
service(request).get_chat,
actor_id=user.id,
chat_id=parse_uuid(chat_id),
)
)
@app.post("/api/v1/chats/{chat_id}/messages")
async def send(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
data = await body(request, CreateMessage)
return response(
await run_io(
service(request).send,
actor_id=user.id,
chat_id=cid,
idempotency_key=request.headers.get("idempotency-key"),
**data.model_dump(),
),
status=201,
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def history(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
chat = await run_io(
service(request).get_chat, actor_id=user.id, chat_id=cid
)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).history,
actor_id=user.id,
chat=chat,
limit=limit,
cursor=cursor,
)
)
@app.post("/api/v1/auth/sessions")
async def login(request: Request):
return response(
await run_io(
service(request).login,
authorization=request.headers.get("authorization"),
),
status=201,
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(request: Request):
await run_io(
service(request).logout,
authorization=request.headers.get("authorization"),
)
return Response(status_code=204)
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
names = request.query_params.getlist("filename")
if len(names) != 1:
raise invalid()
name = images.filename(names[0])
declared = (
request.headers.get("content-type", "")
.partition(";")[0]
.strip()
.lower()
)
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
length = request.headers.get("content-length")
if length and (not length.isdecimal() or len(length) > 12):
raise invalid()
if length and int(length) > images.MAX_BYTES:
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
try:
with fail_after(20):
async with app.state.upload_slots:
with tempfile.SpooledTemporaryFile(
max_size=1024 * 1024, mode="w+b"
) as stream:
size = 0
digest = hashlib.sha256()
async for chunk in request.stream():
size += len(chunk)
if size > images.MAX_BYTES:
raise DomainError(
413, "payload_too_large", "Byte limit exceeded"
)
digest.update(chunk)
stream.write(chunk)
await run_io(
images.verify, stream=stream, declared=declared
)
result = await run_io(
app.state.attachments.upload,
chat_id=cid,
owner_id=user.id,
filename=name,
content_type=declared,
size=size,
sha256=digest.hexdigest(),
stream=stream,
)
return response(result, status=202)
except ClientDisconnect:
raise invalid() from None
except TimeoutError:
raise DomainError(
503, "unavailable", "Upload deadline exceeded"
) from None
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment_metadata(request: Request, attachment_id: str):
user = await actor(request)
return response(
await run_io(
app.state.attachments.metadata,
identity=parse_uuid(attachment_id),
actor_id=user.id,
)
)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def attachment_content(request: Request, attachment_id: str):
user = await actor(request)
variants = request.query_params.getlist("variant")
if len(variants) > 1:
raise invalid()
row, upstream = await run_io(
app.state.attachments.content,
identity=parse_uuid(attachment_id),
actor_id=user.id,
variant=variants[0] if variants else "original",
)
def chunks():
try:
while data := upstream.read(65536):
yield data
finally:
upstream.close()
upstream.release_conn()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
},
)
+144
View File
@@ -0,0 +1,144 @@
from uuid import uuid4
from src.errors import DomainError, not_found
from src.pipeline import enqueue, fault
def dto(row, variants=()):
return {
k: row[k]
for k in [
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"created_at",
"updated_at",
]
} | {
"status": row["state"],
"variants": [
{
k: v[k]
for k in [
"name",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
]
}
for v in variants
if v["name"] != "original"
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
}
class Attachments:
def __init__(self, *, database, store, instance):
self.database, self.store, self.instance = database, store, instance
def upload(
self,
*,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
stream,
):
identity, job_id = uuid4(), uuid4()
key = f"objects/staging/{identity}/original"
with self.database.connection() as conn:
conn.execute(
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
(identity, key),
)
self.store.put(
key=key,
stream=stream,
size=size,
content_type=content_type,
sha256=sha256,
)
fault(self.database, chat_id, "api_after_staging")
with self.database.connection() as conn, conn.transaction():
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
(identity,),
).fetchone()
if reservation is None:
raise DomainError(
503, "unavailable", "Upload reservation expired"
)
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
(
identity,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
key,
self.instance,
),
).fetchone()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
(job_id, identity, key),
)
enqueue(conn, job_id)
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
)
fault(self.database, chat_id, "api_after_commit")
# Captured committed snapshot remains queued even if worker already finished.
return dto(row)
def row(self, *, identity, actor_id):
with self.database.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
(actor_id, identity),
).fetchone()
if row is None:
raise not_found()
return row
def metadata(self, *, identity, actor_id):
row = self.row(identity=identity, actor_id=actor_id)
variants = []
if row["state"] == "ready":
with self.database.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
(identity, row["published_version"]),
).fetchall()
return dto(row, variants)
def content(self, *, identity, actor_id, variant):
row = self.row(identity=identity, actor_id=actor_id)
if row["state"] != "ready":
raise DomainError(409, "conflict", "Attachment is not ready")
with self.database.connection() as conn:
data = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(identity, row["published_version"], variant),
).fetchone()
if data is None:
raise not_found()
if self.store.head(key=data["object_key"]) is None:
raise DomainError(503, "unavailable", "Variant unavailable")
return {**data, "filename": row["filename"]}, self.store.get(
key=data["object_key"]
)
+66
View File
@@ -0,0 +1,66 @@
import json
import threading
import time
from datetime import datetime
from uuid import UUID
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from src.domain import User, timestamp
class UserCache:
def __init__(self, *, url: str):
self.client = redis.Redis.from_url(
url,
socket_connect_timeout=0.15,
socket_timeout=0.15,
max_connections=16,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
self.disabled_until = 0.0
self.lock = threading.Lock()
def close(self):
self.client.close()
def _command(self, function, *args, **kwargs):
if time.monotonic() < self.disabled_until:
return None
try:
return function(*args, **kwargs)
except redis.RedisError, OSError:
with self.lock:
self.disabled_until = time.monotonic() + 5
return None
def get(self, *, user_id: UUID) -> User | None:
raw = self._command(self.client.get, "user:" + str(user_id))
if raw is None:
return None
try:
row = json.loads(raw)
if UUID(row["id"]) != user_id:
return None
return User(
user_id,
row["username"],
row["display_name"],
datetime.fromisoformat(row["created_at"]),
)
except ValueError, TypeError, KeyError:
return None
def put(self, *, user: User):
raw = json.dumps(
{
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": timestamp(user.created_at),
}
)
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
+64
View File
@@ -0,0 +1,64 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.domain import Key, timestamp
from src.errors import invalid
def parse_uuid(value: str) -> UUID:
if not isinstance(value, str) or not re.fullmatch(
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
):
raise invalid()
return UUID(value)
def encode_cursor(*, scope: str, key: Key) -> str:
payload = {
"v": 1,
"scope": scope,
"at": timestamp(key[0]),
"id": str(key[1]),
}
return (
base64.urlsafe_b64encode(
json.dumps(payload, separators=(",", ":")).encode()
)
.decode()
.rstrip("=")
)
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
if value is None:
return None
try:
if (
not value
or len(value) > 2048
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
):
raise ValueError
data = json.loads(
base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
)
if (
not isinstance(data, dict)
or type(data.get("v")) is not int
or data["v"] != 1
or data.get("scope") != scope
or not isinstance(data.get("at"), str)
):
raise ValueError
at = datetime.fromisoformat(data["at"])
if at.tzinfo is None or timestamp(at) != data["at"]:
raise ValueError
return at, parse_uuid(data["id"])
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
raise invalid() from None
+77
View File
@@ -0,0 +1,77 @@
import time
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from src import telemetry
from src.errors import DomainError
class DeadlineConnection(psycopg.Connection):
"""Клиентский срок действует и при замороженном PostgreSQL."""
def wait(self, gen, interval=0.1, timeout=None):
deadline = getattr(self, "request_deadline", None)
remaining = (
max(0.001, deadline - time.monotonic()) if deadline else 3.0
)
bound = min(remaining, timeout) if timeout is not None else remaining
try:
return super().wait(gen, interval=interval, timeout=bound)
except psycopg.OperationalError:
self.close()
raise
class Database:
def __init__(self, *, url: str, size: int):
self.pool = ConnectionPool(
url,
connection_class=DeadlineConnection,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": (
"-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=6000"
),
},
min_size=1,
max_size=size,
timeout=1,
open=False,
)
telemetry.register_pool(self.pool)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
self.pool.close(timeout=2)
@contextmanager
def connection(self):
try:
with (
telemetry.span(
"db.postgresql",
attributes={"db.system.name": "postgresql"},
),
self.pool.connection(timeout=1) as conn,
):
conn.request_deadline = time.monotonic() + 4
try:
yield conn
finally:
conn.request_deadline = None
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
raise DomainError(
503, "unavailable", "Database unavailable"
) from None
def ping(self):
with self.connection() as conn:
conn.execute("SELECT 1").fetchone()
+45
View File
@@ -0,0 +1,45 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
def utcnow() -> datetime:
return datetime.now(UTC)
def timestamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
type Record = User | Chat | Message
type Key = tuple[datetime, UUID]
def order_key(record: Record) -> Key:
return record.created_at, record.id
+16
View File
@@ -0,0 +1,16 @@
class DomainError(Exception):
def __init__(
self, status: int, code: str, message: str, headers=None
) -> None:
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid() -> DomainError:
return DomainError(400, "invalid_request", "Invalid request")
def not_found() -> DomainError:
return DomainError(404, "not_found", "Resource not found")
+77
View File
@@ -0,0 +1,77 @@
import struct
import threading
import unicodedata
import zlib
from PIL import Image, UnidentifiedImageError
from src.errors import DomainError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
DECODE_SLOT = threading.BoundedSemaphore(1)
def filename(value):
if (
not value
or not 1 <= len(value) <= 255
or not value.strip()
or any(
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
for c in value
)
):
raise invalid()
return value
def verify(stream, *, declared):
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
with DECODE_SLOT:
try:
stream.seek(0)
with Image.open(stream) as image:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
image.format
)
if actual != declared:
raise DomainError(
415, "unsupported_media_type", "Image type mismatch"
)
if image.width * image.height > MAX_PIXELS:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
)
image.verify()
# JPEG verify alone does not decode entropy data.
stream.seek(0)
with Image.open(stream) as image:
image.load()
except Image.DecompressionBombError:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
) from None
except (
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
struct.error,
zlib.error,
EOFError,
):
raise DomainError(
415, "unsupported_media_type", "Invalid image"
) from None
except MemoryError:
raise DomainError(
503, "unavailable", "Image memory budget exceeded"
) from None
finally:
stream.seek(0)
return declared
+17
View File
@@ -0,0 +1,17 @@
import json
import logging
from datetime import UTC, datetime
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = record.getMessage()
try:
event = json.loads(message)
if not isinstance(event, dict):
event = {"event": message}
except ValueError:
event = {"event": message}
event["timestamp"] = datetime.now(UTC).isoformat()
event["level"] = record.levelname.lower()
return json.dumps(event, ensure_ascii=False)
+51
View File
@@ -0,0 +1,51 @@
import hashlib
import json
from pathlib import Path
import psycopg
from src.settings import secret
def main():
with psycopg.connect(
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
) as conn:
conn.execute("SELECT pg_advisory_lock(2301002)")
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
for path in sorted(Path("migrations").glob("*.sql")):
content = path.read_bytes()
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s",
(path.name,),
).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError(
"Applied migration checksum mismatch"
)
continue
with conn.transaction():
conn.execute(content.decode())
conn.execute(
"INSERT INTO schema_migrations(name,checksum) "
"VALUES(%s,%s)",
(path.name, checksum),
)
print(
json.dumps(
{"event": "migration_applied", "name": path.name}
)
)
finally:
conn.execute("SELECT pg_advisory_unlock(2301002)")
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
from contextlib import contextmanager
import urllib3
from minio import Minio
from minio.error import S3Error
from src import telemetry
from src.errors import DomainError
from src.settings import secret
BUCKET = "messenger"
class ObjectStore:
def __init__(self):
self.http = urllib3.PoolManager(
maxsize=32,
block=True,
timeout=urllib3.Timeout(connect=1, read=3),
retries=urllib3.Retry(total=0),
)
self.client = Minio(
"s3:9000",
access_key=secret("S3_ACCESS_KEY"),
secret_key=secret("S3_SECRET_KEY"),
secure=False,
region="us-east-1",
http_client=self.http,
)
@contextmanager
def operation(self, name="request"):
try:
with telemetry.span("s3." + name, attributes={"operation": name}):
yield
except S3Error, urllib3.exceptions.HTTPError, OSError:
raise DomainError(
503, "unavailable", "Object store unavailable"
) from None
def ping(self):
with self.operation("head_bucket"):
if not self.client.bucket_exists(BUCKET):
raise DomainError(
503, "unavailable", "Object store unavailable"
)
def put(self, *, key, stream, size, content_type, sha256):
stream.seek(0)
with self.operation("put"):
self.client.put_object(
BUCKET,
key,
stream,
length=size,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10 * 1024 * 1024,
num_parallel_uploads=1,
)
def head(self, *, key):
with self.operation("head"):
try:
return self.client.stat_object(BUCKET, key)
except S3Error as exc:
if exc.code in {"NoSuchKey", "NoSuchObject"}:
return None
raise
def get(self, *, key):
with self.operation("get"):
return self.client.get_object(BUCKET, key)
def delete(self, *, key):
with self.operation("delete"):
self.client.remove_object(BUCKET, key)
def objects(self):
with self.operation("list"):
yield from self.client.list_objects(
BUCKET, prefix="objects/", recursive=True
)
def close(self):
self.http.clear()
+96
View File
@@ -0,0 +1,96 @@
"""Bounded DB polling; pending tasks missing the deadline count as bad SLIs."""
import time
from uuid import uuid4
from src import telemetry
from src.pipeline import database
def poll(db, epoch):
with db.connection() as conn, conn.transaction():
# Evaluate unfinished jobs at their deadline, not after completion.
# Exclude decided IDs before LIMIT to avoid starving newer jobs.
conn.execute(
"INSERT INTO job_slo_observations(job_id,outcome) "
"SELECT j.id,CASE WHEN e.committed_at<=j.created_at+interval '10 seconds' "
"THEN 'good' ELSE 'bad' END FROM jobs j "
"LEFT JOIN effect_publications e ON e.job_id=j.id "
"WHERE NOT EXISTS(SELECT 1 FROM job_slo_observations o WHERE o.job_id=j.id) "
"AND j.state<>'obsolete' AND (e.committed_at IS NOT NULL "
"OR j.state='failed' OR j.created_at<=clock_timestamp()-interval '10 seconds') "
"ORDER BY j.created_at LIMIT 128 ON CONFLICT DO NOTHING"
)
queue = conn.execute(
"SELECT count(*) FILTER(WHERE state='queued') depth,"
"COALESCE(max(EXTRACT(epoch FROM clock_timestamp()-created_at)) "
"FILTER(WHERE state IN ('queued','processing')),0) oldest FROM jobs"
).fetchone()
decisions = conn.execute(
"SELECT outcome,count(*) n FROM job_slo_observations GROUP BY outcome"
).fetchall()
completed = conn.execute(
"SELECT event,count(*) n FROM pipeline_events "
"WHERE event IN ('effect_committed','failed') GROUP BY event"
).fetchall()
ready = conn.execute(
"SELECT o.job_id,EXTRACT(epoch FROM e.committed_at-j.created_at) seconds "
"FROM job_slo_observations o JOIN effect_publications e ON e.job_id=o.job_id "
"JOIN jobs j ON j.id=o.job_id WHERE o.histogram_epoch IS DISTINCT FROM %s::uuid "
"ORDER BY e.committed_at LIMIT 128 FOR UPDATE OF o SKIP LOCKED",
(epoch,),
).fetchall()
for row in ready:
conn.execute(
"UPDATE job_slo_observations SET histogram_epoch=%s WHERE job_id=%s",
(epoch, row["job_id"]),
)
values = {
"queue_depth": queue["depth"],
"oldest_job_age": float(queue["oldest"]),
"slo_good": 0,
"slo_bad": 0,
"completed_ready": 0,
"completed_failed": 0,
}
values.update({"slo_" + row["outcome"]: row["n"] for row in decisions})
values.update(
{
"completed_"
+ (
"ready" if row["event"] == "effect_committed" else "failed"
): row["n"]
for row in completed
}
)
telemetry.observe(values)
for row in ready:
telemetry.duration(
"time_to_ready",
float(row["seconds"]),
{"job_type": "attachment.process.v1", "outcome": "ready"},
)
def main():
epoch = str(uuid4())
telemetry.configure("messenger-observer", instance=epoch)
db = database()
try:
while True:
try:
poll(db, epoch)
except Exception:
telemetry.log(
"observer.unavailable",
error=True,
error_code="database_unavailable",
)
time.sleep(2)
finally:
db.close()
telemetry.stop()
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
import argparse
import json
from uuid import UUID, uuid4
from src.object_store import ObjectStore
from src.pipeline import database, enqueue, event
def change(db, store, *, job_id, operator, reason, version=None):
if (
not operator.strip()
or not reason.strip()
or len(operator) > 100
or len(reason) > 500
):
raise ValueError("Operator and reason are required")
with db.connection() as conn, conn.transaction():
job = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if not job:
raise ValueError("Unknown job")
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
(job["attachment_id"],),
).fetchone()
if job["pipeline_version"] != attachment["pipeline_version"]:
raise ValueError("Superseded job cannot be replayed")
source = (
job["source_key"]
if attachment["published_version"] is None
else attachment["object_key"]
)
if store.head(key=source) is None:
raise ValueError("Source expired or missing; replay refused")
if version is None:
if job["state"] != "failed":
raise ValueError("Only failed jobs can be replayed")
conn.execute(
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
action = "replay"
target = job_id
else:
if (
version not in {1, 2}
or version <= attachment["pipeline_version"]
):
raise ValueError("Supported version must increase (1 or 2)")
conn.execute(
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
(job_id,),
)
target = uuid4()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
(target, attachment["id"], version, source),
)
conn.execute(
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
(version, attachment["id"]),
)
action = "new_version"
conn.execute(
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment["id"],),
)
conn.execute(
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
(target, action, operator, reason),
)
event(conn, target, action, {"previous_job": str(job_id)})
enqueue(conn, target)
return str(target)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["failed", "replay", "version"])
p.add_argument("--job-id", type=UUID)
p.add_argument("--operator")
p.add_argument("--reason")
p.add_argument("--version", type=int)
a = p.parse_args()
db, store = database(), ObjectStore()
try:
if a.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not all([a.job_id, a.operator, a.reason]):
p.error("job-id/operator/reason required")
if a.action == "version" and a.version is None:
p.error("version required")
print(
json.dumps(
{
"job_id": change(
db,
store,
job_id=a.job_id,
operator=a.operator,
reason=a.reason,
version=a.version
if a.action == "version"
else None,
)
}
)
)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
import threading
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.low_level import Type
from src.errors import DomainError
class Passwords:
def __init__(self):
self.hasher = PasswordHasher(
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
)
self.slots = threading.BoundedSemaphore(2)
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
def _run(self, function, *args):
if not self.slots.acquire(timeout=5):
raise DomainError(503, "unavailable", "Authentication busy")
try:
return function(*args)
finally:
self.slots.release()
def hash(self, *, password: str) -> str:
return self._run(self.hasher.hash, password)
def verify(self, *, password: str, encoded: str | None) -> bool:
try:
return (
self._run(self.hasher.verify, encoded or self.dummy, password)
and encoded is not None
)
except VerificationError, InvalidHashError:
return False
+289
View File
@@ -0,0 +1,289 @@
"""Durable job state is authoritative; stream entries are wake-up notifications."""
import json
import os
import random
import re
import time
from datetime import UTC, datetime
from uuid import UUID
import redis
from psycopg.types.json import Jsonb
from src import telemetry
from src.database import Database
from src.settings import integer, secret
STREAM = "attachments"
GROUP = "image-workers"
def database():
db = Database(url=secret("DATABASE_URL"), size=4)
db.open()
return db
def broker():
return redis.Redis.from_url(
secret("BROKER_URL"),
decode_responses=True,
socket_connect_timeout=1,
socket_timeout=3,
max_connections=8,
)
def envelope(row):
return dict(
schema_version=1,
job_id=str(row["id"]),
type="attachment.process.v1",
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
)
def validate_job(data):
required = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
if (
not isinstance(data, dict)
or not required <= data.keys()
or data.keys() - required - {"traceparent", "tracestate"}
):
raise ValueError("Invalid job envelope")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("Invalid job type")
for k in ["job_id", "attachment_id"]:
if not isinstance(data[k], str):
raise ValueError("Invalid job UUID")
UUID(data[k])
for k in ["pipeline_version", "attempt"]:
if type(data[k]) is not int or data[k] < 1:
raise ValueError("Invalid job number")
timestamp = data["created_at"]
if not isinstance(timestamp, str) or not re.fullmatch(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})",
timestamp,
):
raise ValueError("Invalid job timestamp")
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if "traceparent" in data:
if not isinstance(data["traceparent"], str) or not re.fullmatch(
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
):
raise ValueError("Invalid trace context")
if "tracestate" in data and (
not isinstance(data["tracestate"], str)
or len(data["tracestate"]) > 512
):
raise ValueError("Invalid trace context")
return data
def event(conn, job_id, name, detail=None):
conn.execute(
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
(job_id, name, Jsonb(detail or {})),
)
def enqueue(conn, job_id):
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
context = telemetry.carrier()
if not context and row.get("traceparent"):
context = {"traceparent": row["traceparent"]}
if row.get("tracestate"):
context["tracestate"] = row["tracestate"]
if context:
conn.execute(
"UPDATE jobs SET traceparent=%s,tracestate=%s WHERE id=%s",
(
context.get("traceparent", ""),
context.get("tracestate", ""),
job_id,
),
)
conn.execute(
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
(job_id, Jsonb(envelope(row) | context), row["next_at"]),
)
def fault(db, target, phase):
if os.getenv("PIPELINE_FAULTS") != "1":
return
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
(target, phase),
).fetchone()
if row is None:
return
conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
action = row["action"]
if action == "crash":
os._exit(89)
if action == "permanent":
raise PermanentFailure("invalid_image")
if action == "transient":
raise RuntimeError("injected_transient")
if action == "hang":
time.sleep(120)
if action == "wait":
end = time.monotonic() + 60
while time.monotonic() < end:
with db.connection() as conn:
released = conn.execute(
"SELECT released_at FROM pipeline_faults WHERE id=%s",
(row["id"],),
).fetchone()
if released and released["released_at"]:
return
time.sleep(0.05)
raise RuntimeError("fault_wait_timeout")
class PermanentFailure(Exception):
pass
def retry(conn, row, error, *, permanent=False):
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
failed = permanent or row["attempt"] >= maximum
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
state = "failed" if failed else "queued"
code = error if permanent else ("retry_exhausted" if failed else error)
conn.execute(
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
(state, code, delay, row["id"]),
)
conn.execute(
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(
state,
code if failed else None,
row["attachment_id"],
row["pipeline_version"],
),
)
event(
conn,
row["id"],
"failed" if failed else "retry",
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
)
if not failed:
enqueue(conn, row["id"])
def watchdog(db):
with db.connection() as conn, conn.transaction():
rows = conn.execute(
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
retry(conn, row, "lease_expired")
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
rows = conn.execute(
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
enqueue(conn, row["id"])
def relay_once(db, queue):
watchdog(db)
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
).fetchone()
if not row:
return False
validate_job(row["payload"])
with telemetry.span(
"attachments publish",
kind=telemetry.SpanKind.PRODUCER,
parent=row["payload"],
attributes={
"job.id": str(row["job_id"]),
"messaging.system": "valkey",
"messaging.destination.name": STREAM,
},
):
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
with queue.client() as dedicated:
entry = dedicated.xadd(
STREAM,
{
"job": json.dumps(
row["payload"] | telemetry.carrier(),
separators=(",", ":"),
)
},
)
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
if int(confirmation[0]) < 1:
raise RuntimeError(
"Broker durability confirmation timed out"
)
telemetry.log("job.published", job_id=str(row["job_id"]))
fault(db, row["job_id"], "relay_after_publish")
conn.execute(
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
event(
conn,
row["job_id"],
"published",
{"entry": entry, "outbox": row["id"]},
)
return True
def relay():
telemetry.configure("messenger-relay")
db, queue = database(), broker()
try:
while True:
try:
if not relay_once(db, queue):
time.sleep(0.2)
except Exception:
print(
json.dumps(
{
"event": "relay_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(1)
finally:
queue.close()
db.close()
telemetry.stop()
if __name__ == "__main__":
relay()
+122
View File
@@ -0,0 +1,122 @@
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
import argparse
import json
import time
from src.object_store import ObjectStore
from src.pipeline import database
from src.settings import integer
def reconcile(db, store, *, apply=False, grace=None):
grace = (
grace
if grace is not None
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
)
if grace < 300:
raise ValueError("Retention must be at least 300 seconds")
actions = []
for obj in store.objects():
# Recheck each candidate in a bounded transaction, including late PUT age.
with db.connection() as conn, conn.transaction():
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
"now"
]
if (now - obj.last_modified).total_seconds() < grace:
continue
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
(obj.object_name,),
).fetchone()
if reservation and reservation["expires_at"] > now:
continue
jobs = conn.execute(
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
(obj.object_name, obj.object_name),
).fetchall()
if any(j["state"] in {"queued", "processing"} for j in jobs):
continue
current = conn.execute(
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
(obj.object_name,),
).fetchone()
if current:
continue
if any(
j["state"] == "failed"
and (now - j["updated_at"]).total_seconds() < grace
for j in jobs
):
continue
head = store.head(key=obj.object_name)
if (
head is None
or (now - head.last_modified).total_seconds() < grace
):
continue
action = {
"action": "delete_expired_object",
"key": obj.object_name,
"applied": apply,
}
if apply:
# Taking the reservation lock fences an expired API before deletion.
if reservation:
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s",
(reservation["id"],),
)
store.delete(key=obj.object_name)
for job in jobs:
if (
job["state"] == "failed"
and job["source_key"] == obj.object_name
):
conn.execute(
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
(job["id"],),
)
conn.execute(
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
(job["attachment_id"], job["pipeline_version"]),
)
actions.append(action)
return {"apply": apply, "retention_seconds": grace, "actions": actions}
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--loop", action="store_true")
a = p.parse_args()
db, store = database(), ObjectStore()
try:
while True:
try:
print(
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
)
except Exception:
if not a.loop:
raise
print(
json.dumps(
{
"event": "cleanup_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
if not a.loop:
break
time.sleep(60)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+296
View File
@@ -0,0 +1,296 @@
import hashlib
import json
from uuid import UUID
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from src.cache import UserCache
from src.database import Database
from src.domain import Chat, Key, Message, User
from src.errors import DomainError, invalid, not_found
MESSAGE_COLUMNS = (
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
)
USER_COLUMNS = "id, username, display_name, created_at"
def user_dto(row) -> User:
return User(
row["id"], row["username"], row["display_name"], row["created_at"]
)
def chat_dto(row) -> Chat:
return Chat(
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
)
def message_dto(row) -> Message:
return Message(
row["id"],
row["chat_id"],
row["sender_id"],
row["text"],
row["created_at"],
tuple(row.get("attachment_ids", ())),
)
class PostgresRepository:
def __init__(self, *, database: Database, cache: UserCache):
self.database = database
self.cache = cache
def add_user(self, *, user: User, password_hash: str) -> User:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO users(id,username,display_name,"
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
(
user.id,
user.username,
user.display_name,
password_hash,
user.created_at,
),
)
except UniqueViolation:
raise DomainError(
409, "conflict", "Username already exists"
) from None
return user
def get_user(self, *, user_id: UUID) -> User | None:
cached = self.cache.get(user_id=user_id)
if cached is not None:
return cached
with self.database.connection() as conn:
row = conn.execute(
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(user_id,),
).fetchone()
if row is None:
return None
user = user_dto(row)
self.cache.put(user=user)
return user
def credentials(self, *, username: str):
with self.database.connection() as conn:
row = conn.execute(
"SELECT "
+ USER_COLUMNS
+ ", password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
if row is None:
return None, None
return user_dto(row), row["password_hash"]
def username_exists(self, *, username: str) -> bool:
with self.database.connection() as conn:
return (
conn.execute(
"SELECT 1 FROM users WHERE username=%s", (username,)
).fetchone()
is not None
)
def add_session(
self, *, digest: bytes, user_id: UUID, created_at, expires_at
):
with self.database.connection() as conn:
conn.execute(
"INSERT INTO sessions(token_digest,user_id,created_at,"
"expires_at) VALUES(%s,%s,%s,%s)",
(digest, user_id, created_at, expires_at),
)
def session_user(self, *, digest: bytes) -> UUID | None:
with self.database.connection() as conn:
row = conn.execute(
"SELECT user_id FROM sessions WHERE token_digest=%s "
"AND expires_at>clock_timestamp()",
(digest,),
).fetchone()
return row["user_id"] if row else None
def delete_session(self, *, digest: bytes):
with self.database.connection() as conn:
conn.execute(
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
)
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
members = chat.member_ids
if actor_id not in members or len(set(members)) != len(members):
raise invalid()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute(
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
(chat.id, chat.title, chat.created_at),
)
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO members(chat_id,user_id,position) "
"VALUES(%s,%s,%s)",
[
(chat.id, user_id, position)
for position, user_id in enumerate(members)
],
)
except ForeignKeyViolation:
raise invalid() from None
return chat
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
with self.database.connection() as conn:
row = conn.execute(
"SELECT c.id,c.title,c.created_at,"
"array_agg(m.user_id ORDER BY m.position) member_ids "
"FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
(chat_id, actor_id),
).fetchone()
if row is None:
raise not_found()
return chat_dto(row)
def add_message(
self, *, message: Message, idempotency_key: str
) -> Message:
# Область включает пользователя, метод и нормализованный путь чата.
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
lock = int.from_bytes(
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
)
fingerprint = hashlib.sha256(
json.dumps(
{
"text": message.text,
"attachment_ids": [str(x) for x in message.attachment_ids],
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode()
).digest()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
prior = conn.execute(
"SELECT i.body_hash,i.message_id,"
"i.expires_at>clock_timestamp() active "
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
"AND key=%s FOR UPDATE",
(message.sender_id, message.chat_id, idempotency_key),
).fetchone()
if prior and prior["active"]:
if bytes(prior["body_hash"]) != fingerprint:
raise DomainError(409, "conflict", "Key already used")
row = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.id=%s",
(prior["message_id"],),
).fetchone()
return message_dto(row)
if message.attachment_ids:
rows = conn.execute(
"SELECT id,state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s) FOR SHARE",
(message.chat_id, list(message.attachment_ids)),
).fetchall()
if len(rows) != len(message.attachment_ids):
raise not_found()
if any(row["state"] == "failed" for row in rows):
raise DomainError(
409, "conflict", "Attachment is not ready"
)
conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,"
"text,created_at) "
"VALUES(%s,%s,%s,%s,%s)",
(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
),
)
if message.attachment_ids:
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO message_attachments("
"message_id,chat_id,"
"attachment_id,position) "
"VALUES(%s,%s,%s,%s)",
[
(message.id, message.chat_id, aid, i)
for i, aid in enumerate(message.attachment_ids)
],
)
conn.execute(
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
"clock_timestamp()+interval '24 hours') "
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
"expires_at=excluded.expires_at",
(
message.sender_id,
message.chat_id,
idempotency_key,
fingerprint,
message.id,
),
)
except ForeignKeyViolation:
raise not_found() from None
return message
def list_chats(
self, *, actor_id: UUID, after: Key | None, limit: int
) -> list[Chat]:
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
parameters = (
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
"FROM members WHERE chat_id=c.id "
"ORDER BY position) member_ids "
"FROM chats c JOIN members own ON own.chat_id=c.id "
"WHERE own.user_id=%s "
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
parameters,
).fetchall()
return [chat_dto(row) for row in rows]
def list_messages(
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
) -> list[Message]:
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
parameters = (
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s "
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
parameters,
).fetchall()
return [message_dto(row) for row in rows]
+65
View File
@@ -0,0 +1,65 @@
import re
from typing import Annotated
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictStr,
ValidationInfo,
field_validator,
model_validator,
)
from src.cursor import parse_uuid
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
class Input(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value: object, info: ValidationInfo) -> object:
if isinstance(value, str) and (
(not value.strip() and info.field_name != "text")
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
):
raise ValueError("Invalid text")
return value
class CreateUser(Input):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
class CreateMessage(Input):
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UuidIn] = Field(
default_factory=list, max_length=10, strict=True
)
@model_validator(mode="after")
def content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Duplicate attachment")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+192
View File
@@ -0,0 +1,192 @@
import base64
import hashlib
import re
import secrets
import threading
from datetime import timedelta
from uuid import UUID, uuid4
from src.cursor import decode_cursor, encode_cursor
from src.domain import Chat, Message, Record, User, order_key, utcnow
from src.errors import DomainError, invalid, not_found
from src.password import Passwords
from src.repository import PostgresRepository
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
items = rows[:limit]
cursor = (
encode_cursor(scope=scope, key=order_key(items[-1]))
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": cursor}
def unauthorized(*, basic=False):
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return DomainError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": challenge},
)
def bearer(value):
if not value:
raise unauthorized()
scheme, _, token = value.partition(" ")
if scheme.lower() != "bearer" or not re.fullmatch(
r"[A-Za-z0-9_-]{32,256}", token
):
raise unauthorized()
return token
def token_digest(value):
return hashlib.sha256(value.encode()).digest()
class MessengerService:
def __init__(
self,
*,
repository: PostgresRepository,
passwords: Passwords,
session_ttl: int,
) -> None:
self.repository = repository
self.passwords = passwords
self.session_ttl = session_ttl
self.registration_locks = [threading.Lock() for _ in range(64)]
def identify(self, *, value: str | None) -> User:
token = bearer(value)
user_id = self.repository.session_user(digest=token_digest(token))
user = self.repository.get_user(user_id=user_id) if user_id else None
if user is None:
raise unauthorized()
return user
def register(
self, *, username: str, display_name: str, password: str
) -> User:
# Bound lock memory and avoid running Argon2 for every local duplicate.
# Different replicas still race safely through PostgreSQL UNIQUE.
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
lock = self.registration_locks[stripe]
if not lock.acquire(timeout=8):
raise DomainError(503, "unavailable", "Registration busy")
try:
if self.repository.username_exists(username=username):
raise DomainError(409, "conflict", "Username already exists")
encoded = self.passwords.hash(password=password)
return self.repository.add_user(
user=User(uuid4(), username, display_name, utcnow()),
password_hash=encoded,
)
finally:
lock.release()
def login(self, *, authorization: str | None):
try:
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic":
raise ValueError
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = plain.partition(":")
if (
not separator
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
or not 12 <= len(password) <= 128
or not password.strip()
):
raise ValueError
except ValueError, UnicodeError:
raise unauthorized(basic=True) from None
user, password_hash = self.repository.credentials(username=username)
if not self.passwords.verify(password=password, encoded=password_hash):
raise unauthorized(basic=True)
now = utcnow()
expires = now + timedelta(seconds=self.session_ttl)
token = secrets.token_urlsafe(32)
self.repository.add_session(
digest=token_digest(token),
user_id=user.id,
created_at=now,
expires_at=expires,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires,
"user": user,
}
def logout(self, *, authorization: str | None):
self.identify(value=authorization)
self.repository.delete_session(
digest=token_digest(bearer(authorization))
)
def create_chat(
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
) -> Chat:
return self.repository.add_chat(
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
actor_id=actor_id,
)
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
def send(
self,
*,
actor_id: UUID,
chat_id: UUID,
text: str,
idempotency_key: str | None,
attachment_ids: list[UUID],
) -> Message:
if not idempotency_key or not re.fullmatch(
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
):
raise invalid()
return self.repository.add_message(
message=Message(
uuid4(),
chat_id,
actor_id,
text,
utcnow(),
tuple(attachment_ids),
),
idempotency_key=idempotency_key,
)
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
scope = f"chats:{actor_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_chats(
actor_id=actor_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
def history(
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
) -> dict:
if actor_id not in chat.member_ids:
raise not_found()
chat_id = chat.id
scope = f"messages:{chat_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_messages(
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
+37
View File
@@ -0,0 +1,37 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str) -> str:
file = os.environ.get(name + "_FILE")
if file:
return Path(file).read_text().strip()
value = os.environ.get(name)
if not value:
raise ValueError("Missing required secret setting: " + name)
return value
def integer(name: str, default: int, low: int, high: int) -> int:
value = int(os.environ.get(name, str(default)))
if not low <= value <= high:
raise ValueError("Invalid setting: " + name)
return value
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 10
@classmethod
def load(cls):
return cls(
secret("DATABASE_URL"),
secret("CACHE_URL"),
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
integer("DB_POOL_SIZE", 10, 1, 32),
)
+439
View File
@@ -0,0 +1,439 @@
"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs."""
import json
import os
import re
import threading
import time
import weakref
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import psutil
from opentelemetry import trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
OTLPLogExporter,
)
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
from opentelemetry.metrics import Observation
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import Histogram, MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import (
ExplicitBucketHistogramAggregation,
View,
)
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
_state = None
_lock = threading.Lock()
_pools = weakref.WeakSet()
_instruments = {}
_observer = {}
_propagator = TraceContextTextMapPropagator()
BUCKETS = (
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1,
2.5,
5,
10,
20,
60,
120,
)
LOG_FIELDS = {
"request_id",
"job_id",
"attachment_id",
"method",
"route",
"status",
"duration_seconds",
"attempt",
"version",
"fence",
"outcome",
"error_code",
"operation",
"cycle",
}
TRACE_FIELDS = {
"request_id",
"job.id",
"attachment.id",
"pipeline.version",
"job.attempt",
"db.system.name",
"operation",
"messaging.system",
"messaging.destination.name",
"http.route",
"http.request.method",
"http.response.status_code",
"error.type",
}
def enabled():
return os.getenv("OTEL_ENABLED", "0") == "1"
def register_pool(pool):
_pools.add(pool)
def observe(values):
_observer.update(values)
def _pool_stat(key):
stats = [p.get_stats() for p in list(_pools)]
if key == "used":
return sum(
max(0, s.get("pool_size", 0) - s.get("pool_available", 0))
for s in stats
)
return sum(s.get("requests_waiting", 0) for s in stats)
def configure(service, *, instance=None, with_metrics=True):
global _state
if not enabled():
return
with _lock:
if _state is not None:
return
instance = instance or uuid4().hex
resource = Resource.create(
{
"service.name": service,
"service.version": os.getenv("APP_VERSION", "development"),
"service.instance.id": instance,
"deployment.environment.name": "lab",
}
)
endpoint = os.getenv(
"OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318"
).rstrip("/")
options = {"timeout": 0.75}
# Certificate options are also honored by the exporters in the TLS lab.
mp = None
if with_metrics:
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(
endpoint=endpoint + "/v1/metrics", **options
),
export_interval_millis=2000,
export_timeout_millis=1000,
)
mp = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
View(
instrument_type=Histogram,
aggregation=ExplicitBucketHistogramAggregation(
BUCKETS
),
)
],
)
meter = mp.get_meter("messenger", "1.0")
for name in ["http_requests", "job_attempts"]:
_instruments[name] = meter.create_counter("messenger_" + name)
_instruments["worker_busy"] = meter.create_up_down_counter(
"messenger_worker_busy"
)
for name in ["http_duration", "job_duration", "time_to_ready"]:
_instruments[name] = meter.create_histogram(
"messenger_" + name, unit="s"
)
for key in ["used", "waiting"]:
meter.create_observable_gauge(
"messenger_db_pool_" + key,
callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]],
)
process = psutil.Process()
meter.create_observable_counter(
"messenger_process_cpu",
unit="s",
callbacks=[
lambda _: [Observation(sum(process.cpu_times()[:2]))]
],
)
meter.create_observable_gauge(
"messenger_process_memory",
unit="By",
callbacks=[lambda _: [Observation(process.memory_info().rss)]],
)
def container_memory(_):
p = Path("/sys/fs/cgroup/memory.current")
return [Observation(int(p.read_text()))] if p.exists() else []
meter.create_observable_gauge(
"messenger_container_memory",
unit="By",
callbacks=[container_memory],
)
if service == "messenger-worker":
meter.create_observable_gauge(
"messenger_worker_capacity",
callbacks=[
lambda _: [
Observation(
int(os.getenv("WORKER_CONCURRENCY", "2"))
)
]
],
)
if service == "messenger-observer":
for key in ["queue_depth", "oldest_job_age"]:
meter.create_observable_gauge(
"messenger_" + key,
unit="s" if key.endswith("age") else "",
callbacks=[
lambda _, k=key: [
Observation(
_observer.get(k, 0),
{"queue": "attachments"},
)
]
],
)
for name, prefix in [
("slo_decisions", "slo_"),
("jobs_completed", "completed_"),
]:
meter.create_observable_counter(
"messenger_" + name,
callbacks=[
lambda _, p=prefix: [
Observation(
_observer.get(p + outcome, 0),
{
"outcome": outcome,
"job_type": "attachment.process.v1",
},
)
for outcome in (
["good", "bad"]
if p == "slo_"
else ["ready", "failed"]
)
]
],
)
tp = TracerProvider(
resource=resource,
sampler=ParentBased(
TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1")))
),
)
tp.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
_state = SimpleNamespace(
service=service,
instance=instance,
traces=tp,
metrics=mp,
logs=lp,
tracer=tp.get_tracer("messenger", "1.0"),
logger=lp.get_logger("messenger", "1.0"),
)
def stop():
if _state is None:
return
for provider in [_state.traces, _state.logs, _state.metrics]:
if provider is not None:
provider.force_flush(timeout_millis=1000)
provider.shutdown()
def count(name, value=1, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.add(value, attributes or {})
def duration(name, seconds, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.record(max(0, seconds), attributes or {})
def carrier():
result = {}
_propagator.inject(result)
return result
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None):
tracer = (
_state.tracer if _state is not None else trace.get_tracer("messenger")
)
attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS}
context = _propagator.extract(parent) if parent is not None else None
with tracer.start_as_current_span(
name,
kind=kind,
context=context,
attributes=attrs,
record_exception=False,
set_status_on_exception=False,
) as current:
try:
yield current
except BaseException as exc:
current.set_attribute("error.type", type(exc).__name__)
current.set_status(Status(StatusCode.ERROR, "operation failed"))
raise
def log(event, *, error=False, **fields):
event = (
event
if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event)
else "application.event"
)
attrs = {
k: v[:256] if isinstance(v, str) else v
for k, v in fields.items()
if k in LOG_FIELDS and isinstance(v, (str, int, float, bool))
}
current = trace.get_current_span().get_span_context()
record = {
"event": event,
"timestamp": time.time(),
"severity": "ERROR" if error else "INFO",
"service.name": _state.service if _state else "messenger",
**attrs,
}
if current.is_valid:
record.update(
trace_id=f"{current.trace_id:032x}",
span_id=f"{current.span_id:016x}",
)
print(json.dumps(record, ensure_ascii=False), flush=True)
if _state is not None:
_state.logger.emit(
body=event,
attributes=attrs,
severity_text=record["severity"],
severity_number=SeverityNumber.ERROR
if error
else SeverityNumber.INFO,
)
class TelemetryMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
# Health polling does not enter business SLIs or spans.
if scope.get("path", "").startswith("/health/"):
return await self.app(scope, receive, send)
headers = {
k.decode("latin1"): v.decode("latin1")
for k, v in scope.get("headers", [])
if k.lower() in {b"traceparent", b"tracestate"}
}
status = 500
started = time.monotonic()
async def measured_send(message):
nonlocal status
if message["type"] == "http.response.start":
status = message["status"]
await send(message)
with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current:
try:
await self.app(scope, receive, measured_send)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
method = scope.get("method", "UNKNOWN")
if method not in {
"GET",
"POST",
"PUT",
"PATCH",
"DELETE",
"HEAD",
"OPTIONS",
}:
method = "OTHER"
current.update_name(method + " " + route)
current.set_attributes(
{
"http.route": route,
"http.request.method": method,
"http.response.status_code": status,
}
)
if status >= 500:
current.set_status(
Status(StatusCode.ERROR, "HTTP server error")
)
elapsed = time.monotonic() - started
labels = {
"route": route,
"method": method,
"status_class": str(status // 100) + "xx",
}
count("http_requests", attributes=labels)
duration("http_duration", elapsed, labels)
log(
"http.request",
request_id=scope.get("state", {}).get("request_id", ""),
method=method,
route=route,
status=status,
duration_seconds=round(elapsed, 6),
error=status >= 500,
)
+547
View File
@@ -0,0 +1,547 @@
"""Two bounded consumers; every decode runs in a killable isolated process."""
import hashlib
import io
import json
import multiprocessing
import resource
import threading
import time
from pathlib import Path
from uuid import uuid4
import redis
from PIL import Image, ImageOps, UnidentifiedImageError
from src import telemetry
from src.images import MAX_BYTES, MAX_PIXELS
from src.object_store import ObjectStore
from src.pipeline import (
GROUP,
STREAM,
PermanentFailure,
broker,
database,
event,
fault,
retry,
validate_job,
)
from src.settings import integer
def sizes(width, height, bound):
longest = max(width, height)
return (
(width, height)
if longest <= bound
else (
max(1, width * bound // longest),
max(1, height * bound // longest),
)
)
def make_variants(raw, row):
if (
len(raw) != row["size_bytes"]
or hashlib.sha256(raw).hexdigest() != row["sha256"]
):
raise PermanentFailure("source_checksum_mismatch")
try:
with Image.open(io.BytesIO(raw)) as source:
if source.width * source.height > MAX_PIXELS:
raise PermanentFailure("pixel_limit")
if source.format not in {"PNG", "JPEG"}:
raise PermanentFailure("invalid_image")
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert("RGBA")
rgb = Image.new("RGB", rgba.size, "white")
rgb.paste(rgba, mask=rgba.getchannel("A"))
rgb.info.clear()
yield "original", raw, row["content_type"], source.size
thumbnail = rgb.resize(
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
)
crop = ImageOps.fit(
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
optimized = rgb.resize(
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
)
for name, image, fmt, mime, options in [
(
"thumbnail",
thumbnail,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"crop",
crop,
"JPEG",
"image/jpeg",
{"quality": 85, "subsampling": 0},
),
(
"optimized",
optimized,
"WEBP",
"image/webp",
{
"quality": 82 if row["pipeline_version"] == 1 else 76,
"method": 4,
},
),
]:
output = io.BytesIO()
image.info.clear()
image.save(output, format=fmt, **options)
yield name, output.getvalue(), mime, image.size
except (
UnidentifiedImageError,
OSError,
ValueError,
Image.DecompressionBombError,
) as exc:
raise PermanentFailure("invalid_image") from exc
def process(job_id, fence, sender, trace_context=None):
telemetry.configure("messenger-worker", with_metrics=False)
try:
with telemetry.span(
"attachment.process",
parent=trace_context,
attributes={"job.id": job_id},
):
telemetry.log("job.processing", job_id=job_id, fence=fence)
process_body(job_id, fence, sender)
finally:
telemetry.stop()
def process_body(job_id, fence, sender):
db = store = None
started = time.monotonic()
cpu = time.process_time()
try:
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
db, store = database(), ObjectStore()
with db.connection() as conn:
row = conn.execute(
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
(job_id, fence),
).fetchone()
if not row:
sender.send({"result": "stale"})
return
# Record limits before fault hooks: a slow external inspector must not
# consume the child timeout while attempting to observe the limit.
cgroup = Path("/sys/fs/cgroup")
limits = {
name: (cgroup / name).read_text().strip()
if (cgroup / name).exists()
else "unavailable"
for name in ["memory.max", "cpu.max"]
}
limits["address_space_bytes"] = list(
resource.getrlimit(resource.RLIMIT_AS)
)
with db.connection() as conn:
event(conn, job_id, "worker_resource_limits", limits)
fault(db, job_id, "before_effect")
upstream = store.get(key=row["source_key"])
try:
raw = upstream.read(MAX_BYTES + 1)
finally:
upstream.close()
upstream.release_conn()
results = []
for name, data, mime, (width, height) in make_variants(raw, row):
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
digest = hashlib.sha256(data).hexdigest()
store.put(
key=key,
stream=io.BytesIO(data),
size=len(data),
content_type=mime,
sha256=digest,
)
results.append((name, key, mime, len(data), digest, width, height))
fault(db, job_id, "after_effect")
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
allowed = conn.execute(
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
(row["attachment_id"],),
).fetchone()
valid_lease = conn.execute(
"SELECT %s::timestamptz>clock_timestamp() live",
(current["lease_until"],),
).fetchone()["live"]
if (
current["state"] != "processing"
or current["fence"] != fence
or not valid_lease
or allowed["pipeline_version"] != row["pipeline_version"]
):
event(
conn,
job_id,
"stale_completion_rejected",
{"fence": fence, "version": row["pipeline_version"]},
)
sender.send({"result": "stale"})
return
for result in results:
conn.execute(
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
(row["attachment_id"], row["pipeline_version"], *result),
)
conn.execute(
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
(row["attachment_id"], row["pipeline_version"], job_id, fence),
)
conn.execute(
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(results[0][1], row["attachment_id"]),
)
conn.execute(
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
event(
conn,
job_id,
"effect_committed",
{
"fence": fence,
"version": row["pipeline_version"],
"variants": 4,
},
)
fault(db, job_id, "after_commit")
# Cleanup is independently retryable; it cannot undo the committed effect.
if row["source_key"].startswith("objects/staging/"):
try:
store.delete(key=row["source_key"])
except Exception:
pass
telemetry.log(
"job.ready", job_id=job_id, version=row["pipeline_version"]
)
sender.send(
{
"result": "ready",
"seconds": time.monotonic() - started,
"cpu_seconds": time.process_time() - cpu,
"rss_peak_mib": resource.getrusage(
resource.RUSAGE_SELF
).ru_maxrss
/ 1024,
}
)
except PermanentFailure as exc:
telemetry.log(
"job.processing_failed",
job_id=job_id,
error=True,
error_code=str(exc),
)
telemetry.trace.get_current_span().set_status(
telemetry.Status(
telemetry.StatusCode.ERROR, "permanent processing error"
)
)
sender.send({"result": "permanent", "code": str(exc)})
except MemoryError:
telemetry.log(
"job.processing_failed",
job_id=job_id,
error=True,
error_code="memory_limit",
)
telemetry.trace.get_current_span().set_status(
telemetry.Status(telemetry.StatusCode.ERROR, "memory limit")
)
sender.send({"result": "permanent", "code": "memory_limit"})
except Exception:
telemetry.log(
"job.processing_failed",
job_id=job_id,
error=True,
error_code="processing_unavailable",
)
telemetry.trace.get_current_span().set_status(
telemetry.Status(
telemetry.StatusCode.ERROR, "transient processing error"
)
)
sender.send({"result": "transient", "code": "processing_unavailable"})
finally:
if store:
store.close()
if db:
db.close()
sender.close()
def claim(db, data, consumer):
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
).fetchone()
if (
not row
or str(row["attachment_id"]) != data["attachment_id"]
or row["pipeline_version"] != data["pipeline_version"]
):
return "obsolete", None
if row["state"] in {"ready", "failed", "obsolete"}:
return row["state"], row
if data["attempt"] < row["attempt"] or (
row["state"] == "queued" and data["attempt"] <= row["attempt"]
):
return "obsolete", row
if row["state"] == "processing":
return "busy", row
if data["attempt"] != row["attempt"] + 1:
return "obsolete", row
due = conn.execute(
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
).fetchone()["due"]
if not due:
return "busy", row
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
updated = conn.execute(
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
(consumer, lease, row["id"]),
).fetchone()
conn.execute(
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
)
event(
conn,
row["id"],
"claimed",
{
"attempt": updated["attempt"],
"fence": updated["fence"],
"version": updated["pipeline_version"],
},
)
return "claimed", updated
def ready_effect(db, store, row):
with db.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
(row["attachment_id"], row["pipeline_version"]),
).fetchall()
if len(variants) != 4:
return False
for variant in variants:
head = store.head(key=variant["object_key"])
if (
head is None
or head.size != variant["size_bytes"]
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
):
return False
return True
def consume(slot):
db, queue, store = database(), broker(), ObjectStore()
consumer = str(uuid4()) + f"-{slot}"
context = multiprocessing.get_context("spawn")
cursor = "0-0"
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
if lease <= timeout:
raise ValueError("Lease must exceed child timeout")
try:
while True:
try:
try:
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
reclaimed = queue.xautoclaim(
STREAM,
GROUP,
consumer,
# Do not steal an entry from a consumer inside its live budget.
min_idle_time=lease * 1000,
start_id=cursor,
count=1,
)
cursor, messages = reclaimed[0], reclaimed[1]
if not messages:
batch = queue.xreadgroup(
GROUP, consumer, {STREAM: ">"}, count=1, block=500
)
messages = batch[0][1] if batch else []
for entry, fields in messages:
try:
data = validate_job(
json.loads(fields.get("job", "null"))
)
except ValueError, TypeError, KeyError:
# Malformed notifications cannot become jobs; preserve diagnostic count.
with db.connection() as conn:
event(conn, None, "invalid_envelope")
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
continue
with telemetry.span(
"attachments consume",
kind=telemetry.SpanKind.CONSUMER,
parent=data,
attributes={
"job.id": data["job_id"],
"messaging.system": "valkey",
"messaging.destination.name": STREAM,
},
):
state, row = claim(db, data, consumer)
if state == "busy":
continue
if state == "ready" and not ready_effect(
db, store, row
):
with db.connection() as conn:
event(conn, row["id"], "ready_effect_missing")
continue
if state in {"ready", "obsolete", "failed"}:
with db.connection() as conn:
event(
conn,
data["job_id"],
"duplicate_decision",
{"state": state, "entry": entry},
)
if state == "claimed":
attempt_started = time.monotonic()
receiver, sender = context.Pipe(duplex=False)
child = context.Process(
target=process,
args=(
str(row["id"]),
row["fence"],
sender,
telemetry.carrier(),
),
)
child.start()
telemetry.count("worker_busy", 1)
sender.close()
child.join(timeout)
if child.is_alive():
child.kill()
child.join(2)
result = {
"result": "transient",
"code": "worker_timeout",
}
else:
try:
result = (
receiver.recv()
if receiver.poll()
else {
"result": "transient",
"code": "worker_crashed",
}
)
except EOFError:
result = {
"result": "transient",
"code": "worker_crashed",
}
telemetry.count("worker_busy", -1)
labels = {
"job_type": "attachment.process.v1",
"outcome": result["result"],
}
telemetry.count("job_attempts", attributes=labels)
telemetry.duration(
"job_duration",
time.monotonic() - attempt_started,
labels,
)
telemetry.log(
"job.attempt_finished",
job_id=str(row["id"]),
attempt=row["attempt"],
version=row["pipeline_version"],
outcome=result["result"],
error_code=result.get("code", ""),
)
receiver.close()
child.close()
with db.connection() as conn, conn.transaction():
current = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
(row["id"],),
).fetchone()
event(
conn, row["id"], "child_finished", result
)
if (
current["state"] == "processing"
and current["fence"] == row["fence"]
):
retry(
conn,
current,
result.get(
"code", "processing_failed"
),
permanent=result["result"]
== "permanent",
)
# Commit result or a retry/terminal decision precedes ack.
queue.xack(STREAM, GROUP, entry)
queue.xdel(STREAM, entry)
except Exception:
print(
json.dumps(
{
"event": "worker_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(0.5)
finally:
queue.close()
store.close()
db.close()
def main():
telemetry.configure("messenger-worker")
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
threads = [
threading.Thread(target=consume, args=(i,), daemon=False)
for i in range(count)
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
telemetry.stop()
if __name__ == "__main__":
main()
+119
View File
@@ -0,0 +1,119 @@
import json
import os
from pathlib import Path
from uuid import uuid4
import httpx
import pytest
from jsonschema import Draft202012Validator, FormatChecker
CONTRACT = json.loads(
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
)
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
PASSWORD = "Synthetic-Password-🌍-123"
TOKENS = {}
def check(result: httpx.Response, status: int, schema: str) -> dict:
assert result.status_code == status, result.text
assert result.headers["x-request-id"]
assert result.headers["x-instance-id"]
assert result.headers["content-type"].startswith("application/json")
payload = result.json()
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
payload
)
if schema == "Error":
assert payload["error"]["request_id"] == result.headers["x-request-id"]
assert (
payload["error"]["code"]
== {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
500: "internal_error",
503: "unavailable",
}[status]
)
return payload
def register(client: httpx.Client, **overrides) -> dict:
data = {
"username": "u_" + uuid4().hex[:24],
"display_name": "Алиса 🌍",
"password": PASSWORD,
}
data.update(overrides)
user = check(client.post("/api/v1/users", json=data), 201, "User")
session = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(data["username"], data["password"]),
),
201,
"Session",
)
TOKENS[user["id"]] = session["token"]
return user
@pytest.fixture
def client():
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as client:
yield client
@pytest.fixture
def actors(client):
return register(client), register(client), register(client)
@pytest.fixture
def chat(client, actors):
alice, bob, _ = actors
return check(
client.post(
"/api/v1/chats",
headers=headers(alice),
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
),
201,
"Chat",
)
def headers(actor: dict) -> dict:
return {
"Authorization": "Bearer " + TOKENS[actor["id"]],
"Idempotency-Key": uuid4().hex,
}
def history(client, actor, chat_id, limit=7):
found, cursor = [], None
while True:
params = {"limit": limit}
if cursor:
params["cursor"] = cursor
page = check(
client.get(
f"/api/v1/chats/{chat_id}/messages",
headers=headers(actor),
params=params,
),
200,
"MessagePage",
)
found.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return found
+412
View File
@@ -0,0 +1,412 @@
import base64
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from uuid import UUID, uuid4
import httpx
import pytest
from conftest import BASE_URL, PASSWORD, check, headers, history, register
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
def test_health(client, path):
assert check(client.get(path), 200, "Health") == {"status": "ok"}
@pytest.mark.parametrize(
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
)
def test_identification(client, value):
check(
client.get(
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
),
401,
"Error",
)
def test_identity(client, actors):
assert (
check(
client.get("/api/v1/users/me", headers=headers(actors[0])),
200,
"User",
)
== actors[0]
)
@pytest.mark.parametrize(
"data",
[
{},
[],
None,
{"username": "ABC", "display_name": "A"},
{"username": "abc\n", "display_name": "A"},
{"username": 123, "display_name": "A"},
{"username": "abc", "display_name": " "},
{"username": "abc", "display_name": "x" * 101},
{"username": "abc", "display_name": "A", "extra": True},
{"username": "abc", "display_name": "\ud800"},
],
)
def test_invalid_user(client, data):
if isinstance(data, dict):
data = {"password": PASSWORD, **data}
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
@pytest.mark.parametrize(
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
)
def test_malformed_json(client, raw):
check(client.post("/api/v1/users", content=raw), 400, "Error")
def test_auth_before_json(client):
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
def test_duplicate_username(client, actors):
check(
client.post(
"/api/v1/users",
json={
"username": actors[0]["username"],
"display_name": "B",
"password": PASSWORD,
},
),
409,
"Error",
)
def test_unicode_codepoints(client, actors, chat):
user = register(client, display_name="🌍" * 100)
assert user["display_name"] == "🌍" * 100
text = "🌍" * 4000
result = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": text},
),
201,
"Message",
)
assert result["text"] == text
assert result["sender_id"] == actors[0]["id"]
@pytest.mark.parametrize(
"method,suffix,body",
[
("GET", "", None),
("GET", "/messages", None),
("POST", "/messages", {"text": "hello"}),
],
)
def test_acl(client, actors, chat, method, suffix, body):
outsider = headers(actors[2])
actual = check(
client.request(
method,
f"/api/v1/chats/{chat['id']}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
missing = check(
client.request(
method,
f"/api/v1/chats/{uuid4()}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
assert actual["error"]["message"] == missing["error"]["message"]
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
def test_bad_message(client, actors, chat, value):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": value},
),
400,
"Error",
)
def test_sender_not_client_controlled(client, actors, chat):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "Hi", "sender_id": actors[1]["id"]},
),
400,
"Error",
)
@pytest.mark.parametrize(
"case",
[
"duplicate",
"unknown",
"no_caller",
"empty",
"wrong_type",
"bad_uuid",
"too_many",
"blank_title",
],
)
def test_invalid_chat(client, actors, case):
a, b, _ = actors
members = {
"duplicate": [a["id"], a["id"]],
"unknown": [a["id"], str(uuid4())],
"no_caller": [b["id"]],
"empty": [],
"wrong_type": a["id"],
"bad_uuid": ["broken"],
"too_many": [str(uuid4()) for _ in range(101)],
"blank_title": [a["id"]],
}[case]
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={
"title": " " if case == "blank_title" else "Title",
"member_ids": members,
},
),
400,
"Error",
)
@pytest.mark.parametrize(
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
)
def test_bad_limit(client, actors, chat, limit):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"limit": limit},
),
400,
"Error",
)
@pytest.mark.parametrize(
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
)
def test_bad_cursor(client, actors, chat, cursor):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"cursor": cursor},
),
400,
"Error",
)
def test_pagination(client, actors, chat):
a, b, outsider = actors
sent = []
path = f"/api/v1/chats/{chat['id']}/messages"
for i in range(23):
sent.append(
check(
client.post(
path,
headers=headers(a if i % 2 else b),
json={"text": f"Msg {i}"},
),
201,
"Message",
)
)
got = history(client, a, chat["id"])
assert got == sorted(
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
)
first = check(
client.get(path, headers=headers(a), params={"limit": 1}),
200,
"MessagePage",
)
second = check(
client.get(
path,
headers=headers(b),
params={"limit": 22, "cursor": first["next_cursor"]},
),
200,
"MessagePage",
)
assert second["items"] == got[1:]
assert second["next_cursor"] is None
chats = check(
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
)
assert chats["items"] == []
other = check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": "Other", "member_ids": [a["id"]]},
),
201,
"Chat",
)
check(
client.get(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(a),
params={"cursor": first["next_cursor"]},
),
400,
"Error",
)
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
data = json.loads(
base64.urlsafe_b64decode(
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
)
)
data["at"] = 17
forged = (
base64.urlsafe_b64encode(json.dumps(data).encode())
.decode()
.rstrip("=")
)
check(
client.get(path, headers=headers(a), params={"cursor": forged}),
400,
"Error",
)
def test_chat_pagination_scope(client, actors):
a, b, _ = actors
created = []
for i in range(4):
created.append(
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
),
201,
"Chat",
)
)
page = check(
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
200,
"ChatPage",
)
assert page["items"] == created[:2]
last = check(
client.get(
"/api/v1/chats",
headers=headers(a),
params={"limit": 2, "cursor": page["next_cursor"]},
),
200,
"ChatPage",
)
assert last["items"] == created[2:]
assert last["next_cursor"] is None
check(
client.get(
"/api/v1/chats",
headers=headers(b),
params={"cursor": page["next_cursor"]},
),
400,
"Error",
)
def test_concurrent_messages(client, actors, chat):
barrier = threading.Barrier(10)
a = actors[0]
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(i):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
return [
check(
conn.post(
path, headers=headers(a), json={"text": f"{i}:{j}"}
),
201,
"Message",
)["id"]
for j in range(10)
]
with ThreadPoolExecutor(max_workers=10) as pool:
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
rows = history(client, a, chat["id"])
assert len(rows) == len(ids) == len(set(ids)) == 100
assert set(ids) == {row["id"] for row in rows}
assert len({row["text"] for row in rows}) == 100
def test_registration_race(client):
username = "race_" + uuid4().hex[:20]
barrier = threading.Barrier(10)
def worker(_):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
result = conn.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Test",
"password": PASSWORD,
},
)
check(
result,
result.status_code,
"User" if result.status_code == 201 else "Error",
)
return result.status_code
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(worker, range(10)))
assert statuses.count(201) == 1
assert statuses.count(409) == 9
def test_unknown_path(client):
check(client.get("/does-not-exist"), 404, "Error")
+277
View File
@@ -0,0 +1,277 @@
import hashlib
import io
import time
from pathlib import Path
from uuid import uuid4
import pytest
from conftest import check, headers
from PIL import Image
PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes()
def upload(
client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png"
):
return client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actor), "Content-Type": mime},
params={"filename": filename},
content=data,
)
def wait_ready(client, actor, attachment):
end = time.monotonic() + 60
while time.monotonic() < end:
row = check(
client.get(
f"/api/v1/attachments/{attachment['id']}",
headers=headers(actor),
),
200,
"Attachment",
)
if row["status"] == "ready":
return row
assert row["status"] != "failed", row
time.sleep(0.2)
raise TimeoutError("Pipeline readiness")
def test_attachment_roundtrip_acl_and_message(client, actors, chat):
attachment = check(upload(client, actors[0], chat), 202, "Attachment")
assert attachment["status"] == "queued" and attachment["variants"] == []
attachment = wait_ready(client, actors[0], attachment)
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert attachment["size_bytes"] == len(PNG)
path = f"/api/v1/attachments/{attachment['id']}"
assert (
check(client.get(path, headers=headers(actors[1])), 200, "Attachment")
== attachment
)
downloaded = client.get(path + "/content", headers=headers(actors[1]))
assert downloaded.status_code == 200 and downloaded.content == PNG
assert downloaded.headers["Content-Type"] == "image/png"
assert not downloaded.history
for suffix in ["", "/content"]:
check(
client.get(path + suffix, headers=headers(actors[2])), 404, "Error"
)
check(client.get(path + suffix), 401, "Error")
thumb = client.get(
path + "/content",
headers=headers(actors[1]),
params={"variant": "thumbnail"},
)
assert thumb.status_code == 200
with Image.open(io.BytesIO(thumb.content)) as image:
assert image.format == "JPEG" and image.size == (256, 160)
sent = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[1]),
json={"text": "", "attachment_ids": [attachment["id"]]},
),
201,
"Message",
)
assert sent["attachment_ids"] == [attachment["id"]]
assert sent["sender_id"] == actors[1]["id"]
@pytest.mark.parametrize(
"name",
[
"../x.png",
"x/y.png",
"x\\y.png",
"x\x00.png",
"x\x7f.png",
"x\n.png",
"a" * 256,
"",
],
)
def test_filename_guard(client, actors, chat, name):
check(upload(client, actors[0], chat, filename=name), 400, "Error")
@pytest.mark.parametrize(
"data,mime,status",
[
(b"not an image", "image/png", 415),
(PNG, "image/jpeg", 415),
(PNG, "application/octet-stream", 415),
(PNG[:25], "image/png", 415),
(b"", "image/png", 415),
(b"x" * (10 * 1024 * 1024 + 1), "image/png", 413),
],
)
def test_image_guards(client, actors, chat, data, mime, status):
r = upload(client, actors[0], chat, data=data, mime=mime)
check(r, status, "Error")
def test_jpeg_original_unchanged(client, actors, chat):
stream = io.BytesIO()
Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG")
jpeg = stream.getvalue()
attachment = check(
upload(
client,
actors[0],
chat,
data=jpeg,
filename="camera.jpg",
mime="image/jpeg",
),
202,
"Attachment",
)
attachment = wait_ready(client, actors[0], attachment)
r = client.get(
f"/api/v1/attachments/{attachment['id']}/content",
headers=headers(actors[1]),
)
assert r.content == jpeg
def test_cross_chat_link_and_unknown(client, actors, chat):
a = check(upload(client, actors[0], chat), 202, "Attachment")
other = check(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "Other", "member_ids": [actors[0]["id"]]},
),
201,
"Chat",
)
for identity in [a["id"], str(uuid4())]:
check(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [identity]},
),
404,
"Error",
)
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [a["id"], a["id"]]},
),
400,
"Error",
)
def test_default_attachments_are_semantically_equal(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
first = check(
client.post(path, headers=h, json={"text": "Semantic default"}),
201,
"Message",
)
second = check(
client.post(
path,
headers=h,
json={"attachment_ids": [], "text": "Semantic default"},
),
201,
"Message",
)
assert first == second and first["attachment_ids"] == []
def test_pixel_limit_before_decode(client, actors, chat):
# Compressible synthetic image stays below10MiB but exceeds40M pixels.
stream = io.BytesIO()
Image.new("1", (8000, 5001)).save(stream, format="PNG")
assert len(stream.getvalue()) < 10 * 1024 * 1024
r = upload(client, actors[0], chat, data=stream.getvalue())
assert r.status_code == 413
def test_chunked_body_limit_without_content_length(client, actors, chat):
r = client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actors[0]), "Content-Type": "image/png"},
params={"filename": "chunked.png"},
content=iter([b"x" * (10 * 1024 * 1024 + 1)]),
)
assert r.request.headers.get("content-length") is None
assert r.status_code == 413
assert r.json()["error"]["code"] == "payload_too_large"
assert r.json()["error"]["request_id"] == r.headers["x-request-id"]
@pytest.mark.parametrize("size", [(1, 17), (320, 200), (1600, 900)])
def test_variant_geometry_and_alpha(client, actors, chat, size):
stream = io.BytesIO()
Image.new("RGBA", size, (0, 0, 0, 0)).save(stream, format="PNG")
a = check(
upload(client, actors[0], chat, data=stream.getvalue()),
202,
"Attachment",
)
a = wait_ready(client, actors[0], a)
for v in a["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=headers(actors[1]),
params={"variant": v["name"]},
)
assert len(r.content) == v["size_bytes"]
assert hashlib.sha256(r.content).hexdigest() == v["sha256"]
with Image.open(io.BytesIO(r.content)) as image:
assert image.size == (v["width"], v["height"])
assert not image.getexif()
assert min(image.convert("RGB").getpixel((0, 0))) >= 250
if v["name"] == "crop":
assert image.size == (128, 128)
else:
bound = 256 if v["name"] == "thumbnail" else 1280
assert max(image.size) <= bound
assert image.width <= size[0] and image.height <= size[1]
def test_exif_orientation_applied_but_original_preserved(client, actors, chat):
stream = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
exif[270] = "Synthetic metadata must be removed"
Image.new("RGB", (320, 200), "red").save(stream, format="JPEG", exif=exif)
raw = stream.getvalue()
a = check(
upload(
client,
actors[0],
chat,
data=raw,
mime="image/jpeg",
filename="exif.jpg",
),
202,
"Attachment",
)
a = wait_ready(client, actors[0], a)
r = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=headers(actors[1])
)
assert r.content == raw
v = next(v for v in a["variants"] if v["name"] == "thumbnail")
assert (v["width"], v["height"]) == (160, 256)
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=headers(actors[1]),
params={"variant": "thumbnail"},
)
with Image.open(io.BytesIO(r.content)) as image:
assert image.size == (160, 256) and not image.getexif()
+26
View File
@@ -0,0 +1,26 @@
import json
from uuid import uuid4
import pytest
from conftest import check, headers
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
def test_only_utf8_json(client, encoding):
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
payload["password"] = "Synthetic-encoding-test-123"
result = client.post(
"/api/v1/users",
content=json.dumps(payload).encode(encoding),
headers={"Content-Type": "application/json"},
)
check(result, 400, "Error")
def test_leading_zero_limit(client, actors, chat):
result = client.get(
"/api/v1/chats/" + chat["id"] + "/messages",
headers=headers(actors[0]),
params={"limit": "0001"},
)
check(result, 200, "MessagePage")
+23
View File
@@ -0,0 +1,23 @@
from conftest import check, headers
def test_history_acl_precedes_query_validation(client, actors, chat):
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.get(
path,
headers=headers(actors[2]),
params={"limit": 0, "cursor": "broken"},
),
404,
"Error",
)
check(
client.get(
path,
headers=headers(actors[0]),
params={"limit": 0, "cursor": "broken"},
),
400,
"Error",
)
+75
View File
@@ -0,0 +1,75 @@
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import httpx
import pytest
from conftest import BASE_URL, check, headers, history
@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"])
def test_key_required_and_valid(client, actors, chat, key):
h = headers(actors[0])
h.pop("Idempotency-Key")
if key is not None:
h["Idempotency-Key"] = key
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=h,
json={"text": "Key"},
),
400,
"Error",
)
def test_parallel_replay_across_instances(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
def send(_):
with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c:
r = c.post(path, headers=h, json={"text": "Один результат 🌍"})
return check(r, 201, "Message"), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({r[0]["id"] for r in results}) == 1
assert len({r[1] for r in results}) >= 2
assert history(client, actors[0], chat["id"]) == [results[0][0]]
check(
client.post(path, headers=h, json={"text": "Другой текст"}),
409,
"Error",
)
# Другой пользователь может независимо использовать тот же ключ.
other = headers(actors[1])
other["Idempotency-Key"] = h["Idempotency-Key"]
second = check(
client.post(path, headers=other, json={"text": "Боб"}), 201, "Message"
)
assert second["id"] != results[0][0]["id"]
def test_replay_rechecks_acl(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.post(path, headers=h, json={"text": "Existing"}), 201, "Message"
)
outsider = headers(actors[2])
outsider["Idempotency-Key"] = h["Idempotency-Key"]
check(
client.post(path, headers=outsider, json={"text": "Existing"}),
404,
"Error",
)
check(
client.post(
path,
headers={"Idempotency-Key": uuid4().hex},
json={"text": "Existing"},
),
401,
"Error",
)
+114
View File
@@ -0,0 +1,114 @@
from uuid import uuid4
import httpx
import pytest
from conftest import PASSWORD, TOKENS, check, headers, register
@pytest.mark.parametrize(
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
)
def test_password_validation(client, password):
check(
client.post(
"/api/v1/users",
json={
"username": "pwd_" + uuid4().hex[:20],
"display_name": "Test",
"password": password,
},
),
400,
"Error",
)
def test_full_utf8_password(client):
password = "🌍" * 127 + ":"
user = register(client, password=password)
assert (
check(
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
)
== user
)
@pytest.mark.parametrize(
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
)
def test_bad_basic(client, authorization):
result = client.post(
"/api/v1/auth/sessions",
headers={"Authorization": authorization} if authorization else {},
)
check(result, 401, "Error")
assert (
result.headers["www-authenticate"]
== 'Basic realm="messenger", charset="UTF-8"'
)
def test_wrong_password_and_unknown_user(client):
user = register(client)
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
result = client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
)
data = check(result, 401, "Error")
assert data["error"]["message"] == "Authentication required"
def test_logout_is_not_revived_by_cache(client):
user = register(client)
auth = headers(user)
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
assert result.status_code == 204 and result.content == b""
assert result.headers["x-request-id"]
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
check(
client.delete("/api/v1/auth/sessions/current", headers=auth),
401,
"Error",
)
def test_only_current_session_revoked(client):
user = register(client)
another = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(user["username"], PASSWORD),
),
201,
"Session",
)
assert another["token"] != TOKENS[user["id"]]
assert len(another["token"]) >= 32
assert (
client.delete(
"/api/v1/auth/sessions/current", headers=headers(user)
).status_code
== 204
)
result = client.get(
"/api/v1/users/me",
headers={"Authorization": "Bearer " + another["token"]},
)
assert check(result, 200, "User") == user
def test_bearer_source_and_ignored_user_header(client, actors):
alice, bob, _ = actors
auth = {**headers(alice), "X-User-Id": bob["id"]}
assert (
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
== alice["id"]
)
result = client.get(
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
)
check(result, 401, "Error")
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
+119
View File
@@ -0,0 +1,119 @@
import copy
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from jsonschema import Draft202012Validator, FormatChecker
from src.pipeline import validate_job
def example():
return dict(
schema_version=1,
job_id=str(uuid4()),
type="attachment.process.v1",
attachment_id=str(uuid4()),
pipeline_version=1,
attempt=1,
created_at=datetime.now(UTC).isoformat(),
)
def test_schema_and_runtime_agree_on_valid_job():
import json
from pathlib import Path
schema = json.loads(Path("contracts/jobs.schema.json").read_text())
for version in [1, 2]:
data = example()
data["pipeline_version"] = version
data.update(
traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01",
tracestate="vendor=opaque",
)
Draft202012Validator(schema, format_checker=FormatChecker()).validate(
data
)
assert validate_job(data) == data
@pytest.mark.parametrize(
"field,value",
[
("schema_version", True),
("schema_version", 2),
("type", "other"),
("job_id", 123),
("job_id", "broken"),
("attachment_id", None),
("pipeline_version", 0),
("pipeline_version", True),
("attempt", 0),
("attempt", 1.0),
("created_at", "2026-01-01"),
("created_at", "2026-01-01T12:00:00"),
("created_at", None),
("traceparent", "not-w3c"),
("tracestate", "x" * 513),
("url", "http://invalid.example"),
("bytes", "secret-contents"),
("filename", "../../bad"),
],
)
def test_invalid_notification_is_rejected(field, value):
data = copy.deepcopy(example())
data[field] = value
with pytest.raises((ValueError, TypeError)):
validate_job(data)
def test_worker_rejects_oversized_pixels_before_decode():
import hashlib
import io
from PIL import Image
from src.pipeline import PermanentFailure
from src.worker import make_variants
stream = io.BytesIO()
Image.new("1", (8000, 5001)).save(stream, format="PNG")
raw = stream.getvalue()
row = dict(
size_bytes=len(raw),
sha256=hashlib.sha256(raw).hexdigest(),
content_type="image/png",
pipeline_version=1,
)
with (
pytest.warns(Image.DecompressionBombWarning),
pytest.raises(PermanentFailure, match="pixel_limit"),
):
list(make_variants(raw, row))
def test_worker_rejects_changed_source_checksum():
from src.pipeline import PermanentFailure
from src.worker import make_variants
with pytest.raises(PermanentFailure, match="source_checksum_mismatch"):
list(make_variants(b"changed", dict(size_bytes=7, sha256="0" * 64)))
def test_worker_rejects_invalid_image_as_permanent():
import hashlib
from src.pipeline import PermanentFailure
from src.worker import make_variants
raw = b"not an image"
with pytest.raises(PermanentFailure, match="invalid_image"):
list(
make_variants(
raw,
dict(
size_bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()
),
)
)
+117
View File
@@ -0,0 +1,117 @@
import asyncio
import json
from types import SimpleNamespace
import pytest
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from src import telemetry
@pytest.fixture
def signals(monkeypatch):
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
logs = []
monkeypatch.setattr(
telemetry,
"_state",
SimpleNamespace(
tracer=provider.get_tracer("synthetic-test"),
service="messenger-api",
logger=SimpleNamespace(emit=lambda **record: logs.append(record)),
),
)
yield exporter, logs
provider.shutdown()
def test_trace_context_survives_separate_publish_and_consume(signals):
exporter, _ = signals
with telemetry.span("upload") as upload:
accepted_context = telemetry.carrier()
with telemetry.span(
"publish", kind=telemetry.SpanKind.PRODUCER, parent=accepted_context
) as publish:
notification = telemetry.carrier()
with telemetry.span(
"consume", kind=telemetry.SpanKind.CONSUMER, parent=notification
) as consume:
with telemetry.span("s3.put"):
pass
spans = {s.name: s for s in exporter.get_finished_spans()}
assert len({s.context.trace_id for s in spans.values()}) == 1
assert spans["publish"].parent.span_id == upload.get_span_context().span_id
assert (
spans["consume"].parent.span_id == publish.get_span_context().span_id
)
assert spans["s3.put"].parent.span_id == consume.get_span_context().span_id
def test_secret_in_exception_and_unapproved_fields_is_not_exported(
signals, capsys
):
exporter, logs = signals
sensitive = "SyntheticSensitiveValueForPrivacyProbe"
with pytest.raises(ValueError):
with telemetry.span(
"db.operation", attributes={"db.query.text": sensitive}
):
telemetry.log(
"request.failed",
request_id="safe-request",
password=sensitive,
Authorization=sensitive,
body=sensitive,
)
raise ValueError(sensitive)
exported = exporter.get_finished_spans()[0]
assert exported.events == ()
assert sensitive not in str(exported.attributes)
assert sensitive not in exported.status.description
assert sensitive not in json.dumps(logs, default=str)
assert sensitive not in capsys.readouterr().out
def test_http_metrics_use_route_template_and_bounded_method(
signals, monkeypatch, capsys
):
exporter, logs = signals
counts = []
monkeypatch.setattr(
telemetry, "count", lambda name, **kw: counts.append((name, kw))
)
sensitive = "SyntheticPrivateFilenameAndToken"
async def application(scope, receive, send):
scope["route"] = SimpleNamespace(
path="/api/v1/chats/{chat_id}/messages"
)
await send({"type": "http.response.start", "status": 201})
await send({"type": "http.response.body", "body": sensitive.encode()})
async def send(_):
pass
scope = {
"type": "http",
"path": "/api/v1/chats/" + sensitive + "/messages",
"query_string": sensitive.encode(),
"method": "CUSTOM_UNBOUNDED_METHOD",
"headers": [(b"authorization", sensitive.encode())],
"state": {"request_id": "safe-request"},
}
asyncio.run(telemetry.TelemetryMiddleware(application)(scope, None, send))
assert (
counts[0][1]["attributes"]["route"]
== "/api/v1/chats/{chat_id}/messages"
)
assert counts[0][1]["attributes"]["method"] == "OTHER"
assert sensitive not in json.dumps(logs, default=str)
assert sensitive not in str(exporter.get_finished_spans()[0].attributes)
assert sensitive not in capsys.readouterr().out
Generated
+1230
View File
File diff suppressed because it is too large Load Diff