103 changed files with 7114 additions and 35 deletions
+37 -14
View File
@@ -1,30 +1,53 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Local synthetic lab environment only. Copy to .env and replace every change-me value.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
LOG_LEVEL=info
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-postgres
POSTGRES_DB=messenger
DATABASE_URL=postgresql+asyncpg://messenger:change-me-postgres@postgres:5432/messenger
REDIS_URL=redis://redis:6379/0
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=change-me-minio-root
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=change-me-minio-api
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
BROKER_USER=messenger
BROKER_PASSWORD=change-me-rabbitmq
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=90
RETRY_BASE_SECONDS=2
RETRY_MAX_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
THUMBNAIL_JPEG_QUALITY=85
CROP_JPEG_QUALITY=85
OPTIMIZED_WEBP_QUALITY=82
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
OTEL_SERVICE_VERSION=1.6.0
OTEL_DEPLOYMENT_ENVIRONMENT=lab
OTEL_SDK_DISABLED=false
OTEL_EXPORTER_OTLP_TIMEOUT=2
OTEL_BSP_MAX_QUEUE_SIZE=2048
OTEL_BLRP_MAX_QUEUE_SIZE=2048
OTEL_METRIC_EXPORT_INTERVAL=5000
GRAFANA_PORT=3000
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=change-me-grafana
+17
View File
@@ -0,0 +1,17 @@
FROM python:3.13.15-slim
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install -r requirements.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
COPY app ./app
COPY alembic ./alembic
COPY alembic.ini .
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.13.15-slim
WORKDIR /tests
COPY requirements.txt requirements-dev.txt ./
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
COPY lab.json .
COPY app ./app
COPY contracts ./contracts
COPY tests ./tests
RUN chown -R tests:tests /tests
USER 10001:10001
CMD ["python", "-m", "pytest", "-q"]
+18 -2
View File
@@ -3,11 +3,12 @@ BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
OBSERVABILITY_COMPOSE = docker compose -f compose.yaml -f compose.observability-tests.yaml
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-observability test-retry-trace test-alert-worker test-alert-database test-telemetry-outage load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: offline structure; validate: full OpenAPI; test: public API; test-observability: Grafana/Prometheus/Loki/Tempo"
check:
$(PYTHON) scripts/check_contract.py
@@ -18,6 +19,21 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-observability:
$(OBSERVABILITY_COMPOSE) run --rm observability-test
test-retry-trace:
$(PYTHON) tests/retry_trace.py --base-url "$(BASE_URL)"
test-alert-worker:
$(PYTHON) tests/alert_scenarios.py worker --base-url "$(BASE_URL)"
test-alert-database:
$(PYTHON) tests/alert_scenarios.py database --base-url "$(BASE_URL)"
test-telemetry-outage:
$(PYTHON) tests/telemetry_outage.py --base-url "$(BASE_URL)"
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+188 -18
View File
@@ -1,37 +1,207 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Сергей Возжин, БПИ2301.
- Целевая оценка: **4**. Трек оценки 5 не заявлен.
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/61
- Использованы Python 3.13, FastAPI, OpenTelemetry Python 1.31.1,
OTel Collector Contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2,
Tempo 2.7.2 и Grafana 11.6.0. Для планирования и реализации применялся ИИ.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Compose v2, GNU Make и Python 3.9+ для host-side тестов.
Все образы и Python-зависимости закреплены в репозитории.
```sh
cp .env.example .env
# Заменить все change-me значения согласованными локальными секретами.
# Пароли в DATABASE_URL/BROKER_URL должны совпадать с POSTGRES_*/BROKER_*.
docker compose config --quiet
docker compose up -d --build --wait
make check
make test BASE_URL=http://localhost:8080
make test-observability
```
Grafana доступна только локально на `http://127.0.0.1:3000`. Dashboard,
datasources и alerts создаются provisioning-механизмом автоматически. Миграции
Alembic выполняет одноразовый сервис `migrate`; MinIO bucket и технического
пользователя создаёт `s3-bootstrap`.
Безопасная остановка без удаления данных:
```sh
docker compose down
```
Намеренное удаление всех **локальных учебных** данных выполняется отдельно и
необратимо:
```sh
docker compose down --volumes
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
client -> HAProxy -> API x2 -> PostgreSQL / Redis / S3
|
+-> transactional outbox -> publisher -> RabbitMQ -> worker -> S3 + PostgreSQL
API / publisher / worker -> OTLP HTTP -> OTel Collector
|-> Prometheus exporter <- Prometheus
|-> OTLP logs -> Loki
`-> OTLP traces -> Tempo
Prometheus + Loki + Tempo <- provisioned datasources <- Grafana
```
Из ЛР 5 без изменения бизнес-API перенесены две API-реплики, HAProxy,
PostgreSQL, Redis sessions/idempotency, MinIO S3, RabbitMQ, transactional
outbox, publisher, worker и обработка изображений. PostgreSQL остаётся источником
истины для attachments/jobs/outbox. Durable acceptance атомарно создаёт attachment,
job и outbox row; публикация и обработка повторяемы.
В outbox добавлены nullable `traceparent` и `tracestate`. API сохраняет W3C context
в той же транзакции, publisher восстанавливает его, создаёт PRODUCER span и передаёт
новый context в allowlisted AMQP headers. Worker извлекает headers и создаёт CONSUMER
span. В одной трассе видны `messenger-api`, `messenger-publisher`,
`messenger-worker`, SQL и ручные S3 CLIENT spans. Retry создаёт новую публикацию с
текущим context; повторная обработка остаётся наблюдаемой и идемпотентной.
SDK использует bounded batch queues по 2048 элементов и timeout 2 с. Collector:
`memory_limiter` 192 MiB со spike 48 MiB, `batch` до 1024 элементов, exporter queues
по 500 элементов, два consumer, exponential retry не дольше 30 с. Поэтому отказ
Loki/Tempo приводит к ограниченной потере telemetry, а не блокировке API или
неограниченному росту RAM.
Collector self-metrics экспортируются через `readers.pull` на `:8888`;
Prometheus сохраняет RSS, размеры очередей и `send_failed` counters.
Секреты и пользовательское содержимое не экспортируются. `safe_extra` отбрасывает
пароли, tokens, Authorization, secrets, message body, image bytes и signed URL.
Метрики содержат только низкокардинальные route templates, method/status class,
job type/outcome и queue name; UUID и filename labels отсутствуют. Технические IDs
допустимы только в logs/spans для синтетических данных.
## Сигналы, dashboard и единицы
| Контракт | Реальная серия | Единица / основные labels |
| --- | --- | --- |
| HTTP requests | `http_server_request_count_total` | requests; method, route template, status class |
| HTTP duration | `http_server_request_duration_seconds` | seconds, histogram |
| Jobs outcome | `messenger_jobs_processed_total` | jobs; job type, outcome |
| Job duration | `messenger_job_processing_duration_seconds` | seconds, histogram |
| Time to ready | `messenger_attachment_time_to_ready_seconds` | seconds, histogram |
| Queue | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs / seconds |
| DB pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections / waiters |
| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots |
| Process | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | CPU seconds / bytes |
Dashboard UID `messenger-observability` содержит 12 panels: RPS, 5xx ratio,
HTTP p50/p95/p99, jobs по outcome, worker p50/p95/p99, end-to-end p50/p95/p99,
queue depth/oldest, DB pool, worker saturation, CPU, RSS и correlated logs.
Percentiles вычисляются из объединённых buckets всех реплик, например:
```promql
histogram_quantile(0.99,
sum by (le) (rate(http_server_request_duration_seconds_bucket{
service_name="messenger-api",http_route!~"/health/.*"}[5m])))
```
Границы HTTP histogram: 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms,
1 s, 2.5 s, 5 s. Worker: 50 ms…60 s. Time-to-ready: 100 ms…120 s.
Средние значения и средние p99 отдельных реплик не используются.
## SLI, SLO и alerts
API availability SLI: доля бизнес-запросов со status `<500` среди всех
бизнес-запросов; `/health/live` и `/health/ready` исключены. Учебный SLO — не менее
99% good requests в rolling window 30 минут. 4xx входят в denominator, но не в
server-error numerator.
Pipeline SLI: доля durably accepted attachments, достигших `ready` не позднее
60 секунд. Учебный SLO — не менее 95% за rolling 30 минут при доступном worker.
Provisioned rules:
- `ApiHighServerErrorRate`: 5xx / all business requests за 2 минуты > 5%, `for: 1m`;
- `QueueOldestJobGrowing`: oldest job > 20 s при depth > 0, `for: 15s`.
Пороги и короткие окна выбраны для воспроизводимой учебной демонстрации, а не как
production defaults. У каждого правила есть dashboard/panel link и локальный runbook
в `infra/grafana/runbooks/`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Offline contract | `make check` | PASS | консоль и Git diff |
| Публичный API с telemetry | `make test` | `PASS: lab6 public API scenario` | `tests/smoke.py` |
| Dashboard/alerts/3 backends | `make test-observability` | PASS: dashboard, alerts, metrics, logs, cross-process trace | `tests/observability_smoke.py` |
| API → publisher → worker | тот же smoke после upload | один `trace_id`, services API/publisher/worker | Loki + Tempo API assertion |
| Retry в одной трассе | `make test-retry-trace` | 2 PRODUCER + 2 CONSUMER, SQL/S3, attachment ready | `tests/retry_trace.py` |
| Worker outage | `make test-alert-worker` | Alerting 61.8 s, Normal 15.5 s после запуска | `tests/alert_scenarios.py` |
| PostgreSQL outage | `make test-alert-database` | Alerting/Normal + linked log/failed PostgreSQL span | `tests/alert_scenarios.py` |
| Telemetry outage | `make test-telemetry-outage` | 49.5 s, smoke PASS, bounded RSS, loss counters | `tests/telemetry_outage.py` |
| Конфиги | `docker compose config --quiet`; JSON parse; Grafana logs | provisioning завершён без error | `infra/` и `compose.yaml` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Повторная проверка выполнена 2026-11-20: host Apple M4 Pro, 24 GiB RAM;
Docker Desktop VM — 12 CPU и 9.70 GiB RAM. Каждый Compose-сервис дополнительно
ограничен `cpus`/`mem_limit` из `compose.yaml`. Load evidence создал
25 638 синтетических сообщений за 120 с с concurrency 8.
## Эксперименты и отказы
## Разбор контролируемых инцидентов
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Временная ошибка S3 и retry
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Worker был остановлен до upload, затем S3 выключен только на первую
попытку. Worker сохранил `job.retry_scheduled`, после восстановления S3
то же attachment дошло до `ready`. Трасса
`e8e2aff13b7bdb632c9cec5007dcc6d4` содержит два PRODUCER и два CONSUMER
span, а также SQL/S3 CLIENT spans. Это доказывает перенос context
через retry outbox event.
### Worker недоступен
Гипотеза: API продолжит durable acceptance, RabbitMQ и PostgreSQL сохранят работу,
но `queue_depth` и `queue_oldest_age_seconds` вырастут. Сценарий создал отдельные
синтетические users/chat, остановил только worker, загрузил PNG и опрашивал состояние
Grafana rule. Alert перешёл в `Alerting` за 61.8 с. После `docker compose start worker`
attachment стал ready, backlog уменьшился, rule стал `Normal` через 15.5 с. Потери
бизнес-данных: 0. Диагноз по runbook: HTTP остаётся доступным, jobs не завершаются,
worker saturation отсутствует, oldest age растёт — причина на consumer-side.
### PostgreSQL недоступен
Гипотеза: прямые business requests к API дадут 5xx, DB spans/логи укажут connection
failure, а queue-only alert не обязан срабатывать. PostgreSQL остановлен, отдельный
контейнер создавал synthetic `POST /api/v1/users` каждые 250 ms. Error-rate alert
стал `Alerting` за 72.2 с. PostgreSQL запущен в `finally`, readiness восстановилась,
rule вернулся в `Normal` через 113.1 с — старые 5xx должны были выйти из 2-минутного
окна. Потери подтверждённых данных: 0; неуспешные запросы клиент может повторить.
Связанный log и trace `34f96d6fc934f7c54075c44a7b39c176` содержат
socket connection error и `connect` CLIENT span с `db.system=postgresql` и
`STATUS_CODE_ERROR`. Это отделяет отказ БД от просто медленного HTTP handler.
### Loki и Tempo недоступны
Оба backend остановлены одновременно на 49.5 с, что дольше настроенного
30-секундного retry. Полный бизнес-smoke прошёл, Collector остался running.
RSS Collector изменился с 194.6 до пиковых 198.0 MiB при container limit
256 MiB. Internal counters зафиксировали потерю 34 log records и 30 spans.
Порог `memory_limiter=192 MiB` — мягкий порог отказа/сброса данных, а не
обещание RSS ниже 192 MiB; жёсткий предел задан Compose-лимитом 256 MiB.
Это ожидаемая bounded telemetry loss: наблюдаемость не является частью
business commit.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Стенд single-node: PostgreSQL, RabbitMQ, MinIO и telemetry backends не имеют HA.
- Sampling всегда 100%; исследование стоимости и tail sampling относится к оценке 5.
- Grafana notification delivery не настроена: по заданию достаточно локальных
firing/resolved states.
- Queue depth берётся из durable job table, поэтому отражает бизнес-backlog, а не
точное число RabbitMQ messages в каждую микросекунду.
- Collector Prometheus exporter хранит только текущее состояние; долговременную
историю сохраняет Prometheus volume. Loki и Tempo используют локальные volumes.
- Для production нужны TLS/mTLS OTLP, secrets manager, HA backends, remote storage,
более длинные SLO windows и проверенные notification routes.
+36
View File
@@ -0,0 +1,36 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = placeholder://configured-at-runtime
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from app.config import load_settings
from app.db.postgres import Base
from app import models # noqa: F401 - imports ORM models into Base.metadata
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _url() -> str:
return load_settings().database_url
def run_migrations_offline() -> None:
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
with context.begin_transaction():
context.run_migrations()
def _run_migrations(connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def _run_migrations_async() -> None:
configuration = config.get_section(config.config_ini_section, {})
configuration['sqlalchemy.url'] = _url()
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(_run_migrations_async())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+22
View File
@@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,64 @@
"""Initial PostgreSQL schema for lab 2.
Revision ID: 20261019_0001
Revises:
Create Date: 2026-10-19
"""
from alembic import op
import sqlalchemy as sa
revision = '20261019_0001'
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'users',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('username', sa.String(length=32), nullable=False),
sa.Column('display_name', sa.String(length=100), nullable=False),
sa.Column('password_hash', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('username'),
)
op.create_table(
'chats',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
)
op.create_table(
'chat_members',
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('user_id', sa.Uuid(), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
)
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
op.create_table(
'messages',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('sender_id', sa.Uuid(), nullable=False),
sa.Column('text', sa.String(length=4000), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
def downgrade() -> None:
op.drop_index('messages_history_idx', table_name='messages')
op.drop_table('messages')
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
op.drop_table('chat_members')
op.drop_table('chats')
op.drop_table('users')
@@ -0,0 +1,35 @@
"""Add shared message idempotency for lab 3.
Revision ID: 20261023_0002
Revises: 20261019_0001
Create Date: 2026-10-23
"""
from alembic import op
import sqlalchemy as sa
revision = '20261023_0002'
down_revision = '20261019_0001'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
'messages',
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
)
op.create_unique_constraint(
'messages_sender_chat_idempotency_key',
'messages',
['sender_id', 'chat_id', 'idempotency_key'],
)
def downgrade() -> None:
op.drop_constraint(
'messages_sender_chat_idempotency_key',
'messages',
type_='unique',
)
op.drop_column('messages', 'idempotency_key')
@@ -0,0 +1,69 @@
"""Add S3 attachment metadata and message links for lab 4.
Revision ID: 20261030_0003
Revises: 20261023_0002
Create Date: 2026-10-30
"""
from alembic import op
import sqlalchemy as sa
revision = '20261030_0003'
down_revision = '20261023_0002'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'attachments',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('owner_id', sa.Uuid(), nullable=False),
sa.Column('filename', sa.String(length=255), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('status', sa.String(length=16), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
)
op.create_index(
'attachments_chat_created_idx',
'attachments',
['chat_id', 'created_at', 'id'],
)
op.create_table(
'message_attachments',
sa.Column('message_id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('position', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
sa.UniqueConstraint(
'message_id',
'position',
name='message_attachments_message_position_key',
),
)
op.create_index(
'message_attachments_attachment_idx',
'message_attachments',
['attachment_id'],
)
def downgrade() -> None:
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
op.drop_table('message_attachments')
op.drop_index('attachments_chat_created_idx', table_name='attachments')
op.drop_table('attachments')
@@ -0,0 +1,126 @@
"""Add asynchronous attachment jobs, variants and transactional outbox.
Revision ID: 20261116_0004
Revises: 20261030_0003
Create Date: 2026-11-16
"""
from alembic import op
import sqlalchemy as sa
revision = '20261116_0004'
down_revision = '20261030_0003'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.alter_column('attachments', 'object_key', new_column_name='original_key')
op.alter_column('attachments', 'original_key', nullable=True)
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
op.add_column(
'attachments',
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
)
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
op.create_table(
'attachment_variants',
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=16), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
)
op.create_table(
'attachment_jobs',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('state', sa.String(length=16), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('max_attempts', sa.Integer(), nullable=False),
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('lease_owner', sa.String(length=128), nullable=True),
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint(
"state IN ('queued', 'processing', 'ready', 'failed')",
name='attachment_job_state_check',
),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
)
op.create_index(
'attachment_jobs_lease_idx',
'attachment_jobs',
['state', 'lease_until'],
)
op.create_table(
'outbox_events',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('event_type', sa.String(length=64), nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('payload', sa.JSON(), nullable=False),
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('claim_owner', sa.String(length=128), nullable=True),
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index(
'outbox_events_pending_idx',
'outbox_events',
['published_at', 'available_at'],
)
op.create_table(
'job_events',
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('kind', sa.String(length=32), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
def downgrade() -> None:
op.drop_index('job_events_job_created_idx', table_name='job_events')
op.drop_table('job_events')
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
op.drop_table('outbox_events')
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
op.drop_table('attachment_jobs')
op.drop_table('attachment_variants')
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
op.drop_column('attachments', 'pipeline_version')
op.drop_column('attachments', 'staging_key')
op.alter_column('attachments', 'original_key', nullable=False)
op.alter_column('attachments', 'original_key', new_column_name='object_key')
@@ -0,0 +1,24 @@
"""Persist W3C trace context with outbox events.
Revision ID: 20261117_0005
Revises: 20261116_0004
Create Date: 2026-11-17
"""
from alembic import op
import sqlalchemy as sa
revision = '20261117_0005'
down_revision = '20261116_0004'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column('outbox_events', sa.Column('traceparent', sa.String(length=55), nullable=True))
op.add_column('outbox_events', sa.Column('tracestate', sa.String(length=512), nullable=True))
def downgrade() -> None:
op.drop_column('outbox_events', 'tracestate')
op.drop_column('outbox_events', 'traceparent')
+1
View File
@@ -0,0 +1 @@
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
+1
View File
@@ -0,0 +1 @@
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
+28
View File
@@ -0,0 +1,28 @@
import base64
import json
from uuid import UUID
from ..domain.errors import Invalid
from ..domain.models import Keyset
from .timefmt import from_rfc3339, to_rfc3339
MAX_LENGTH = 512
def encode(scope: str, position: Keyset) -> str:
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
def decode(scope: str, value: str) -> Keyset:
if not value or len(value) > MAX_LENGTH:
raise Invalid('cursor недействителен для этого запроса')
try:
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
payload = json.loads(raw)
if not isinstance(payload, dict) or payload.get('s') != scope:
raise ValueError('scope mismatch')
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
except Exception:
raise Invalid('cursor недействителен для этого запроса') from None
+79
View File
@@ -0,0 +1,79 @@
from __future__ import annotations
from dataclasses import dataclass
from fastapi import Depends, HTTPException, Request, Security, status
from ..db.redis import RedisSessionStore
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..services.chats import ChatService
from ..services.attachments import AttachmentService
from ..services.messages import MessageService
from ..services.users import UserService
from .security import bearer_auth
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
@dataclass(frozen=True, slots=True)
class Services:
attachments: AttachmentService
users: UserService
chats: ChatService
messages: MessageService
sessions: RedisSessionStore
def services(request: Request) -> Services:
return request.app.state.services
def user_service(container: Services = Depends(services)) -> UserService:
return container.users
def chat_service(container: Services = Depends(services)) -> ChatService:
return container.chats
def message_service(container: Services = Depends(services)) -> MessageService:
return container.messages
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
return container.attachments
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
return container.sessions
def _bearer_error() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='требуется действующая Bearer-сессия',
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
)
async def bearer_token(authorization: str | None = Security(bearer_auth)) -> str:
if not authorization:
raise _bearer_error()
scheme, separator, token = authorization.partition(' ')
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
raise _bearer_error()
return token
async def current_user(
token: str = Depends(bearer_token),
container: Services = Depends(services),
) -> User:
user_id = await container.sessions.get_user_id(token)
if user_id is None:
raise _bearer_error()
try:
return await container.users.identify(user_id)
except Unauthorized:
raise _bearer_error() from None
+107
View File
@@ -0,0 +1,107 @@
import logging
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from sqlalchemy.exc import SQLAlchemyError
from starlette.exceptions import HTTPException as StarletteHTTPException
from ..domain.errors import (
Conflict,
DomainError,
Invalid,
NotFound,
PayloadTooLarge,
Unauthorized,
Unavailable,
UnsupportedMediaType,
)
logger = logging.getLogger('app.error')
CODE_BY_STATUS = {
400: 'invalid_request',
401: 'unauthorized',
404: 'not_found',
409: 'conflict',
413: 'payload_too_large',
415: 'unsupported_media_type',
429: 'rate_limited',
500: 'internal_error',
503: 'unavailable',
}
STATUS_BY_DOMAIN = {
Invalid: 400,
Unauthorized: 401,
NotFound: 404,
Conflict: 409,
PayloadTooLarge: 413,
UnsupportedMediaType: 415,
Unavailable: 503,
}
def code_for(status: int) -> str:
if status in CODE_BY_STATUS:
return CODE_BY_STATUS[status]
return 'invalid_request' if status < 500 else 'internal_error'
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
return JSONResponse(
status_code=status,
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
)
def _domain_status(error: DomainError) -> int:
for kind, status in STATUS_BY_DOMAIN.items():
if isinstance(error, kind):
return status
return 500
def _describe(error: RequestValidationError) -> str:
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
for item in error.errors():
if item.get('type') == 'json_invalid':
return 'тело запроса не является корректным JSON'
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
return f'некорректное поле {location}' if location else 'некорректный запрос'
return 'некорректный запрос'
def request_id(request) -> str:
return getattr(request.state, 'request_id', 'unknown')
def install(app: FastAPI) -> None:
@app.exception_handler(DomainError)
async def _on_domain(request, error: DomainError) -> JSONResponse:
return error_response(_domain_status(error), error.message, request_id(request))
@app.exception_handler(RequestValidationError)
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
return error_response(400, _describe(error), request_id(request))
@app.exception_handler(SQLAlchemyError)
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'база данных недоступна', request_id(request))
@app.exception_handler(OSError)
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
@app.exception_handler(StarletteHTTPException)
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
response = error_response(error.status_code, message, request_id(request))
for name, value in (error.headers or {}).items():
response.headers[name] = value
return response
__all__ = ['code_for', 'error_response', 'install']
+83
View File
@@ -0,0 +1,83 @@
import logging
import os
import socket
import time
from uuid import uuid4
from opentelemetry import trace
from starlette.datastructures import MutableHeaders
from ..observability import get_metrics
from ..observability.logging import safe_extra
from .errors import error_response
logger = logging.getLogger('app')
class RequestContext:
def __init__(self, app) -> None:
self.app = app
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
async def __call__(self, scope, receive, send) -> None:
if scope['type'] != 'http':
await self.app(scope, receive, send)
return
request_id = uuid4().hex
scope.setdefault('state', {})['request_id'] = request_id
is_health = scope.get('path') in {'/health/live', '/health/ready'}
started_at = time.perf_counter()
started = False
status_code = 500
async def send_wrapper(message) -> None:
nonlocal started, status_code
if message['type'] == 'http.response.start':
started = True
status_code = message['status']
headers = MutableHeaders(scope=message)
headers['X-Request-Id'] = request_id
headers['X-Instance-Id'] = self.instance_id
await send(message)
try:
await self.app(scope, receive, send_wrapper)
except Exception:
logger.exception(
'unhandled HTTP error',
extra=safe_extra('http.request.failed', request_id=request_id),
)
if started:
raise
response = error_response(500, 'внутренняя ошибка', request_id)
await response(scope, receive, send_wrapper)
finally:
duration = time.perf_counter() - started_at
route = getattr(scope.get('route'), 'path', None) or 'unmatched'
method = scope.get('method', 'UNKNOWN')
status_class = f'{status_code // 100}xx'
span = trace.get_current_span()
if span.is_recording():
span.set_attribute('request.id', request_id)
span.set_attribute('http.route', route)
if not is_health:
attributes = {
'http.request.method': method,
'http.route': route,
'http.response.status_class': status_class,
}
instruments = get_metrics()
instruments.http_requests.add(1, attributes)
instruments.http_duration.record(duration, attributes)
logger.info(
'HTTP request completed',
extra=safe_extra(
'http.request.completed',
request_id=request_id,
http_method=method,
http_route=route,
http_status_code=status_code,
duration_seconds=round(duration, 6),
),
)
+3
View File
@@ -0,0 +1,3 @@
from . import auth, chats, health, messages, users
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
+79
View File
@@ -0,0 +1,79 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query, Request
from fastapi.responses import StreamingResponse
from ...domain.models import User
from ...services.attachments import AttachmentService
from ..deps import attachment_service, current_user
from ..schemas import AttachmentOut
router = APIRouter(tags=['attachments'])
@router.post(
'/api/v1/chats/{chat_id}/attachments',
status_code=202,
response_model=AttachmentOut,
summary='Принять PNG/JPEG на асинхронную обработку',
openapi_extra={
'requestBody': {
'required': True,
'description': 'Сырые байты изображения, не multipart/form-data.',
'content': {
'image/png': {
'schema': {'type': 'string', 'format': 'binary'},
},
'image/jpeg': {
'schema': {'type': 'string', 'format': 'binary'},
},
},
},
},
)
async def upload(
chat_id: UUID,
request: Request,
filename: str = Query(..., min_length=1, max_length=255),
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
attachment = await attachments.upload(
actor,
chat_id,
filename,
content_type or '',
request.stream(),
)
return AttachmentOut.from_domain(attachment)
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
async def metadata(
attachment_id: UUID,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
@router.get('/api/v1/attachments/{attachment_id}/content')
async def content(
attachment_id: UUID,
variant: str = Query('original'),
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> StreamingResponse:
attachment, stream, content_type, size_bytes = await attachments.content(
actor, attachment_id, variant
)
return StreamingResponse(
stream.chunks(),
media_type=content_type,
headers={
'Content-Length': str(size_bytes),
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
},
)
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
import base64
from binascii import Error as Base64Error
from fastapi import APIRouter, Depends, HTTPException, Security, status
from ...auth.tokens import create_token
from ...db.redis import RedisSessionStore
from ...domain.models import User
from ...services.users import UserService
from ..deps import bearer_token, current_user, session_store, user_service
from ..security import basic_auth
from ..schemas import SessionOut, UserOut
from ..timefmt import to_rfc3339
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
def _unauthorized_basic() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='неверные учётные данные',
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
)
def _parse_basic(authorization: str | None) -> tuple[str, str]:
if not authorization:
raise _unauthorized_basic()
scheme, separator, encoded = authorization.partition(' ')
if scheme.lower() != 'basic' or not separator or not encoded:
raise _unauthorized_basic()
try:
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
except (Base64Error, UnicodeDecodeError):
raise _unauthorized_basic() from None
username, separator, password = decoded.partition(':')
if not separator or not username:
raise _unauthorized_basic()
return username, password
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
async def create_session(
authorization: str | None = Security(basic_auth),
users: UserService = Depends(user_service),
sessions: RedisSessionStore = Depends(session_store),
) -> SessionOut:
username, password = _parse_basic(authorization)
user = await users.authenticate(username, password)
if user is None:
raise _unauthorized_basic()
token = create_token()
expires_at = await sessions.create(user.id, token)
return SessionOut(
token=token,
token_type='Bearer',
expires_at=to_rfc3339(expires_at),
user=UserOut.from_domain(user),
)
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
async def revoke_session(
token: str = Depends(bearer_token),
_: User = Depends(current_user),
sessions: RedisSessionStore = Depends(session_store),
) -> None:
await sessions.revoke(token)
+49
View File
@@ -0,0 +1,49 @@
from typing import Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from ...domain.models import User
from ...services.chats import ChatService
from .. import cursors
from ..deps import chat_service, current_user
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
ChatOut,
ChatPageOut,
CreateChat,
)
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
@router.post('', status_code=201, response_model=ChatOut)
async def create(
body: CreateChat,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
@router.get('', response_model=ChatPageOut)
async def page(
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> ChatPageOut:
scope = f'chats:{actor.id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
@router.get('/{chat_id}', response_model=ChatOut)
async def get(
chat_id: UUID,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.get(actor, chat_id))
+29
View File
@@ -0,0 +1,29 @@
from fastapi import APIRouter, HTTPException, Request
from redis.exceptions import RedisError
from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError
from ...domain.errors import Unavailable
from ..schemas import HealthOut
router = APIRouter(tags=['health'])
@router.get('/health/live', response_model=HealthOut)
async def live() -> HealthOut:
return HealthOut(status='ok')
@router.get('/health/ready', response_model=HealthOut)
async def ready(request: Request) -> HealthOut:
if not request.app.state.ready:
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
try:
async with request.app.state.engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await request.app.state.redis.ping()
await request.app.state.storage.ping()
except (SQLAlchemyError, RedisError, Unavailable, OSError):
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
return HealthOut(status='ok')
+53
View File
@@ -0,0 +1,53 @@
from typing import Annotated, Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query
from ...domain.models import User
from ...services.messages import MessageService
from .. import cursors
from ..deps import current_user, message_service
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
CreateMessage,
MessageOut,
MessagePageOut,
)
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
@router.post('', status_code=201, response_model=MessageOut)
async def send(
chat_id: UUID,
body: CreateMessage,
idempotency_key: Annotated[
str,
Header(
alias='Idempotency-Key',
min_length=1,
max_length=128,
pattern=r'^[A-Za-z0-9._:-]+$',
),
],
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
) -> MessageOut:
return MessageOut.from_domain(
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
)
@router.get('', response_model=MessagePageOut)
async def page(
chat_id: UUID,
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> MessagePageOut:
scope = f'messages:{chat_id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
+18
View File
@@ -0,0 +1,18 @@
from fastapi import APIRouter, Depends
from ...domain.models import User
from ...services.users import UserService
from ..deps import current_user, user_service
from ..schemas import CreateUser, UserOut
router = APIRouter(prefix='/api/v1', tags=['users'])
@router.post('/users', status_code=201, response_model=UserOut)
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
@router.get('/users/me', response_model=UserOut)
async def me(actor: User = Depends(current_user)) -> UserOut:
return UserOut.from_domain(actor)
+209
View File
@@ -0,0 +1,209 @@
from typing import Optional
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from ..domain.models import Attachment, Chat, Message, Page, User
from . import cursors
from .timefmt import to_rfc3339
NAME_MAX = 100
TEXT_MAX = 4000
MEMBERS_MAX = 100
LIMIT_MIN = 1
LIMIT_MAX = 100
LIMIT_DEFAULT = 50
def not_blank(value: str) -> str:
if not value.strip():
raise ValueError('строка не может состоять только из пробельных символов')
return value
class Strict(BaseModel):
model_config = ConfigDict(extra='forbid')
class HealthOut(Strict):
status: str = 'ok'
class CreateUser(Strict):
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=NAME_MAX)
password: str = Field(min_length=12, max_length=128)
@field_validator('display_name')
@classmethod
def _display_name(cls, value: str) -> str:
return not_blank(value)
class UserOut(Strict):
id: UUID
username: str
display_name: str
created_at: str
@classmethod
def from_domain(cls, user: User) -> 'UserOut':
return cls(
id=user.id,
username=user.username,
display_name=user.display_name,
created_at=to_rfc3339(user.created_at),
)
class SessionOut(Strict):
token: str = Field(min_length=32)
token_type: str = 'Bearer'
expires_at: str
user: UserOut
class CreateChat(Strict):
title: str = Field(min_length=1, max_length=NAME_MAX)
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
@field_validator('title')
@classmethod
def _title(cls, value: str) -> str:
return not_blank(value)
@field_validator('member_ids')
@classmethod
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('member_ids не должен содержать дубликаты')
return value
class ChatOut(Strict):
id: UUID
title: str
member_ids: list[UUID]
created_at: str
@classmethod
def from_domain(cls, chat: Chat) -> 'ChatOut':
return cls(
id=chat.id,
title=chat.title,
member_ids=list(chat.member_ids),
created_at=to_rfc3339(chat.created_at),
)
class CreateMessage(Strict):
text: str = Field(max_length=TEXT_MAX)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
@classmethod
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('attachment_ids не должен содержать дубликаты')
return value
@model_validator(mode='after')
def _content_present(self) -> 'CreateMessage':
if not self.text.strip() and not self.attachment_ids:
raise ValueError('сообщение должно содержать текст или вложение')
return self
class MessageOut(Strict):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: str
attachment_ids: list[UUID]
@classmethod
def from_domain(cls, message: Message) -> 'MessageOut':
return cls(
id=message.id,
chat_id=message.chat_id,
sender_id=message.sender_id,
text=message.text,
created_at=to_rfc3339(message.created_at),
attachment_ids=list(message.attachment_ids),
)
class VariantOut(Strict):
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class AttachmentOut(Strict):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list[VariantOut] = Field(default_factory=list)
error_code: str | None
created_at: str
updated_at: str
@classmethod
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
return cls(
id=attachment.id,
chat_id=attachment.chat_id,
owner_id=attachment.owner_id,
filename=attachment.filename,
content_type=attachment.content_type,
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
status=attachment.status,
variants=[
VariantOut(
name=item.name,
content_type=item.content_type,
size_bytes=item.size_bytes,
width=item.width,
height=item.height,
sha256=item.sha256,
)
for item in attachment.variants
],
error_code=attachment.error_code,
created_at=to_rfc3339(attachment.created_at),
updated_at=to_rfc3339(attachment.updated_at),
)
class ChatPageOut(Strict):
items: list[ChatOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
return cls(
items=[ChatOut.from_domain(chat) for chat in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
class MessagePageOut(Strict):
items: list[MessageOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
return cls(
items=[MessageOut.from_domain(message) for message in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
+29
View File
@@ -0,0 +1,29 @@
"""OpenAPI security-схемы с сохранением собственного разбора Authorization."""
from fastapi import Request
from fastapi.openapi.models import HTTPBase as HTTPBaseModel
from fastapi.openapi.models import HTTPBearer as HTTPBearerModel
from fastapi.security.base import SecurityBase
class RawHTTPAuthorization(SecurityBase):
"""Объявляет HTTP security scheme и возвращает исходный Authorization header."""
def __init__(self, model, scheme_name: str) -> None:
self.model = model
self.scheme_name = scheme_name
async def __call__(self, request: Request) -> str | None:
"""Возвращает заголовок без разбора, чтобы сохранить текущую валидацию API."""
return request.headers.get('Authorization')
basic_auth = RawHTTPAuthorization(
HTTPBaseModel(scheme='basic'),
scheme_name='basicAuth',
)
bearer_auth = RawHTTPAuthorization(
HTTPBearerModel(scheme='bearer', bearerFormat='opaque session token'),
scheme_name='bearerAuth',
)
+11
View File
@@ -0,0 +1,11 @@
from datetime import datetime, timezone
def to_rfc3339(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def from_rfc3339(value: str) -> datetime:
if not isinstance(value, str) or not value.endswith('Z'):
raise ValueError('timestamp must end with Z')
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
+4
View File
@@ -0,0 +1,4 @@
from .passwords import hash_password, password_matches
from .tokens import create_token
__all__ = ['create_token', 'hash_password', 'password_matches']
+15
View File
@@ -0,0 +1,15 @@
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, VerifyMismatchError
_hasher = PasswordHasher()
def hash_password(password: str) -> str:
return _hasher.hash(password)
def password_matches(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError):
return False
+11
View File
@@ -0,0 +1,11 @@
import hashlib
import secrets
def create_token() -> str:
return secrets.token_urlsafe(32)
def session_key(token: str) -> str:
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
return f'session:{digest}'
+3
View File
@@ -0,0 +1,3 @@
from .settings import Settings, load_settings
__all__ = ['Settings', 'load_settings']
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
def _read_value(name: str, file_name: str) -> str:
value = os.getenv(name)
if value:
return value
path = os.getenv(file_name)
if path:
return Path(path).read_text(encoding='utf-8').strip()
raise RuntimeError(f'не задана настройка {name} или {file_name}')
@dataclass(frozen=True, slots=True)
class Settings:
database_url: str
redis_url: str
session_ttl_seconds: int
s3_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
max_upload_bytes: int
max_image_pixels: int
broker_url: str
worker_concurrency: int
worker_prefetch: int
job_max_attempts: int
job_timeout_seconds: int
job_lease_seconds: int
retry_base_seconds: int
retry_max_seconds: int
staging_retention_seconds: int
thumbnail_jpeg_quality: int
crop_jpeg_quality: int
optimized_webp_quality: int
def _positive_int(name: str, default: str) -> int:
try:
value = int(os.getenv(name, default))
except ValueError as exc:
raise RuntimeError(f'{name} должен быть целым числом') from exc
if value <= 0:
raise RuntimeError(f'{name} должен быть больше нуля')
return value
def _quality(name: str, default: str) -> int:
value = _positive_int(name, default)
if value > 100:
raise RuntimeError(f'{name} должен быть не больше 100')
return value
def load_settings() -> Settings:
return Settings(
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
s3_region=os.getenv('S3_REGION', 'us-east-1'),
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
)
+4
View File
@@ -0,0 +1,4 @@
from .postgres import Base, create_engine, create_session_factory
from .redis import RedisSessionStore, create_redis
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
+40
View File
@@ -0,0 +1,40 @@
from __future__ import annotations
from sqlalchemy import event
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
from ..observability import get_metrics
class Base(DeclarativeBase):
pass
def create_engine(url: str) -> AsyncEngine:
engine = create_async_engine(
url,
pool_size=5,
max_overflow=5,
pool_timeout=5,
pool_pre_ping=True,
connect_args={'server_settings': {'statement_timeout': '5000'}},
)
instruments = get_metrics()
attributes = {'pool.name': 'postgres'}
def update_pool_metrics(*_args) -> None:
pool = engine.sync_engine.pool
instruments.db_pool_used.set(float(pool.checkedout()), attributes)
waiting = len(getattr(getattr(pool, '_pool', None), '_getters', ()))
instruments.db_pool_waiting.set(float(waiting), attributes)
event.listen(engine.sync_engine, 'checkout', update_pool_metrics)
event.listen(engine.sync_engine, 'checkin', update_pool_metrics)
instruments.db_pool_used.set(0, attributes)
instruments.db_pool_waiting.set(0, attributes)
return engine
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(engine, expire_on_commit=False)
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from uuid import UUID
from redis.asyncio import Redis
from redis.exceptions import RedisError
from ..auth.tokens import session_key
from ..domain.errors import Unavailable
def create_redis(url: str) -> Redis:
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
class RedisSessionStore:
def __init__(self, client: Redis, ttl_seconds: int) -> None:
self._client = client
self._ttl_seconds = ttl_seconds
async def create(self, user_id: UUID, token: str) -> datetime:
try:
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
async def get_user_id(self, token: str) -> UUID | None:
try:
value = await self._client.get(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
if value is None:
return None
try:
return UUID(value)
except ValueError:
return None
async def revoke(self, token: str) -> None:
try:
await self._client.delete(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
async def ping(self) -> None:
try:
await self._client.ping()
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
View File
+37
View File
@@ -0,0 +1,37 @@
from __future__ import annotations
class DomainError(Exception):
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
def __init__(self, message: str) -> None:
super().__init__(message)
self.message = message
class Invalid(DomainError):
"""Запрос нарушает правила предметной области."""
class Unauthorized(DomainError):
"""Нет действительной идентификации."""
class NotFound(DomainError):
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
class Conflict(DomainError):
"""Состояние не позволяет выполнить операцию."""
class Unavailable(DomainError):
"""Обязательная внешняя зависимость временно недоступна."""
class PayloadTooLarge(DomainError):
"""Тело запроса или декодированное содержимое превышает лимит."""
class UnsupportedMediaType(DomainError):
"""Формат содержимого не поддерживается или не соответствует MIME."""
+116
View File
@@ -0,0 +1,116 @@
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from uuid import UUID
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
def has_member(self, user_id: UUID) -> bool:
return user_id in self.member_ids
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
original_key: str | None
staging_key: str | None
pipeline_version: int
content_type: str
size_bytes: int
sha256: str
width: int
height: int
status: str
error_code: str | None
created_at: datetime
updated_at: datetime
variants: tuple['AttachmentVariant', ...] = ()
@dataclass(frozen=True, slots=True)
class AttachmentVariant:
name: str
object_key: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
@dataclass(frozen=True, slots=True)
class AttachmentJob:
id: UUID
attachment_id: UUID
pipeline_version: int
state: str
attempt: int
max_attempts: int
next_attempt_at: datetime
lease_owner: str | None
lease_until: datetime | None
last_error_code: str | None
replay_count: int
created_at: datetime
updated_at: datetime
@dataclass(frozen=True, slots=True)
class OutboxEvent:
id: UUID
job_id: UUID
payload: dict
available_at: datetime
traceparent: str | None = None
tracestate: str | None = None
@dataclass(frozen=True, slots=True)
class Keyset:
created_at: datetime
id: UUID
@property
def order(self) -> tuple[datetime, int]:
return (self.created_at, self.id.int)
@classmethod
def from_item(cls, item) -> 'Keyset':
return cls(created_at=item.created_at, id=item.id)
def order_key(item) -> tuple[datetime, int]:
return (item.created_at, item.id.int)
@dataclass(frozen=True, slots=True)
class Page:
items: tuple
next: Optional[Keyset]
+1
View File
@@ -0,0 +1 @@
"""Durable attachment job publisher and worker."""
+57
View File
@@ -0,0 +1,57 @@
from __future__ import annotations
import asyncio
import logging
import aio_pika
from aio_pika import DeliveryMode, ExchangeType
MAIN_EXCHANGE = 'attachments'
MAIN_QUEUE = 'attachments.process'
ROUTING_KEY = 'attachment.process.v1'
DEAD_EXCHANGE = 'attachments.dead'
DEAD_QUEUE = 'attachments.failed'
logger = logging.getLogger('app.broker')
async def connect(url: str) -> aio_pika.RobustConnection:
delay = 0.5
while True:
try:
return await aio_pika.connect_robust(url, timeout=5)
except (OSError, aio_pika.exceptions.AMQPException):
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
await asyncio.sleep(delay)
delay = min(delay * 2, 5.0)
async def topology(channel: aio_pika.abc.AbstractChannel):
dead_exchange = await channel.declare_exchange(
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
)
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
await dead_queue.bind(dead_exchange, ROUTING_KEY)
exchange = await channel.declare_exchange(
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
)
queue = await channel.declare_queue(
MAIN_QUEUE,
durable=True,
arguments={
'x-dead-letter-exchange': DEAD_EXCHANGE,
'x-dead-letter-routing-key': ROUTING_KEY,
},
)
await queue.bind(exchange, ROUTING_KEY)
return exchange, queue
def message(body: bytes, event_id: str, headers: dict[str, str] | None = None) -> aio_pika.Message:
return aio_pika.Message(
body=body,
content_type='application/json',
delivery_mode=DeliveryMode.PERSISTENT,
message_id=event_id,
type=ROUTING_KEY,
headers=headers or {},
)
+98
View File
@@ -0,0 +1,98 @@
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
class PermanentImageError(Exception):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
@dataclass(frozen=True, slots=True)
class GeneratedVariant:
name: str
content_type: str
width: int
height: int
sha256: str
content: bytes
def _jpeg_image(image: Image.Image) -> Image.Image:
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
rgba = image.convert('RGBA')
background = Image.new('RGBA', rgba.size, 'white')
background.alpha_composite(rgba)
return background.convert('RGB')
return image.convert('RGB')
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
output = BytesIO()
image.save(output, format=fmt, **options)
return output.getvalue()
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
return GeneratedVariant(
name=name,
content_type=content_type,
width=image.width,
height=image.height,
sha256=hashlib.sha256(content).hexdigest(),
content=content,
)
def process_image(
source: bytes,
declared_type: str,
expected_sha256: str,
max_pixels: int,
thumbnail_quality: int,
crop_quality: int,
webp_quality: int,
) -> tuple[GeneratedVariant, ...]:
if hashlib.sha256(source).hexdigest() != expected_sha256:
raise PermanentImageError('checksum_mismatch')
try:
with Image.open(BytesIO(source)) as opened:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
if actual != declared_type:
raise PermanentImageError('unsupported_media_type')
if opened.width * opened.height > max_pixels:
raise PermanentImageError('image_too_large')
opened.load()
oriented = ImageOps.exif_transpose(opened)
base = oriented.copy()
except PermanentImageError:
raise
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
raise PermanentImageError('invalid_image') from error
thumbnail = _jpeg_image(base.copy())
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
thumbnail_bytes = _encode(
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
)
crop_source = _jpeg_image(base.copy())
crop = ImageOps.fit(
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
optimized = base.copy()
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
return (
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
_variant('crop', 'image/jpeg', crop, crop_bytes),
_variant('optimized', 'image/webp', optimized, optimized_bytes),
)
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import socket
from datetime import datetime, timezone
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from ..config import load_settings
from ..observability import get_metrics
from ..observability.logging import safe_extra
from ..observability.propagation import extract_trace_context, inject_trace_context
from ..repositories import JobRepository, OutboxRepository
from .broker import ROUTING_KEY, connect, message, topology
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.publisher')
tracer = trace.get_tracer('messenger.publisher', '1.6.0')
async def run(once: bool = False) -> None:
settings = load_settings()
engine, sessions, _storage = dependencies(settings)
outbox = OutboxRepository(sessions)
jobs = JobRepository(sessions)
instruments = get_metrics()
queue_attributes = {'messaging.destination.name': 'attachments.process'}
owner = f'{socket.gethostname()}:{os.getpid()}'
connection = await connect(settings.broker_url)
try:
channel = await connection.channel(publisher_confirms=True)
exchange, _queue = await topology(channel)
while True:
now = datetime.now(timezone.utc)
await jobs.recover_expired(now)
depth, oldest_age = await jobs.queue_stats(now)
instruments.queue_depth.set(depth, queue_attributes)
instruments.queue_oldest_age.set(oldest_age, queue_attributes)
events = await outbox.claim(owner, now, lease_seconds=30)
for event in events:
try:
parent = extract_trace_context({
'traceparent': event.traceparent,
'tracestate': event.tracestate,
})
with tracer.start_as_current_span(
'attachments publish',
context=parent,
kind=SpanKind.PRODUCER,
attributes={
'messaging.system': 'rabbitmq',
'messaging.destination.name': 'attachments.process',
'messaging.operation.name': 'publish',
'messaging.operation.type': 'send',
'job.id': str(event.job_id),
},
):
body = json.dumps(event.payload, separators=(',', ':')).encode()
headers = inject_trace_context()
await exchange.publish(
message(body, str(event.id), headers),
routing_key=ROUTING_KEY,
mandatory=True,
)
logger.info(
'outbox event published',
extra=safe_extra(
'outbox.published', event_id=str(event.id),
job_id=str(event.job_id),
),
)
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
os._exit(91)
await outbox.published(event.id, owner, datetime.now(timezone.utc))
except Exception:
logger.exception(
'outbox publish failed',
extra=safe_extra(
'outbox.publish_failed', event_id=str(event.id),
job_id=str(event.job_id),
),
)
await outbox.failed(
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
)
if once:
return
await asyncio.sleep(0.5 if events else 1.0)
finally:
await connection.close()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
telemetry = configure_logging('messenger-publisher')
try:
asyncio.run(run(args.once))
finally:
telemetry.shutdown()
if __name__ == '__main__':
main()
+25
View File
@@ -0,0 +1,25 @@
from __future__ import annotations
from ..config import Settings
from ..db import create_engine, create_session_factory
from ..observability import initialize_observability
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from ..storage import S3Storage
def configure_logging(service_name: str):
return initialize_observability(service_name)
def dependencies(settings: Settings):
engine = create_engine(settings.database_url)
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
sessions = create_session_factory(engine)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
return engine, sessions, storage
+390
View File
@@ -0,0 +1,390 @@
from __future__ import annotations
import asyncio
import json
import logging
import multiprocessing
import os
import random
import socket
import time
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from uuid import UUID
import aio_pika
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from ..config import Settings, load_settings
from ..domain.errors import Unavailable
from ..domain.models import AttachmentVariant
from ..observability import get_metrics
from ..observability.logging import safe_extra
from ..observability.propagation import extract_trace_context
from ..repositories import JobRepository
from .broker import connect, topology
from .pipeline import GeneratedVariant, PermanentImageError, process_image
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.worker')
tracer = trace.get_tracer('messenger.worker', '1.6.0')
class ProcessingTimeout(Exception):
pass
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
try:
if memory_bytes:
try:
import resource
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
except (ImportError, OSError, ValueError):
pass
connection.send(('ok', process_image(*arguments)))
except PermanentImageError as error:
connection.send(('permanent', error.code))
except MemoryError:
connection.send(('temporary', 'processing_memory_limit'))
except BaseException:
connection.send(('temporary', 'processing_failed'))
finally:
connection.close()
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
context = multiprocessing.get_context('spawn')
receiver, sender = context.Pipe(duplex=False)
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
process.start()
sender.close()
if not receiver.poll(timeout):
process.terminate()
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_timeout')
try:
kind, result = receiver.recv()
except EOFError as error:
process.join(5)
receiver.close()
raise ProcessingTimeout('processing_failed') from error
receiver.close()
process.join(5)
if process.is_alive():
process.terminate()
process.join(5)
raise ProcessingTimeout('processing_failed')
if kind == 'ok':
return result
if kind == 'permanent':
raise PermanentImageError(result)
raise ProcessingTimeout(result)
class Worker:
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
self.settings = settings
self.jobs = jobs
self.storage = storage
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
self.metrics = get_metrics()
self.metric_attributes = {'job.type': 'attachment.process.v1'}
self.metrics.worker_busy.set(0, self.metric_attributes)
self.metrics.worker_capacity.set(settings.worker_concurrency, self.metric_attributes)
@asynccontextmanager
async def _busy_slot(self):
async with self.semaphore:
self.metrics.worker_busy.add(1, self.metric_attributes)
try:
yield
finally:
self.metrics.worker_busy.add(-1, self.metric_attributes)
def _delay(self, attempt: int) -> float:
base = min(
self.settings.retry_max_seconds,
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
)
return base + random.uniform(0, base)
async def _handle_failure(
self,
incoming: aio_pika.IncomingMessage,
job_id: UUID,
attempt: int,
code: str,
permanent: bool,
) -> None:
now = datetime.now(timezone.utc)
available_at = now + timedelta(seconds=self._delay(attempt))
try:
state = await self.jobs.retry_or_fail(
job_id,
self.worker_id,
code,
permanent,
available_at,
now,
)
except Exception:
logger.exception('failed to persist job failure job_id=%s', job_id)
await incoming.nack(requeue=True)
return
if state == 'failed':
logger.error(
'job reached terminal failure',
extra=safe_extra(
'job.failed', job_id=str(job_id), attempt=attempt, error_code=code
),
)
await incoming.reject(requeue=False)
else:
logger.warning(
'job retry scheduled',
extra=safe_extra(
'job.retry_scheduled', job_id=str(job_id), attempt=attempt,
error_code=code,
),
)
await incoming.ack()
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
context = extract_trace_context(incoming.headers)
job_id: UUID | None = None
before = None
try:
preview = json.loads(incoming.body)
job_id = UUID(preview['job_id'])
before = await self.jobs.state(job_id)
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
pass
started_at = time.perf_counter()
attributes = {
'messaging.system': 'rabbitmq',
'messaging.destination.name': 'attachments.process',
'messaging.operation.name': 'process',
'messaging.operation.type': 'process',
}
if job_id is not None:
attributes['job.id'] = str(job_id)
with tracer.start_as_current_span(
'attachments process', context=context, kind=SpanKind.CONSUMER,
attributes=attributes,
) as span:
await self._handle_message(incoming)
if before is None or before.state != 'queued' or job_id is None:
return
after = await self.jobs.state(job_id)
if after is None or after.state not in {'queued', 'ready', 'failed'}:
return
if after.state == 'queued' and after.attempt == before.attempt:
return
outcome = 'retry' if after.state == 'queued' else after.state
duration = time.perf_counter() - started_at
metric_attributes = {**self.metric_attributes, 'outcome': outcome}
self.metrics.jobs_processed.add(1, metric_attributes)
self.metrics.job_duration.record(duration, metric_attributes)
span.set_attribute('job.outcome', outcome)
span.set_attribute('job.attempt', before.attempt)
if outcome == 'ready':
ready_seconds = max(
0.0, (datetime.now(timezone.utc) - after.created_at).total_seconds()
)
self.metrics.time_to_ready.record(
ready_seconds,
{'pipeline.version': str(after.pipeline_version), 'outcome': 'ready'},
)
async def _handle_message(self, incoming: aio_pika.IncomingMessage) -> None:
async with self._busy_slot():
try:
payload = json.loads(incoming.body)
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
await incoming.reject(requeue=False)
return
job_id = UUID(payload['job_id'])
attachment_id = UUID(payload['attachment_id'])
attempt = int(payload['attempt'])
pipeline_version = int(payload['pipeline_version'])
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
await incoming.reject(requeue=False)
return
current = await self.jobs.state(job_id)
if current is None:
await incoming.reject(requeue=False)
return
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
await incoming.reject(requeue=False)
return
if current.state == 'ready':
if await self.jobs.ready_effect_complete(job_id):
await incoming.ack()
else:
await incoming.reject(requeue=False)
return
if current.state == 'failed':
await incoming.reject(requeue=False)
return
now = datetime.now(timezone.utc)
claimed = await self.jobs.claim(
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
)
if claimed is None:
await incoming.ack()
return
job, attachment = claimed
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
await self._handle_failure(
incoming, job.id, job.attempt, 'fault_injected', True
)
os._exit(94)
if not attachment.staging_key:
await self._handle_failure(
incoming, job.id, job.attempt, 'staging_missing', True
)
return
try:
source = await self.storage.read(
attachment.staging_key, self.settings.max_upload_bytes
)
arguments = (
source,
attachment.content_type,
attachment.sha256,
self.settings.max_image_pixels,
self.settings.thumbnail_jpeg_quality,
self.settings.crop_jpeg_quality,
self.settings.optimized_webp_quality,
)
generated = await asyncio.to_thread(
_run_isolated,
arguments,
self.settings.job_timeout_seconds,
512 * 1024 * 1024,
)
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
await self.storage.put_bytes(original_key, source, attachment.content_type)
variants: list[AttachmentVariant] = []
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
for item in generated:
key = (
f'variants/{attachment.id}/v{job.pipeline_version}/'
f'{item.name}.{suffixes[item.name]}'
)
await self.storage.put_bytes(key, item.content, item.content_type)
variants.append(
AttachmentVariant(
name=item.name,
object_key=key,
content_type=item.content_type,
size_bytes=len(item.content),
width=item.width,
height=item.height,
sha256=item.sha256,
)
)
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
os._exit(92)
completed = await self.jobs.complete(
job.id,
self.worker_id,
original_key,
tuple(variants),
datetime.now(timezone.utc),
)
if not completed:
await incoming.nack(requeue=True)
return
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
os._exit(93)
try:
await self.storage.remove(attachment.staging_key)
except Unavailable:
logger.warning(
'staging cleanup deferred',
extra=safe_extra(
'staging.cleanup_deferred', attachment_id=str(attachment.id)
),
)
logger.info(
'job completed',
extra=safe_extra(
'job.completed', job_id=str(job.id),
attachment_id=str(attachment.id), attempt=job.attempt,
),
)
await incoming.ack()
except PermanentImageError as error:
await self._handle_failure(
incoming, job.id, job.attempt, error.code, True
)
except ProcessingTimeout as error:
await self._handle_failure(
incoming, job.id, job.attempt, str(error), False
)
except (Unavailable, OSError, ValueError):
logger.exception(
'temporary processing failure',
extra=safe_extra(
'job.processing_failed', job_id=str(job.id),
error_code='storage_unavailable',
),
)
await self._handle_failure(
incoming, job.id, job.attempt, 'storage_unavailable', False
)
except Exception:
logger.exception(
'unexpected processing failure',
extra=safe_extra(
'job.processing_failed', job_id=str(job.id),
error_code='processing_failed',
),
)
await self._handle_failure(
incoming, job.id, job.attempt, 'processing_failed', False
)
async def run() -> None:
settings = load_settings()
engine, sessions, storage = dependencies(settings)
jobs = JobRepository(sessions)
worker = Worker(settings, jobs, storage)
connection = await connect(settings.broker_url)
try:
channel = await connection.channel()
await channel.set_qos(prefetch_count=settings.worker_prefetch)
_exchange, queue = await topology(channel)
await queue.consume(worker.handle, no_ack=False)
logger.info(
'worker started id=%s concurrency=%d prefetch=%d',
worker.worker_id,
settings.worker_concurrency,
settings.worker_prefetch,
)
await asyncio.Future()
finally:
await connection.close()
await engine.dispose()
def main() -> None:
telemetry = configure_logging('messenger-worker')
try:
asyncio.run(run())
finally:
telemetry.shutdown()
if __name__ == '__main__':
main()
+120
View File
@@ -0,0 +1,120 @@
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
from __future__ import annotations
from contextlib import asynccontextmanager
from fastapi import FastAPI
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from sqlalchemy import text
from .api import errors
from .api.deps import Services
from .api.middleware import RequestContext
from .api.routers import attachments, auth, chats, health, messages, users
from .config import load_settings
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
from .observability import get_metrics, initialize_observability
from .observability.logging import safe_extra
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
from .services.attachments import AttachmentService
from .services.chats import ChatService
from .services.messages import MessageService
from .services.users import UserService
from .storage import S3Storage
import logging
logger = logging.getLogger('app')
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
def create_app() -> FastAPI:
telemetry = initialize_observability('messenger-api')
get_metrics()
@asynccontextmanager
async def lifespan(application: FastAPI):
application.state.ready = False
settings = load_settings()
engine = create_engine(settings.database_url)
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
redis = create_redis(settings.redis_url)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
try:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
await session_store.ping()
await storage.ping()
session_factory = create_session_factory(engine)
user_repository = UserRepository(session_factory)
chat_repository = ChatRepository(session_factory)
message_repository = MessageRepository(session_factory)
attachment_repository = AttachmentRepository(session_factory)
application.state.services = Services(
attachments=AttachmentService(
attachment_repository,
storage,
settings.max_upload_bytes,
settings.max_image_pixels,
settings.job_max_attempts,
),
users=UserService(user_repository),
chats=ChatService(chat_repository),
messages=MessageService(message_repository),
sessions=session_store,
)
application.state.engine = engine
application.state.redis = redis
application.state.storage = storage
application.state.ready = True
logger.info(
'application started',
extra=safe_extra(
'api.startup', storage='postgres+s3', redis_sessions=True,
async_attachments=True,
),
)
yield
finally:
application.state.ready = False
await redis.aclose()
await engine.dispose()
logger.info('application stopped', extra=safe_extra('api.shutdown'))
telemetry.shutdown()
api = FastAPI(
title='MTUSI Messenger. LR 6',
version='1.6.0',
description=DESCRIPTION,
lifespan=lifespan,
)
api.router.redirect_slashes = False
api.state.ready = False
api.add_middleware(RequestContext)
errors.install(api)
for router in (
health.router,
users.router,
auth.router,
chats.router,
messages.router,
attachments.router,
):
api.include_router(router)
FastAPIInstrumentor.instrument_app(
api,
excluded_urls='/health/live,/health/ready',
)
return api
app = create_app()
+18
View File
@@ -0,0 +1,18 @@
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
from .chat import ChatMemberRow, ChatRow
from .message import MessageRow
from .user import UserRow
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
__all__ = [
'AttachmentRow',
'AttachmentVariantRow',
'AttachmentJobRow',
'ChatMemberRow',
'ChatRow',
'MessageAttachmentRow',
'MessageRow',
'UserRow',
'JobEventRow',
'OutboxEventRow',
]
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentRow(Base):
__tablename__ = 'attachments'
__table_args__ = (
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint('original_key', name='attachments_object_key_key'),
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(16), nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class AttachmentVariantRow(Base):
__tablename__ = 'attachment_variants'
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(String(16), primary_key=True)
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MessageAttachmentRow(Base):
__tablename__ = 'message_attachments'
__table_args__ = (
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
Index('message_attachments_attachment_idx', 'attachment_id'),
)
message_id: Mapped[UUID] = mapped_column(
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, nullable=False)
+26
View File
@@ -0,0 +1,26 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class ChatRow(Base):
__tablename__ = 'chats'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
title: Mapped[str] = mapped_column(String(100), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ChatMemberRow(Base):
__tablename__ = 'chat_members'
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentJobRow(Base):
__tablename__ = 'attachment_jobs'
__table_args__ = (
UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
state: Mapped[str] = mapped_column(String(16), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class OutboxEventRow(Base):
__tablename__ = 'outbox_events'
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
traceparent: Mapped[str | None] = mapped_column(String(55), nullable=True)
tracestate: Mapped[str | None] = mapped_column(String(512), nullable=True)
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class JobEventRow(Base):
__tablename__ = 'job_events'
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
kind: Mapped[str] = mapped_column(String(32), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class MessageRow(Base):
__tablename__ = 'messages'
__table_args__ = (
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint(
'sender_id',
'chat_id',
'idempotency_key',
name='messages_sender_chat_idempotency_key',
),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
text: Mapped[str] = mapped_column(String(4000), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class UserRow(Base):
__tablename__ = 'users'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+9
View File
@@ -0,0 +1,9 @@
from .metrics import Metrics, get_metrics
from .runtime import ObservabilityRuntime, initialize_observability
__all__ = [
'Metrics',
'ObservabilityRuntime',
'get_metrics',
'initialize_observability',
]
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
import os
from dataclasses import dataclass
def _positive_int(name: str, default: str) -> int:
try:
value = int(os.getenv(name, default))
except ValueError as exc:
raise RuntimeError(f'{name} должен быть целым числом') from exc
if value <= 0:
raise RuntimeError(f'{name} должен быть больше нуля')
return value
@dataclass(frozen=True, slots=True)
class ObservabilitySettings:
enabled: bool
endpoint: str
service_name: str
service_version: str
environment: str
export_timeout_seconds: int
span_queue_size: int
log_queue_size: int
metric_export_interval_ms: int
def load_observability_settings(default_service_name: str) -> ObservabilitySettings:
disabled = os.getenv('OTEL_SDK_DISABLED', 'false').strip().lower()
if disabled not in {'true', 'false'}:
raise RuntimeError('OTEL_SDK_DISABLED должен быть true или false')
endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318')
return ObservabilitySettings(
enabled=disabled != 'true',
endpoint=endpoint.rstrip('/'),
service_name=os.getenv('OTEL_SERVICE_NAME', default_service_name),
service_version=os.getenv('OTEL_SERVICE_VERSION', '1.6.0'),
environment=os.getenv('OTEL_DEPLOYMENT_ENVIRONMENT', 'lab'),
export_timeout_seconds=_positive_int('OTEL_EXPORTER_OTLP_TIMEOUT', '2'),
span_queue_size=_positive_int('OTEL_BSP_MAX_QUEUE_SIZE', '2048'),
log_queue_size=_positive_int('OTEL_BLRP_MAX_QUEUE_SIZE', '2048'),
metric_export_interval_ms=_positive_int('OTEL_METRIC_EXPORT_INTERVAL', '5000'),
)
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from opentelemetry import trace
_STANDARD_FIELDS = {
'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 'funcName',
'levelname', 'levelno', 'lineno', 'module', 'msecs', 'message', 'msg', 'name',
'pathname', 'process', 'processName', 'relativeCreated', 'stack_info', 'thread',
'threadName', 'taskName',
}
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, object] = {
'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
'severity': record.levelname,
'logger': record.name,
'message': record.getMessage(),
'event': getattr(record, 'event', 'log.message'),
}
span_context = trace.get_current_span().get_span_context()
if span_context.is_valid:
payload['trace_id'] = format(span_context.trace_id, '032x')
payload['span_id'] = format(span_context.span_id, '016x')
for key, value in record.__dict__.items():
if key in _STANDARD_FIELDS or key.startswith('_'):
continue
if isinstance(value, (str, int, float, bool)) or value is None:
payload[key] = value
if record.exc_info:
payload['exception.type'] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False, separators=(',', ':'))
def safe_extra(event: str, **attributes: object) -> dict[str, object]:
blocked = {
'authorization', 'password', 'token', 'session_token', 'message_body',
'image_bytes', 's3_secret_access_key', 'signed_url',
}
extra: dict[str, object] = {'event': event}
for key, value in attributes.items():
normalized = key.lower()
if normalized in blocked or any(part in normalized for part in ('password', 'secret')):
continue
extra[key] = value
return extra
+122
View File
@@ -0,0 +1,122 @@
from __future__ import annotations
import os
import threading
from dataclasses import dataclass
from typing import Mapping
import psutil
from opentelemetry import metrics
from opentelemetry.metrics import Observation
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0)
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0)
class GaugeState:
def __init__(self) -> None:
self._lock = threading.Lock()
self._values: dict[tuple[tuple[str, str], ...], float] = {}
def set(self, value: float, attributes: Mapping[str, str] | None = None) -> None:
key = tuple(sorted((attributes or {}).items()))
with self._lock:
self._values[key] = float(value)
def add(self, amount: float, attributes: Mapping[str, str] | None = None) -> None:
key = tuple(sorted((attributes or {}).items()))
with self._lock:
self._values[key] = self._values.get(key, 0.0) + amount
def observations(self, _options=None):
with self._lock:
snapshot = tuple(self._values.items())
return [Observation(value, dict(attributes)) for attributes, value in snapshot]
@dataclass(slots=True)
class Metrics:
http_requests: object
http_duration: object
jobs_processed: object
job_duration: object
time_to_ready: object
queue_depth: GaugeState
queue_oldest_age: GaugeState
worker_busy: GaugeState
worker_capacity: GaugeState
db_pool_used: GaugeState
db_pool_waiting: GaugeState
_metrics: Metrics | None = None
def _process_rss(_options=None):
process = psutil.Process(os.getpid())
return [Observation(process.memory_info().rss)]
def _process_cpu(_options=None):
process = psutil.Process(os.getpid())
times = process.cpu_times()
return [Observation(times.user + times.system)]
def get_metrics() -> Metrics:
global _metrics
if _metrics is not None:
return _metrics
meter = metrics.get_meter('messenger.observability', '1.6.0')
queue_depth = GaugeState()
queue_oldest_age = GaugeState()
worker_busy = GaugeState()
worker_capacity = GaugeState()
db_pool_used = GaugeState()
db_pool_waiting = GaugeState()
meter.create_observable_gauge(
'messenger.queue.depth', callbacks=[queue_depth.observations], unit='{job}'
)
meter.create_observable_gauge(
'messenger.queue.oldest.age', callbacks=[queue_oldest_age.observations], unit='s'
)
meter.create_observable_gauge(
'messenger.worker.busy', callbacks=[worker_busy.observations], unit='{worker}'
)
meter.create_observable_gauge(
'messenger.worker.capacity', callbacks=[worker_capacity.observations], unit='{worker}'
)
meter.create_observable_gauge(
'messenger.db.pool.used', callbacks=[db_pool_used.observations], unit='{connection}'
)
meter.create_observable_gauge(
'messenger.db.pool.waiting', callbacks=[db_pool_waiting.observations], unit='{request}'
)
meter.create_observable_gauge('process.memory.rss', callbacks=[_process_rss], unit='By')
meter.create_observable_counter('process.cpu.time', callbacks=[_process_cpu], unit='s')
_metrics = Metrics(
http_requests=meter.create_counter(
'http.server.request.count', unit='{request}', description='HTTP requests'
),
http_duration=meter.create_histogram(
'http.server.request.duration', unit='s', description='HTTP request duration'
),
jobs_processed=meter.create_counter(
'messenger.jobs.processed', unit='{job}', description='Finished job attempts'
),
job_duration=meter.create_histogram(
'messenger.job.processing.duration', unit='s', description='Job processing duration'
),
time_to_ready=meter.create_histogram(
'messenger.attachment.time_to_ready', unit='s', description='Durable accept to ready'
),
queue_depth=queue_depth,
queue_oldest_age=queue_oldest_age,
worker_busy=worker_busy,
worker_capacity=worker_capacity,
db_pool_used=db_pool_used,
db_pool_waiting=db_pool_waiting,
)
return _metrics
+26
View File
@@ -0,0 +1,26 @@
from __future__ import annotations
from collections.abc import Mapping
from opentelemetry import propagate
_ALLOWED = {'traceparent', 'tracestate'}
def inject_trace_context(context=None) -> dict[str, str]:
carrier: dict[str, str] = {}
propagate.inject(carrier, context=context)
return {key: str(value) for key, value in carrier.items() if key in _ALLOWED}
def extract_trace_context(carrier: Mapping[str, object] | None):
normalized: dict[str, str] = {}
for key, value in (carrier or {}).items():
lowered = str(key).lower()
if lowered not in _ALLOWED or value is None:
continue
if isinstance(value, bytes):
value = value.decode('ascii', errors='ignore')
normalized[lowered] = str(value)
return propagate.extract(normalized)
+117
View File
@@ -0,0 +1,117 @@
from __future__ import annotations
import logging
import os
import socket
import sys
from dataclasses import dataclass
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from .config import load_observability_settings
from .logging import JsonFormatter
from .metrics import HTTP_BUCKETS, JOB_BUCKETS, READY_BUCKETS
@dataclass(slots=True)
class ObservabilityRuntime:
tracer_provider: TracerProvider | None = None
meter_provider: MeterProvider | None = None
logger_provider: LoggerProvider | None = None
def shutdown(self) -> None:
if self.logger_provider is not None:
self.logger_provider.shutdown()
if self.meter_provider is not None:
self.meter_provider.shutdown()
if self.tracer_provider is not None:
self.tracer_provider.shutdown()
_runtime: ObservabilityRuntime | None = None
def initialize_observability(default_service_name: str) -> ObservabilityRuntime:
global _runtime
if _runtime is not None:
return _runtime
settings = load_observability_settings(default_service_name)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(JsonFormatter())
handlers: list[logging.Handler] = [console]
runtime = ObservabilityRuntime()
if settings.enabled:
resource = Resource.create({
'service.name': settings.service_name,
'service.version': settings.service_version,
'service.instance.id': os.getenv(
'INSTANCE_ID', f'{socket.gethostname()}:{os.getpid()}'
),
'deployment.environment.name': settings.environment,
})
timeout = settings.export_timeout_seconds
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=f'{settings.endpoint}/v1/traces', timeout=timeout),
max_queue_size=settings.span_queue_size,
max_export_batch_size=min(512, settings.span_queue_size),
schedule_delay_millis=1000,
export_timeout_millis=timeout * 1000,
))
trace.set_tracer_provider(tracer_provider)
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=f'{settings.endpoint}/v1/metrics', timeout=timeout),
export_interval_millis=settings.metric_export_interval_ms,
export_timeout_millis=timeout * 1000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[metric_reader],
views=[
View(
instrument_name='http.server.request.duration',
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
),
View(
instrument_name='messenger.job.processing.duration',
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
),
View(
instrument_name='messenger.attachment.time_to_ready',
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS),
),
],
)
metrics.set_meter_provider(meter_provider)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(endpoint=f'{settings.endpoint}/v1/logs', timeout=timeout),
max_queue_size=settings.log_queue_size,
max_export_batch_size=min(512, settings.log_queue_size),
schedule_delay_millis=1000,
export_timeout_millis=timeout * 1000,
))
set_logger_provider(logger_provider)
otlp_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
otlp_handler.setFormatter(JsonFormatter())
handlers.append(otlp_handler)
runtime = ObservabilityRuntime(tracer_provider, meter_provider, logger_provider)
logging.basicConfig(
level=os.getenv('LOG_LEVEL', 'info').upper(), handlers=handlers, force=True
)
_runtime = runtime
return runtime
+10
View File
@@ -0,0 +1,10 @@
from .attachments import AttachmentRepository
from .chats import ChatRepository
from .messages import MessageRepository
from .users import UserRepository
from .jobs import JobRepository, OutboxRepository
__all__ = [
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
'OutboxRepository', 'UserRepository',
]
+177
View File
@@ -0,0 +1,177 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment, AttachmentVariant
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.job import AttachmentJobRow, OutboxEventRow
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
return {
'schema_version': 1,
'job_id': str(job_id),
'type': 'attachment.process.v1',
'attachment_id': str(attachment_id),
'pipeline_version': pipeline_version,
'attempt': attempt,
'created_at': now.isoformat().replace('+00:00', 'Z'),
}
class AttachmentRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
return AttachmentVariant(
name=row.name,
object_key=row.object_key,
content_type=row.content_type,
size_bytes=row.size_bytes,
width=row.width,
height=row.height,
sha256=row.sha256,
)
@classmethod
def _to_domain(
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
) -> Attachment:
return Attachment(
id=row.id,
chat_id=row.chat_id,
owner_id=row.owner_id,
filename=row.filename,
original_key=row.original_key,
staging_key=row.staging_key,
pipeline_version=row.pipeline_version,
content_type=row.content_type,
size_bytes=row.size_bytes,
sha256=row.sha256,
width=row.width,
height=row.height,
status=row.status,
error_code=row.error_code,
created_at=row.created_at,
updated_at=row.updated_at,
variants=tuple(cls._variant(item) for item in variants),
)
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
async with self._sessions() as session:
found = await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
return found is not None
async def create_queued(
self,
*,
attachment_id: UUID,
job_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
staging_key: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
max_attempts: int,
now: datetime,
traceparent: str | None = None,
tracestate: str | None = None,
) -> Attachment:
row = AttachmentRow(
id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
original_key=None,
staging_key=staging_key,
pipeline_version=1,
content_type=content_type,
size_bytes=size_bytes,
sha256=sha256,
width=width,
height=height,
status='queued',
error_code=None,
created_at=now,
updated_at=now,
)
job = AttachmentJobRow(
id=job_id,
attachment_id=attachment_id,
pipeline_version=1,
state='queued',
attempt=1,
max_attempts=max_attempts,
next_attempt_at=now,
lease_owner=None,
lease_until=None,
last_error_code=None,
replay_count=0,
created_at=now,
updated_at=now,
)
outbox = OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job_id,
payload=job_payload(job_id, attachment_id, 1, 1, now),
traceparent=traceparent,
tracestate=tracestate,
available_at=now,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
async with self._sessions() as session:
async with session.begin():
session.add_all((row, job, outbox))
await session.flush()
return self._to_domain(row)
async def _variants(
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
) -> tuple[AttachmentVariantRow, ...]:
rows = await session.scalars(
select(AttachmentVariantRow)
.where(
AttachmentVariantRow.attachment_id == attachment_id,
AttachmentVariantRow.pipeline_version == pipeline_version,
)
.order_by(AttachmentVariantRow.name)
)
return tuple(rows)
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
async with self._sessions() as session:
row = await session.scalar(
select(AttachmentRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
.where(
AttachmentRow.id == attachment_id,
ChatMemberRow.user_id == user_id,
)
)
if row is None:
return None
return self._to_domain(
row, await self._variants(session, row.id, row.pipeline_version)
)
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Invalid
from ..domain.models import Chat, Keyset
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
class ChatRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
@staticmethod
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(ChatMemberRow.user_id)
.where(ChatMemberRow.chat_id == chat_id)
.order_by(ChatMemberRow.user_id)
)
return tuple(result)
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
async with self._sessions() as session:
async with session.begin():
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
if len(found) != len(member_ids):
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
row = ChatRow(id=chat_id, title=title, created_at=created_at)
session.add(row)
await session.flush()
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
await session.flush()
return self._to_domain(row, member_ids)
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
async with self._sessions() as session:
row = await session.scalar(
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
if row is None:
return None
return self._to_domain(row, await self._members(session, row.id))
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
async with self._sessions() as session:
query = (
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatMemberRow.user_id == user_id)
.order_by(ChatRow.created_at, ChatRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
+388
View File
@@ -0,0 +1,388 @@
from __future__ import annotations
from datetime import datetime, timedelta
from uuid import UUID, uuid4
from sqlalchemy import func, or_, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
from ..observability.propagation import inject_trace_context
from .attachments import AttachmentRepository, job_payload
class JobRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _job(row: AttachmentJobRow) -> AttachmentJob:
return AttachmentJob(
id=row.id,
attachment_id=row.attachment_id,
pipeline_version=row.pipeline_version,
state=row.state,
attempt=row.attempt,
max_attempts=row.max_attempts,
next_attempt_at=row.next_attempt_at,
lease_owner=row.lease_owner,
lease_until=row.lease_until,
last_error_code=row.last_error_code,
replay_count=row.replay_count,
created_at=row.created_at,
updated_at=row.updated_at,
)
async def claim(
self,
job_id: UUID,
attempt: int,
worker_id: str,
now: datetime,
lease_seconds: int,
) -> tuple[AttachmentJob, Attachment] | None:
async with self._sessions() as session:
async with session.begin():
row = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if row is None or row.state != 'queued' or row.attempt != attempt:
return None
attachment_row = await session.get(AttachmentRow, row.attachment_id)
if attachment_row is None:
return None
row.state = 'processing'
row.lease_owner = worker_id
row.lease_until = now + timedelta(seconds=lease_seconds)
row.updated_at = now
attachment_row.status = 'processing'
attachment_row.updated_at = now
session.add(
JobEventRow(
job_id=row.id,
kind='claimed',
attempt=row.attempt,
error_code=None,
created_at=now,
)
)
await session.flush()
return self._job(row), AttachmentRepository._to_domain(attachment_row)
async def state(self, job_id: UUID) -> AttachmentJob | None:
async with self._sessions() as session:
row = await session.get(AttachmentJobRow, job_id)
return self._job(row) if row is not None else None
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentJobRow.id).where(
AttachmentJobRow.attachment_id == attachment_id
)
)
async def failed_staging_key(self, job_id: UUID) -> str | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentRow.staging_key)
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
)
async def ready_effect_complete(self, job_id: UUID) -> bool:
async with self._sessions() as session:
job = await session.get(AttachmentJobRow, job_id)
if job is None or job.state != 'ready':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.original_key:
return False
names = set(
await session.scalars(
select(AttachmentVariantRow.name).where(
AttachmentVariantRow.attachment_id == job.attachment_id,
AttachmentVariantRow.pipeline_version == job.pipeline_version,
)
)
)
return names == {'thumbnail', 'crop', 'optimized'}
async def complete(
self,
job_id: UUID,
worker_id: str,
original_key: str,
variants: tuple[AttachmentVariant, ...],
now: datetime,
) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state == 'ready':
return job is not None
if job.state != 'processing' or job.lease_owner != worker_id:
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None:
return False
for variant in variants:
statement = (
insert(AttachmentVariantRow)
.values(
attachment_id=attachment.id,
pipeline_version=job.pipeline_version,
name=variant.name,
object_key=variant.object_key,
content_type=variant.content_type,
size_bytes=variant.size_bytes,
width=variant.width,
height=variant.height,
sha256=variant.sha256,
created_at=now,
updated_at=now,
)
.on_conflict_do_update(
index_elements=[
AttachmentVariantRow.attachment_id,
AttachmentVariantRow.pipeline_version,
AttachmentVariantRow.name,
],
set_={
'object_key': variant.object_key,
'content_type': variant.content_type,
'size_bytes': variant.size_bytes,
'width': variant.width,
'height': variant.height,
'sha256': variant.sha256,
'updated_at': now,
},
)
)
await session.execute(statement)
attachment.original_key = original_key
attachment.status = 'ready'
attachment.error_code = None
attachment.updated_at = now
job.state = 'ready'
job.lease_owner = None
job.lease_until = None
job.last_error_code = None
job.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind='ready',
attempt=job.attempt,
error_code=None,
created_at=now,
)
)
return True
async def retry_or_fail(
self,
job_id: UUID,
worker_id: str | None,
error_code: str,
permanent: bool,
available_at: datetime,
now: datetime,
) -> str:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state in {'ready', 'failed'}:
return job.state if job is not None else 'missing'
if job.state != 'processing':
return job.state
if worker_id is not None:
if job.lease_owner != worker_id:
return job.state
elif job.lease_until is None or job.lease_until >= now:
return job.state
attachment = await session.get(AttachmentRow, job.attachment_id)
exhausted = job.attempt >= job.max_attempts
if permanent or exhausted:
job.state = 'failed'
attachment.status = 'failed'
attachment.error_code = error_code if permanent else 'retry_exhausted'
kind = 'failed'
else:
job.attempt += 1
job.state = 'queued'
job.next_attempt_at = available_at
attachment.status = 'queued'
attachment.error_code = None
trace_context = inject_trace_context()
session.add(
OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job.id,
payload=job_payload(
job.id,
job.attachment_id,
job.pipeline_version,
job.attempt,
now,
),
traceparent=trace_context.get('traceparent'),
tracestate=trace_context.get('tracestate'),
available_at=available_at,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
)
kind = 'retry_scheduled'
job.last_error_code = error_code
job.lease_owner = None
job.lease_until = None
job.updated_at = now
attachment.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind=kind,
attempt=job.attempt,
error_code=error_code,
created_at=now,
)
)
await session.flush()
return job.state
async def queue_stats(self, now: datetime) -> tuple[int, float]:
async with self._sessions() as session:
depth, oldest = (
await session.execute(
select(func.count(AttachmentJobRow.id), func.min(AttachmentJobRow.created_at))
.where(AttachmentJobRow.state.in_(('queued', 'processing')))
)
).one()
age = max(0.0, (now - oldest).total_seconds()) if oldest is not None else 0.0
return int(depth or 0), age
async def recover_expired(self, now: datetime) -> int:
async with self._sessions() as session:
ids = tuple(
await session.scalars(
select(AttachmentJobRow.id).where(
AttachmentJobRow.state == 'processing',
AttachmentJobRow.lease_until < now,
)
)
)
recovered = 0
for job_id in ids:
state = await self.retry_or_fail(
job_id, None, 'lease_expired', False, now, now
)
recovered += state in {'queued', 'failed'}
return recovered
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state != 'failed':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.staging_key:
return False
if dry_run:
return True
job.state = 'queued'
job.attempt += 1
job.max_attempts = job.attempt + 4
job.replay_count += 1
job.last_error_code = None
job.next_attempt_at = now
job.updated_at = now
attachment.status = 'queued'
attachment.error_code = None
attachment.updated_at = now
session.add_all(
(
OutboxEventRow(
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
traceparent=None, tracestate=None,
available_at=now, published_at=None, claim_owner=None, claim_until=None,
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
),
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
)
)
return True
class OutboxRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
async with self._sessions() as session:
async with session.begin():
rows = tuple(
await session.scalars(
select(OutboxEventRow)
.where(
OutboxEventRow.published_at.is_(None),
OutboxEventRow.available_at <= now,
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
)
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
)
until = now + timedelta(seconds=lease_seconds)
for row in rows:
row.claim_owner = owner
row.claim_until = until
row.publish_attempts += 1
row.updated_at = now
return tuple(
OutboxEvent(
row.id, row.job_id, row.payload, row.available_at,
row.traceparent, row.tracestate,
)
for row in rows
)
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(
OutboxEventRow.id == event_id,
OutboxEventRow.claim_owner == owner,
OutboxEventRow.published_at.is_(None),
)
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
)
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
)
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict, NotFound
from ..domain.models import Keyset, Message
from ..models.attachment import AttachmentRow, MessageAttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.message import MessageRow
class MessageRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
return Message(
id=row.id,
chat_id=row.chat_id,
sender_id=row.sender_id,
text=row.text,
created_at=row.created_at,
attachment_ids=tuple(attachment_ids),
)
@staticmethod
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(MessageAttachmentRow.attachment_id)
.where(MessageAttachmentRow.message_id == message_id)
.order_by(MessageAttachmentRow.position)
)
return tuple(result)
@staticmethod
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
return await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
async def create_for_member(
self,
message_id: UUID,
chat_id: UUID,
sender_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
created_at: datetime,
idempotency_key: str,
) -> Message | None:
async with self._sessions() as session:
async with session.begin():
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
if accessible_chat_id is None:
return None
if attachment_ids:
found = {
attachment_id: status
for attachment_id, status in (
await session.execute(
select(AttachmentRow.id, AttachmentRow.status).where(
AttachmentRow.id.in_(attachment_ids),
AttachmentRow.chat_id == accessible_chat_id,
)
)
).all()
}
if set(found) != set(attachment_ids):
raise NotFound('вложение не найдено')
if any(status == 'failed' for status in found.values()):
raise Conflict('неуспешное вложение нельзя прикрепить')
statement = (
insert(MessageRow)
.values(
id=message_id,
chat_id=accessible_chat_id,
sender_id=sender_id,
text=text,
created_at=created_at,
idempotency_key=idempotency_key,
)
.on_conflict_do_nothing(
index_elements=[
MessageRow.sender_id,
MessageRow.chat_id,
MessageRow.idempotency_key,
]
)
.returning(MessageRow.id)
)
created_id = await session.scalar(statement)
if created_id is not None:
row = await session.get(MessageRow, created_id)
session.add_all(
MessageAttachmentRow(
message_id=created_id,
attachment_id=attachment_id,
position=position,
)
for position, attachment_id in enumerate(attachment_ids)
)
await session.flush()
result_attachment_ids = tuple(attachment_ids)
else:
row = await session.scalar(
select(MessageRow).where(
MessageRow.sender_id == sender_id,
MessageRow.chat_id == accessible_chat_id,
MessageRow.idempotency_key == idempotency_key,
)
)
if row is None:
raise RuntimeError('результат идемпотентной операции не найден')
result_attachment_ids = await self._attachment_ids(session, row.id)
if row.text != text or result_attachment_ids != tuple(attachment_ids):
raise Conflict('Idempotency-Key уже использован с другим сообщением')
return self._to_domain(row, result_attachment_ids)
async def page_for_member(
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
) -> list[Message] | None:
async with self._sessions() as session:
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
if accessible_chat_id is None:
return None
query = (
select(MessageRow)
.where(MessageRow.chat_id == accessible_chat_id)
.order_by(MessageRow.created_at, MessageRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [
self._to_domain(row, await self._attachment_ids(session, row.id))
for row in rows
]
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict
from ..domain.models import User
from ..models.user import UserRow
class UserRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: UserRow) -> User:
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
async def register(
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
) -> User:
row = UserRow(
id=user_id,
username=username,
display_name=display_name,
password_hash=password_hash,
created_at=created_at,
)
try:
async with self._sessions() as session:
async with session.begin():
session.add(row)
await session.flush()
except IntegrityError as exc:
raise Conflict('username уже занят') from exc
return self._to_domain(row)
async def get_by_id(self, user_id: UUID) -> User | None:
async with self._sessions() as session:
row = await session.get(UserRow, user_id)
return self._to_domain(row) if row is not None else None
async def get_credentials(self, username: str) -> UserRow | None:
async with self._sessions() as session:
return await session.scalar(select(UserRow).where(UserRow.username == username))
@classmethod
def to_domain(cls, row: UserRow) -> User:
return cls._to_domain(row)
View File
+185
View File
@@ -0,0 +1,185 @@
from __future__ import annotations
import asyncio
import hashlib
import logging
from datetime import datetime, timezone
from tempfile import SpooledTemporaryFile
from typing import AsyncIterator, BinaryIO
import unicodedata
from uuid import UUID, uuid4
from PIL import Image, UnidentifiedImageError
from ..domain.errors import (
Conflict,
Invalid,
NotFound,
PayloadTooLarge,
Unavailable,
UnsupportedMediaType,
)
from ..domain.models import Attachment, User
from ..observability.logging import safe_extra
from ..observability.propagation import inject_trace_context
from ..repositories import AttachmentRepository
from ..storage import ObjectStream, S3Storage
logger = logging.getLogger('app.attachments')
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
def _validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255:
raise Invalid('filename должен содержать от 1 до 255 символов')
if any(
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
for character in filename
):
raise Invalid('filename содержит запрещённые символы')
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
try:
source.seek(0)
with Image.open(source) as image:
actual_type = MIME_BY_FORMAT.get(image.format or '')
if actual_type is None or actual_type != declared_type:
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
width, height = image.size
if width <= 0 or height <= 0:
raise UnsupportedMediaType('изображение имеет неверные размеры')
if width * height > max_pixels:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
image.verify()
except Image.DecompressionBombError as error:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
except (UnidentifiedImageError, OSError, SyntaxError) as error:
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
finally:
source.seek(0)
return actual_type, width, height
class AttachmentService:
def __init__(
self,
repository: AttachmentRepository,
storage: S3Storage,
max_upload_bytes: int,
max_image_pixels: int,
job_max_attempts: int,
) -> None:
self._repository = repository
self._storage = storage
self._max_upload_bytes = max_upload_bytes
self._max_image_pixels = max_image_pixels
self._job_max_attempts = job_max_attempts
async def upload(
self,
actor: User,
chat_id: UUID,
filename: str,
declared_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
_validate_filename(filename)
if declared_type not in MIME_BY_FORMAT.values():
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
raise NotFound('чат не найден')
digest = hashlib.sha256()
size = 0
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
async for chunk in chunks:
size += len(chunk)
if size > self._max_upload_bytes:
raise PayloadTooLarge('файл превышает лимит 10 MiB')
digest.update(chunk)
source.write(chunk)
if size == 0:
raise UnsupportedMediaType('пустой файл не является изображением')
actual_type, width, height = await asyncio.to_thread(
_inspect_image,
source,
declared_type,
self._max_image_pixels,
)
attachment_id = uuid4()
job_id = uuid4()
staging_key = f'staging/{attachment_id}/original'
await self._storage.put(staging_key, source, size, actual_type)
now = datetime.now(timezone.utc)
trace_context = inject_trace_context()
try:
attachment = await self._repository.create_queued(
attachment_id=attachment_id,
job_id=job_id,
chat_id=chat_id,
owner_id=actor.id,
filename=filename,
staging_key=staging_key,
content_type=actual_type,
size_bytes=size,
sha256=digest.hexdigest(),
width=width,
height=height,
max_attempts=self._job_max_attempts,
now=now,
traceparent=trace_context.get('traceparent'),
tracestate=trace_context.get('tracestate'),
)
logger.info(
'attachment accepted for asynchronous processing',
extra=safe_extra(
'attachment.accepted', attachment_id=str(attachment_id),
job_id=str(job_id), pipeline_version=1,
),
)
return attachment
except Exception:
try:
await self._storage.remove(staging_key)
except Unavailable:
logger.warning(
'failed to compensate staging object',
extra=safe_extra(
'staging.cleanup_deferred', attachment_id=str(attachment_id)
),
)
raise
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
attachment = await self._repository.get_for_member(attachment_id, actor.id)
if attachment is None:
raise NotFound('вложение не найдено')
return attachment
async def content(
self, actor: User, attachment_id: UUID, variant: str
) -> tuple[Attachment, ObjectStream, str, int]:
attachment = await self.get(actor, attachment_id)
if attachment.status != 'ready':
raise Conflict('вложение ещё не готово')
if variant == 'original':
key = attachment.original_key
content_type = attachment.content_type
size_bytes = attachment.size_bytes
else:
found = next((item for item in attachment.variants if item.name == variant), None)
if found is None:
raise NotFound('вариант вложения не найден')
key = found.object_key
content_type = found.content_type
size_bytes = found.size_bytes
if key is None:
raise Unavailable('объект вложения отсутствует в метаданных')
try:
stream = await self._storage.get(key)
except FileNotFoundError as error:
raise Unavailable('объект вложения отсутствует в хранилище') from error
return attachment, stream, content_type, size_bytes
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import Invalid, NotFound
from ..domain.models import Chat, Keyset, Page, User
from ..repositories import ChatRepository
class ChatService:
def __init__(self, repository: ChatRepository) -> None:
self._repository = repository
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
if actor.id not in member_ids:
raise Invalid('вызывающий должен входить в member_ids')
return await self._repository.create(
chat_id=uuid4(),
title=title,
member_ids=member_ids,
created_at=datetime.now(timezone.utc),
)
async def get(self, actor: User, chat_id: UUID) -> Chat:
chat = await self._repository.get_for_member(chat_id, actor.id)
if chat is None:
raise NotFound('чат не найден')
return chat
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import NotFound
from ..domain.models import Keyset, Message, Page, User
from ..repositories import MessageRepository
class MessageService:
def __init__(self, repository: MessageRepository) -> None:
self._repository = repository
async def send(
self,
actor: User,
chat_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
idempotency_key: str,
) -> Message:
message = await self._repository.create_for_member(
message_id=uuid4(),
chat_id=chat_id,
sender_id=actor.id,
text=text,
attachment_ids=attachment_ids,
created_at=datetime.now(timezone.utc),
idempotency_key=idempotency_key,
)
if message is None:
raise NotFound('чат не найден')
return message
async def history(
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
) -> Page:
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
if rows is None:
raise NotFound('чат не найден')
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from uuid import UUID, uuid4
from ..auth.passwords import hash_password, password_matches
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..repositories import UserRepository
class UserService:
def __init__(self, repository: UserRepository) -> None:
self._repository = repository
async def register(self, username: str, display_name: str, password: str) -> User:
return await self._repository.register(
user_id=uuid4(),
username=username,
display_name=display_name,
password_hash=hash_password(password),
created_at=datetime.now(timezone.utc),
)
async def identify(self, user_id: UUID) -> User:
user = await self._repository.get_by_id(user_id)
if user is None:
raise Unauthorized('требуется действующая сессия')
return user
async def authenticate(self, username: str, password: str) -> User | None:
row = await self._repository.get_credentials(username)
if row is None or not password_matches(row.password_hash, password):
return None
return self._repository.to_domain(row)
+3
View File
@@ -0,0 +1,3 @@
from .s3 import ObjectStream, S3Storage, StoredObject
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
+170
View File
@@ -0,0 +1,170 @@
from __future__ import annotations
import asyncio
import certifi
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, Iterator
from io import BytesIO
from urllib.parse import urlparse
from minio import Minio
from minio.error import InvalidResponseError, S3Error
from urllib3 import PoolManager, Retry, Timeout
from urllib3.exceptions import HTTPError
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from ..domain.errors import Unavailable
tracer = trace.get_tracer('messenger.s3', '1.6.0')
def _unavailable(error: Exception) -> Unavailable:
return Unavailable('объектное хранилище недоступно')
@dataclass(slots=True)
class ObjectStream:
response: object
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
try:
while chunk := self.response.read(size):
yield chunk
finally:
self.response.close()
self.response.release_conn()
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
last_modified: datetime
class S3Storage:
def __init__(
self,
endpoint: str,
access_key: str,
secret_key: str,
bucket: str,
region: str,
) -> None:
parsed = urlparse(endpoint)
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
self._client = Minio(
parsed.netloc,
access_key=access_key,
secret_key=secret_key,
secure=parsed.scheme == 'https',
region=region,
http_client=PoolManager(
maxsize=8,
timeout=Timeout(connect=2, read=5),
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
),
)
self.bucket = bucket
@staticmethod
def _translate(error: Exception) -> None:
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
raise FileNotFoundError from error
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
raise _unavailable(error) from error
raise error
async def ping(self) -> None:
try:
with tracer.start_as_current_span(
's3 bucket exists', kind=SpanKind.CLIENT,
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
):
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
except Exception as error:
self._translate(error)
if not exists:
raise Unavailable('bucket объектного хранилища не создан')
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
source.seek(0)
try:
with tracer.start_as_current_span(
's3 put object', kind=SpanKind.CLIENT,
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
):
await asyncio.to_thread(
self._client.put_object,
self.bucket,
key,
source,
length,
content_type,
)
except Exception as error:
self._translate(error)
async def get(self, key: str) -> ObjectStream:
try:
with tracer.start_as_current_span(
's3 get object', kind=SpanKind.CLIENT,
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
):
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
except Exception as error:
self._translate(error)
return ObjectStream(response)
async def read(self, key: str, max_bytes: int) -> bytes:
stream = await self.get(key)
result = bytearray()
for chunk in stream.chunks():
result.extend(chunk)
if len(result) > max_bytes:
raise ValueError('stored object exceeds configured byte limit')
return bytes(result)
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
await self.put(key, BytesIO(content), len(content), content_type)
async def exists(self, key: str) -> bool:
try:
with tracer.start_as_current_span(
's3 stat object', kind=SpanKind.CLIENT,
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
):
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
return True
except Exception as error:
try:
self._translate(error)
except FileNotFoundError:
return False
raise
async def remove(self, key: str) -> None:
try:
with tracer.start_as_current_span(
's3 remove object', kind=SpanKind.CLIENT,
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
):
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
except Exception as error:
self._translate(error)
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
def collect() -> tuple[StoredObject, ...]:
return tuple(
StoredObject(item.object_name, item.last_modified)
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
)
try:
return await asyncio.to_thread(collect)
except Exception as error:
self._translate(error)
+1
View File
@@ -0,0 +1 @@
"""Операторские CLI-инструменты приложения."""
+65
View File
@@ -0,0 +1,65 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
sessions = create_session_factory(engine)
async with sessions() as session:
known = set(
key
for key in await session.scalars(select(AttachmentRow.original_key))
if key is not None
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
removed = 0
for item in await storage.list('originals/'):
if item.key in known or item.last_modified > cutoff:
continue
action = 'DELETE' if apply else 'WOULD_DELETE'
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
if apply:
await storage.remove(item.key)
removed += 1
print(
f'summary mode={"apply" if apply else "dry-run"} '
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, retention_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
try:
async with sessions() as session:
active = set(
key
for key in await session.scalars(
select(AttachmentRow.staging_key).where(
AttachmentRow.staging_key.is_not(None),
AttachmentRow.status.in_(('queued', 'processing')),
)
)
if key is not None
)
removed = 0
for item in await storage.list('staging/'):
if item.key in active or item.last_modified > cutoff:
continue
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
if apply:
await storage.remove(item.key)
removed += 1
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--retention-seconds', type=int)
args = parser.parse_args()
settings = load_settings()
retention = args.retention_seconds or settings.staging_retention_seconds
if retention < 60:
parser.error('retention must be at least 60 seconds')
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
if __name__ == '__main__':
main()
+250
View File
@@ -0,0 +1,250 @@
from __future__ import annotations
import argparse
import asyncio
import json
import time
import tracemalloc
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Sequence
from uuid import UUID
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage, StoredObject
MISSING_ERROR_CODE = 'original_missing'
@dataclass(frozen=True, slots=True)
class AttachmentSnapshot:
id: UUID
object_key: str
status: str
updated_at: datetime
@dataclass(frozen=True, slots=True)
class ReconcilePlan:
orphan: tuple[StoredObject, ...]
fresh_orphan: tuple[StoredObject, ...]
missing: tuple[AttachmentSnapshot, ...]
fresh_missing: tuple[AttachmentSnapshot, ...]
correct: int
def build_plan(
attachments: Sequence[AttachmentSnapshot],
objects: Sequence[StoredObject],
cutoff: datetime,
) -> ReconcilePlan:
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
known_keys = {attachment.object_key for attachment in attachments}
object_keys = {item.key for item in objects}
unknown_objects = [item for item in objects if item.key not in known_keys]
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
ready_without_object = [
attachment
for attachment in attachments
if attachment.status == 'ready' and attachment.object_key not in object_keys
]
missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
)
fresh_missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
)
correct = sum(attachment.object_key in object_keys for attachment in attachments)
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
async def _load_attachments(
sessions: async_sessionmaker[AsyncSession],
) -> tuple[AttachmentSnapshot, ...]:
async with sessions() as session:
rows = (
await session.execute(
select(
AttachmentRow.id,
AttachmentRow.original_key,
AttachmentRow.status,
AttachmentRow.updated_at,
)
)
).all()
return tuple(AttachmentSnapshot(*row) for row in rows)
async def _metadata_key_exists(
sessions: async_sessionmaker[AsyncSession], object_key: str
) -> bool:
async with sessions() as session:
attachment_id = await session.scalar(
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
)
return attachment_id is not None
async def reconcile(
sessions: async_sessionmaker[AsyncSession],
storage: S3Storage,
*,
apply: bool,
grace_seconds: int,
now: datetime | None = None,
output: Callable[[str], None] = print,
) -> dict[str, int | float | str]:
started = time.perf_counter()
tracemalloc.start()
current_time = now or datetime.now(timezone.utc)
cutoff = current_time - timedelta(seconds=grace_seconds)
attachments = await _load_attachments(sessions)
objects = await storage.list('originals/')
plan = build_plan(attachments, objects, cutoff)
for item in plan.fresh_orphan:
output(
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
)
for attachment in plan.fresh_missing:
output(
'SKIP_FRESH_MISSING '
f'attachment_id={attachment.id} key={attachment.object_key} '
f'updated={attachment.updated_at.isoformat()}'
)
removed = 0
race_protected = 0
for item in plan.orphan:
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
if not apply:
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
continue
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
# metadata immediately before a destructive action so a row created after
# the initial snapshot protects its object.
if await _metadata_key_exists(sessions, item.key):
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
race_protected += 1
continue
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
await storage.remove(item.key)
removed += 1
marked_failed = 0
for attachment in plan.missing:
if not apply:
output(
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
continue
# The object may have been restored after list_objects(). Do not turn
# healthy metadata into failed based only on the earlier snapshot.
if await storage.exists(attachment.object_key):
output(
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
continue
async with sessions() as session:
async with session.begin():
result = await session.execute(
update(AttachmentRow)
.where(
AttachmentRow.id == attachment.id,
AttachmentRow.original_key == attachment.object_key,
AttachmentRow.status == 'ready',
AttachmentRow.updated_at == attachment.updated_at,
)
.values(
status='failed',
error_code=MISSING_ERROR_CODE,
updated_at=current_time,
)
)
changed = result.rowcount or 0
if changed:
output(
f'MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
marked_failed += changed
else:
output(
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
_, peak_bytes = tracemalloc.get_traced_memory()
tracemalloc.stop()
summary: dict[str, int | float | str] = {
'mode': 'apply' if apply else 'dry-run',
'metadata': len(attachments),
'objects': len(objects),
'correct': plan.correct,
'orphan': len(plan.orphan),
'missing': len(plan.missing),
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
'removed': removed,
'marked_failed': marked_failed,
'race_protected': race_protected,
'grace_seconds': grace_seconds,
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
'peak_memory_bytes': peak_bytes,
}
output(f'summary {json.dumps(summary, sort_keys=True)}')
return summary
async def run(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
await reconcile(
create_session_factory(engine),
storage,
apply=apply,
grace_seconds=grace_seconds,
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timezone
from uuid import UUID
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..repositories import JobRepository
from ..storage import S3Storage
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
try:
repository = JobRepository(create_session_factory(engine))
if job_id is None:
assert attachment_id is not None
job_id = await repository.id_for_attachment(attachment_id)
if job_id is None:
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
return 2
staging_key = await repository.failed_staging_key(job_id)
if staging_key is None:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
if not await storage.exists(staging_key):
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
return 2
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
if not accepted:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument('--job-id', type=UUID)
target.add_argument('--attachment-id', type=UUID)
parser.add_argument('--dry-run', action='store_true')
args = parser.parse_args()
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
if __name__ == '__main__':
main()
+138
View File
@@ -0,0 +1,138 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from io import BytesIO
from uuid import NAMESPACE_URL, uuid5
from sqlalchemy import delete, insert
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
from ..storage import S3Storage
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
WORKERS = 16
async def _put_objects(
storage: S3Storage,
keys: list[str],
) -> None:
for offset in range(0, len(keys), WORKERS):
batch = keys[offset : offset + WORKERS]
await asyncio.gather(
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
)
async def prepare(size: int) -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
old = datetime.now(timezone.utc) - timedelta(hours=2)
rows = [
{
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
'chat_id': CHAT_ID,
'owner_id': USER_ID,
'filename': f'benchmark-{index}.png',
'original_key': f'{OBJECT_PREFIX}{index}',
'staging_key': None,
'pipeline_version': 1,
'content_type': 'image/png',
'size_bytes': 1,
'sha256': '0' * 64,
'width': 1,
'height': 1,
'status': 'ready',
'error_code': None,
'created_at': old,
'updated_at': old,
}
for index in range(size)
]
try:
async with sessions() as session:
async with session.begin():
session.add(UserRow(
id=USER_ID,
username=f'bench-{USER_ID.hex[:16]}',
display_name='Reconciler benchmark',
password_hash='not-used',
created_at=old,
))
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
await session.flush()
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
await session.flush()
for offset in range(0, size, 500):
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
await _put_objects(storage, [row['original_key'] for row in rows])
print(f'prepared metadata={size} objects={size}')
finally:
await engine.dispose()
async def cleanup() -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
objects = await storage.list(OBJECT_PREFIX)
keys = [item.key for item in objects]
for offset in range(0, len(keys), WORKERS):
await asyncio.gather(
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
)
async with sessions() as session:
async with session.begin():
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
print(f'cleaned objects={len(keys)}')
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
)
subparsers = parser.add_subparsers(dest='command', required=True)
prepare_parser = subparsers.add_parser('prepare')
prepare_parser.add_argument('--size', type=int, required=True)
subparsers.add_parser('cleanup')
args = parser.parse_args()
if args.command == 'prepare':
if args.size < 1 or args.size > 10000:
parser.error('--size must be between 1 and 10000')
asyncio.run(prepare(args.size))
else:
asyncio.run(cleanup())
if __name__ == '__main__':
main()
+14
View File
@@ -0,0 +1,14 @@
services:
observability-test:
image: python:3.13.15-slim
working_dir: /tests
command: ["python", "observability_smoke.py", "--grafana-url", "http://grafana:3000"]
environment:
GRAFANA_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GRAFANA_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
volumes:
- ./tests:/tests:ro
depends_on:
grafana:
condition: service_healthy
restart: "no"
+198
View File
@@ -0,0 +1,198 @@
services:
postgres:
image: postgres:17.4-alpine
environment:
POSTGRES_USER: messenger_test
POSTGRES_PASSWORD: messenger_test
POSTGRES_DB: messenger_test
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
interval: 2s
timeout: 2s
retries: 30
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
broker:
image: rabbitmq:4.1.4-alpine
environment:
RABBITMQ_DEFAULT_USER: messenger_test
RABBITMQ_DEFAULT_PASS: messenger_test
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 2s
timeout: 5s
retries: 30
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 2s
timeout: 2s
retries: 30
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
ports:
- "127.0.0.1:18080:8080"
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
JOB_MAX_ATTEMPTS: 5
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 30
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
WORKER_CONCURRENCY: 2
WORKER_PREFETCH: 2
JOB_MAX_ATTEMPTS: 5
JOB_TIMEOUT_SECONDS: 60
JOB_LEASE_SECONDS: 90
RETRY_BASE_SECONDS: 1
RETRY_MAX_SECONDS: 4
STAGING_RETENTION_SECONDS: 86400
THUMBNAIL_JPEG_QUALITY: 85
CROP_JPEG_QUALITY: 85
OPTIMIZED_WEBP_QUALITY: 82
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
tests:
build:
context: .
dockerfile: Dockerfile.test
environment:
BASE_URL: http://api:8080
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
api:
condition: service_healthy
restart: "no"
+346
View File
@@ -0,0 +1,346 @@
x-otel-environment: &otel-environment
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2}
OTEL_SERVICE_VERSION: ${OTEL_SERVICE_VERSION:-1.6.0}
OTEL_DEPLOYMENT_ENVIRONMENT: ${OTEL_DEPLOYMENT_ENVIRONMENT:-lab}
OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048}
OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048}
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-5000}
services:
postgres:
image: postgres:17.4-alpine
cpus: "1.0"
mem_limit: 512m
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 20
redis:
image: redis:7.4.2-alpine
cpus: "0.25"
mem_limit: 128m
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
broker:
image: rabbitmq:4.1.4-alpine
cpus: "0.5"
mem_limit: 384m
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
volumes:
- broker_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
cpus: "0.5"
mem_limit: 256m
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- s3_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 3s
retries: 20
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
cpus: "0.25"
mem_limit: 128m
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
cpus: "0.5"
mem_limit: 256m
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
cpus: "0.5"
mem_limit: 256m
environment:
<<: *otel-environment
OTEL_SERVICE_NAME: messenger-api
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
LOG_LEVEL: ${LOG_LEVEL:-info}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
expose:
- "8080"
deploy:
replicas: 2
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 10s
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
cpus: "0.25"
mem_limit: 192m
environment:
<<: *otel-environment
OTEL_SERVICE_NAME: messenger-publisher
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 10s
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
cpus: "1.0"
mem_limit: 768m
environment:
<<: *otel-environment
OTEL_SERVICE_NAME: messenger-worker
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 70s
lb:
image: haproxy:3.2.26-alpine
cpus: "0.25"
mem_limit: 128m
ports:
- "${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 5s
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
cpus: "0.5"
mem_limit: 256m
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
depends_on:
loki:
condition: service_started
tempo:
condition: service_started
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.2.1
cpus: "0.5"
mem_limit: 384m
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
depends_on:
otel-collector:
condition: service_started
restart: unless-stopped
loki:
image: grafana/loki:3.4.2
user: "0"
cpus: "0.5"
mem_limit: 384m
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.2
user: "0"
cpus: "0.5"
mem_limit: 384m
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
restart: unless-stopped
grafana:
image: grafana/grafana:11.6.0
cpus: "0.5"
mem_limit: 384m
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./infra/grafana/runbooks:/var/lib/grafana/runbooks:ro
- grafana_data:/var/lib/grafana
depends_on:
prometheus:
condition: service_started
loki:
condition: service_started
tempo:
condition: service_started
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
volumes:
postgres_data:
redis_data:
s3_data:
broker_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
+37
View File
@@ -0,0 +1,37 @@
# Результаты повторной проверки 2026-11-20
Стенд: Apple M4 Pro, 24 GiB RAM; Docker Desktop VM 12 CPU / 9.70 GiB;
лимиты сервисов из `compose.yaml`, только synthetic data.
```text
make test
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
make test-observability
PASS: dashboard, alerts, metrics, correlated logs and cross-process trace
make test-retry-trace
{"scenario":"retry_trace","attachment_id":"7ee0da1b-0c50-4d2e-9bf8-e2c68efb2ce4",
"trace_id":"e8e2aff13b7bdb632c9cec5007dcc6d4","producer_spans":2,
"consumer_spans":2,"result":"PASS"}
make test-alert-worker
{"scenario":"worker_outage","firing_state":"Alerting","detect_seconds":61.8,
"resolved_state":"Normal","recover_seconds":15.5,"result":"PASS"}
make test-alert-database
{"scenario":"database_outage","firing_state":"Alerting","detect_seconds":72.2,
"resolved_state":"Normal","recover_seconds":113.1,
"trace_id":"34f96d6fc934f7c54075c44a7b39c176","db_span":"connect",
"db_system":"postgresql","result":"PASS"}
make test-telemetry-outage
{"scenario":"telemetry_backend_outage","business_smoke":"PASS",
"collector":"running","duration_seconds":49.5,"rss_before_mib":194.6,
"rss_peak_mib":198.0,"memory_limit_mib":256,"failed_log_records":34,
"failed_spans":30,"result":"PASS"}
```
Worker incident использовал attachment `a88a0cfd-bbda-4151-8584-de27154a8dcb`.
Это идентификатор только локальных синтетических данных; credentials/tokens/body в
доказательство не сохранялись.
+21
View File
@@ -0,0 +1,21 @@
{
"lab": 6,
"implementation_commit": "9e6428d",
"host": "Apple M4 Pro, 24 GiB RAM",
"docker_vm": "12 CPU, 9.70 GiB RAM; per-service limits from compose.yaml",
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 120.0,
"elapsed_seconds": 120.084,
"concurrency": 8,
"successful_requests": 25638,
"generated_messages": 25638,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 213.501,
"successful_latency_ms": {
"p50": 17.158,
"p95": 99.774,
"p99": 171.24
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,183 @@
{
"annotations": {"list": []},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"description": "Business requests per second; health routes are excluded.",
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 0},
"id": 1,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (http_route, http_request_method) (rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[$__rate_interval]))", "legendFormat": "{{http_request_method}} {{http_route}}", "refId": "A"}],
"title": "API RPS",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"description": "5xx business requests divided by all business requests over 2 minutes.",
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 0},
"id": 2,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
"targets": [{"expr": "(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\",http_response_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[2m])), 0.001)) or vector(0)", "legendFormat": "5xx ratio", "refId": "A"}],
"title": "API server error ratio",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"description": "Percentiles are computed from aggregate histogram buckets across API replicas.",
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 0},
"id": 3,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "API duration p50 / p95 / p99",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8},
"id": 4,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", "legendFormat": "{{outcome}}", "refId": "A"}],
"title": "Jobs by outcome",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8},
"id": 5,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "Worker duration p50 / p95 / p99",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"description": "Time from durable acceptance to ready commit.",
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
"id": 6,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "End-to-end time to ready",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"description": "Logical non-terminal jobs from PostgreSQL and age of the oldest one.",
"fieldConfig": {"defaults": {}, "overrides": [{"matcher": {"id": "byName", "options": "oldest age"}, "properties": [{"id": "unit", "value": "s"}]}]},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16},
"id": 7,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "max(messenger_queue_depth{messaging_destination_name=\"attachments.process\"})", "legendFormat": "depth", "refId": "A"},
{"expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process\"})", "legendFormat": "oldest age", "refId": "B"}
],
"title": "Queue depth and oldest job",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16},
"id": 8,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "messenger_db_pool_used", "legendFormat": "used {{service_name}} {{service_instance_id}}", "refId": "A"},
{"expr": "messenger_db_pool_waiting", "legendFormat": "waiting {{service_name}} {{service_instance_id}}", "refId": "B"}
],
"title": "PostgreSQL pool used / waiting",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16},
"id": 9,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "saturation", "refId": "A"}],
"title": "Worker saturation",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
"id": 10,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "rate(process_cpu_time_seconds_total[5m])", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
"title": "Process CPU",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
"id": 11,
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "process_memory_rss_bytes", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
"title": "Process RSS",
"type": "timeseries"
},
{
"datasource": {"type": "loki", "uid": "loki"},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
"id": 12,
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
"targets": [{"expr": "{service_name=~\"$service\"} | json | http_route!~\"/health/.*\" | line_format \"{{.severity}} {{.service_name}} {{.event}}{{if .http_method}} {{.http_method}} {{.http_route}} status={{.http_status_code}} duration={{.duration_seconds}}{{end}} trace_id={{.trace_id}} {{.message}}\"", "refId": "A"}],
"title": "Correlated application logs",
"type": "logs"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": ["mtusi", "lab6", "opentelemetry"],
"templating": {
"list": [
{
"current": {"selected": false, "text": ".*", "value": ".*"},
"datasource": {"type": "loki", "uid": "loki"},
"definition": "label_values(service_name)",
"hide": 0,
"includeAll": true,
"label": "Service",
"multi": true,
"name": "service",
"options": [],
"query": "label_values(service_name)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {"from": "now-30m", "to": "now"},
"timepicker": {},
"timezone": "browser",
"title": "Messenger Observability",
"uid": "messenger-observability",
"version": 1,
"weekStart": ""
}
@@ -0,0 +1,120 @@
apiVersion: 1
groups:
- orgId: 1
name: Messenger SLO alerts
folder: MTUSI Lab 6
interval: 10s
rules:
- uid: messenger-api-high-error-rate
title: ApiHighServerErrorRate
condition: C
data:
- refId: A
relativeTimeRange:
from: 120
to: 0
datasourceUid: prometheus
model:
datasource:
type: prometheus
uid: prometheus
editorMode: code
expr: '(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*",http_response_status_class="5xx"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*"}[2m])), 0.001)) or vector(0)'
instant: true
intervalMs: 1000
maxDataPoints: 43200
range: false
refId: A
- refId: C
relativeTimeRange:
from: 0
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: [0.05]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
refId: C
type: threshold
dashboardUid: messenger-observability
panelId: 2
noDataState: OK
execErrState: Error
for: 1m
annotations:
summary: Более 5% бизнес-запросов завершаются 5xx.
runbook_url: /var/lib/grafana/runbooks/api-high-error-rate.md
labels:
service: messenger-api
severity: warning
isPaused: false
- uid: messenger-queue-oldest-growing
title: QueueOldestJobGrowing
condition: C
data:
- refId: A
relativeTimeRange:
from: 60
to: 0
datasourceUid: prometheus
model:
datasource:
type: prometheus
uid: prometheus
editorMode: code
expr: '(max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process"}) > 20 and max(messenger_queue_depth{messaging_destination_name="attachments.process"}) > 0) or vector(0)'
instant: true
intervalMs: 1000
maxDataPoints: 43200
range: false
refId: A
- refId: C
relativeTimeRange:
from: 0
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: [0.5]
type: gt
operator:
type: and
query:
params: [C]
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
refId: C
type: threshold
dashboardUid: messenger-observability
panelId: 7
noDataState: OK
execErrState: Error
for: 15s
annotations:
summary: Возраст старейшей незавершённой job превысил 20 секунд.
runbook_url: /var/lib/grafana/runbooks/queue-oldest-job.md
labels:
service: messenger-worker
severity: warning
isPaused: false
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: MTUSI Lab 6
type: file
disableDeletion: true
updateIntervalSeconds: 10
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,47 @@
apiVersion: 1
deleteDatasources:
- name: Prometheus
orgId: 1
- name: Loki
orgId: 1
- name: Tempo
orgId: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherRegex: 'trace[_]?id["=: ]+([0-9a-f]{32})'
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -1m
spanEndTimeShift: 1m
filterByTraceID: true
filterBySpanID: false
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
@@ -0,0 +1 @@
@@ -0,0 +1,13 @@
# ApiHighServerErrorRate
Alert использует только 5xx бизнес-запросов и исключает `/health/*`. Клиентские 4xx
не считаются отказом сервера.
1. Открыть панели API error ratio и DB pool.
2. В Tempo найти медленные/error server spans за окно alert.
3. Сравнить длительность server span с дочерними PostgreSQL/S3 spans.
4. В Loki отфильтровать `{service_name="messenger-api"} | trace_id="..."`.
5. Проверить readiness зависимостей, не удаляя volumes и не перезапуская весь стенд.
После восстановления убедиться, что новые запросы успешны, error ratio падает и alert
переходит в Normal. Короткий единичный 5xx может не сработать из-за окна и `for: 1m`.
@@ -0,0 +1,13 @@
# QueueOldestJobGrowing
Alert означает, что в PostgreSQL остаётся хотя бы одна `queued`/`processing` job старше
20 секунд. Это возраст backlog, а не duration одного consumer span.
1. Проверить queue depth, oldest age и worker busy/capacity.
2. В Loki открыть `{service_name="messenger-worker"}` и проверить retry/failure events.
3. По trace ID перейти в Tempo и проверить consumer, S3 и PostgreSQL spans.
4. Восстановить worker или отказавшую зависимость, не создавая новую attachment job.
5. Убедиться, что исходная job стала `ready`, backlog уменьшился и alert resolved.
Порог учебный и рассчитан на локальную демонстрацию; для production его связывают с
реальным SLO времени до ready.
+31
View File
@@ -0,0 +1,31 @@
global
log stdout format raw local0
defaults
log global
mode http
option httplog
timeout connect 2s
timeout client 30s
timeout server 30s
timeout check 2s
resolvers docker
nameserver docker_dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
frontend public
bind :8080
default_backend api_backend
backend api_backend
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 2 api:8080
+41
View File
@@ -0,0 +1,41 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
filesystem:
directory: /loki/chunks
limits_config:
allow_structured_metadata: true
retention_period: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+76
View File
@@ -0,0 +1,76 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
timeout: 1s
send_batch_size: 512
send_batch_max_size: 1024
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
sending_queue:
enabled: true
num_consumers: 2
queue_size: 500
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
sending_queue:
enabled: true
num_consumers: 2
queue_size: 500
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
extensions: [health_check]
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
+11
View File
@@ -0,0 +1,11 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: otel-application-metrics
static_configs:
- targets: [otel-collector:8889]
- job_name: otel-collector
static_configs:
- targets: [otel-collector:8888]
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
}
]
}
+29
View File
@@ -0,0 +1,29 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+16
View File
@@ -0,0 +1,16 @@
fastapi==0.115.12
uvicorn[standard]==0.34.2
SQLAlchemy[asyncio]==2.0.40
asyncpg==0.30.0
alembic==1.15.2
redis==5.2.1
argon2-cffi==25.1.0
minio==7.2.20
Pillow==13.0.0
aio-pika==9.5.5
opentelemetry-api==1.31.1
opentelemetry-sdk==1.31.1
opentelemetry-exporter-otlp-proto-http==1.31.1
opentelemetry-instrumentation-fastapi==0.52b1
opentelemetry-instrumentation-sqlalchemy==0.52b1
psutil==7.0.0
+28 -1
View File
@@ -1,6 +1,11 @@
# Публичные проверки
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; публичные
проверки не создают готовое приложение и сами не управляют Docker. Отказные проверки
lab6 ниже управляют только явно указанными Compose-сервисами. `make check` можно
выполнить на исходном skeleton, `make test` — только после реализации API. Полная
OpenAPI-валидация отдельно: `make validate` после установки
`requirements-dev.txt` в venv.
```sh
make check
@@ -13,6 +18,28 @@ python3 tests/smoke.py --base-url http://localhost:8080
Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
## Наблюдаемость и контролируемые отказы, lab6
При запущенном Compose-стенде доступны дополнительные проверки реализации:
```sh
make test-observability # dashboard, alerts, Prometheus, Loki и Tempo
make test-retry-trace # временная S3-ошибка и retry в одной трассе
make test-alert-worker # временно останавливает и восстанавливает worker
make test-alert-database # временно останавливает и восстанавливает PostgreSQL
make test-telemetry-outage # временно останавливает и восстанавливает Loki/Tempo
```
Отказные сценарии изменяют состояние только перечисленных контейнеров и всегда
выполняют восстановление в `finally`. Используются уникальные синтетические данные;
секреты и tokens не печатаются. Retry-тест временно отключает S3 и проверяет
две producer/consumer попытки в одной трассе. Worker-тест ждёт `QueueOldestJobGrowing` в
`Alerting`, затем `Normal`; DB-тест аналогично проверяет `ApiHighServerErrorRate`.
Тест telemetry outage держит backends выключенными более 30 с и фиксирует RSS
Collector и `send_failed` counters.
Окна alerts намеренно делают эти два теста не мгновенными. Запускайте их
последовательно, а не параллельно.
## Проверка сохранности, lab2+
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
"""Воспроизводит ограниченные отказы worker и PostgreSQL для проверки alerts."""
import argparse
import base64
import json
import os
import subprocess
import time
import urllib.parse
from client import Client
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def compose(*args: str, capture: bool = False):
"""Запускает Docker Compose из корня проекта и при необходимости захватывает вывод."""
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
text=True,
capture_output=capture,
)
def grafana_rules() -> dict:
"""Получает текущие состояния provisioned alert rules через API Grafana."""
user = os.getenv("GRAFANA_ADMIN_USER", "admin")
password = os.getenv("GRAFANA_ADMIN_PASSWORD", "change-me-grafana")
token = base64.b64encode(f"{user}:{password}".encode()).decode()
result = compose(
"exec", "-T", "grafana", "wget", "-qO-",
"--header", f"Authorization: Basic {token}",
"http://127.0.0.1:3000/api/prometheus/grafana/api/v1/rules",
capture=True,
)
return json.loads(result.stdout)
def internal_json(url: str) -> dict:
"""Читает JSON из внутреннего observability-сервиса через Grafana-контейнер."""
result = compose(
"exec", "-T", "grafana", "wget", "-qO-", url, capture=True
)
return json.loads(result.stdout)
def database_failure_evidence(start_ns: int) -> dict[str, str]:
"""Находит связанный лог и ошибочный PostgreSQL CLIENT span."""
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
query = urllib.parse.urlencode({
"query": (
'{service_name="messenger-api"} | json '
'| logger="app.error" | message=~".*(database|socket).*"'
),
"start": str(start_ns),
"end": str(time.time_ns()),
"direction": "backward",
"limit": "20",
})
loki = internal_json(
"http://loki:3100/loki/api/v1/query_range?" + query
)
streams = loki.get("data", {}).get("result", [])
if not streams:
time.sleep(2)
continue
entry = streams[0]["values"][0]
metadata = entry[2] if len(entry) > 2 else {}
body = json.loads(entry[1])
trace_id = metadata.get("trace_id") or body.get("trace_id")
if not trace_id:
time.sleep(2)
continue
trace = internal_json(f"http://tempo:3200/api/traces/{trace_id}")
for batch in trace.get("batches", []):
scopes = batch.get(
"scopeSpans", batch.get("instrumentationLibrarySpans", [])
)
for scope in scopes:
for span in scope.get("spans", []):
attributes = {
item["key"]: next(iter(item.get("value", {}).values()), "")
for item in span.get("attributes", [])
}
if (
span.get("kind") == "SPAN_KIND_CLIENT"
and attributes.get("db.system") == "postgresql"
and span.get("status", {}).get("code") == "STATUS_CODE_ERROR"
):
return {
"trace_id": trace_id,
"log_message": body.get("message", ""),
"db_span": span.get("name", ""),
"db_system": "postgresql",
}
time.sleep(2)
raise RuntimeError("linked PostgreSQL error log/trace was not found")
def rule_state(title: str) -> str:
"""Находит alert по заголовку и возвращает его текущее состояние."""
groups = grafana_rules().get("data", {}).get("groups", [])
for group in groups:
for rule in group.get("rules", []):
if rule.get("name") == title:
alerts = rule.get("alerts", [])
return alerts[0].get("state", rule.get("state", "unknown")) if alerts else rule.get("state", "unknown")
raise RuntimeError(f"Grafana rule not found: {title}")
def wait_state(title: str, expected: set[str], timeout: int) -> tuple[str, float]:
"""Ожидает одно из требуемых состояний alert и возвращает время ожидания."""
started = time.monotonic()
deadline = started + timeout
last = "unknown"
while time.monotonic() < deadline:
last = rule_state(title)
if last.lower() in expected:
return last, time.monotonic() - started
time.sleep(5)
raise RuntimeError(f"{title}: expected {sorted(expected)}, last state {last}")
def worker_scenario(base_url: str):
"""Останавливает worker и проверяет переход queue alert в Alerting и Normal."""
client = Client(base_url, timeout=10)
alice, bob, _, chat = client.bootstrap()
compose("stop", "worker")
try:
attachment, _ = client.attachment(alice, chat)
firing, detect_seconds = wait_state(
"QueueOldestJobGrowing", {"alerting", "firing"}, 100
)
finally:
compose("start", "worker")
ready = client.wait_ready(bob, attachment, wait_seconds=90)
normal, recover_seconds = wait_state(
"QueueOldestJobGrowing", {"normal", "inactive"}, 90
)
print(json.dumps({
"scenario": "worker_outage",
"attachment_id": ready["id"],
"firing_state": firing,
"detect_seconds": round(detect_seconds, 1),
"resolved_state": normal,
"recover_seconds": round(recover_seconds, 1),
"result": "PASS",
}, ensure_ascii=False))
def database_scenario(base_url: str):
"""Останавливает PostgreSQL и проверяет firing/resolved для alert по 5xx."""
Client(base_url).call("GET", "/health/ready", schema="Health")
incident_started_ns = time.time_ns()
compose("stop", "postgres")
load = subprocess.Popen(
[
"docker", "compose", "-f", "compose.yaml", "-f",
"compose.observability-tests.yaml", "run", "--rm",
"observability-test", "python", "database_error_load.py",
"--base-url", "http://api:8080", "--duration", "150",
],
cwd=ROOT,
)
evidence = None
try:
firing, detect_seconds = wait_state(
"ApiHighServerErrorRate", {"alerting", "firing"}, 130
)
evidence = database_failure_evidence(incident_started_ns)
finally:
load.terminate()
try:
load.wait(timeout=10)
except subprocess.TimeoutExpired:
load.kill()
compose("start", "postgres")
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
Client(base_url).call("GET", "/health/ready", schema="Health")
break
except Exception:
time.sleep(3)
else:
raise RuntimeError("API readiness did not recover after PostgreSQL restart")
normal, recover_seconds = wait_state(
"ApiHighServerErrorRate", {"normal", "inactive"}, 180
)
print(json.dumps({
"scenario": "database_outage",
"firing_state": firing,
"detect_seconds": round(detect_seconds, 1),
"resolved_state": normal,
"recover_seconds": round(recover_seconds, 1),
"trace_id": evidence["trace_id"],
"log_message": evidence["log_message"],
"db_span": evidence["db_span"],
"db_system": evidence["db_system"],
"result": "PASS",
}, ensure_ascii=False))
def main():
"""Разбирает аргументы и запускает выбранный отказный сценарий."""
parser = argparse.ArgumentParser()
parser.add_argument("scenario", choices=("worker", "database"))
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
if args.scenario == "worker":
worker_scenario(args.base_url)
else:
database_scenario(args.base_url)
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Создаёт ограниченный поток DB-зависимых запросов для проверки alert по 5xx."""
import argparse
import json
import time
import urllib.error
import urllib.request
import uuid
def main():
"""Отправляет synthetic POST-запросы заданное время и считает ошибки."""
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://api:8080")
parser.add_argument("--duration", type=int, default=120)
args = parser.parse_args()
deadline = time.monotonic() + args.duration
attempts = errors = 0
while time.monotonic() < deadline:
suffix = uuid.uuid4().hex[:12]
body = json.dumps({
"username": "incident_" + suffix,
"display_name": "Synthetic incident",
"password": "Synthetic-test-only-" + suffix,
}).encode()
request = urllib.request.Request(
args.base_url.rstrip("/") + "/api/v1/users",
data=body,
method="POST",
headers={"Content-Type": "application/json"},
)
attempts += 1
try:
with urllib.request.urlopen(request, timeout=3) as response:
response.read()
errors += response.status >= 500
except urllib.error.HTTPError as error:
error.read()
errors += error.code >= 500
except (urllib.error.URLError, TimeoutError, OSError):
errors += 1
time.sleep(0.25)
print(f"database incident load: attempts={attempts} errors={errors}")
if __name__ == "__main__":
main()

Some files were not shown because too many files have changed in this diff Show More