Author SHA1 Message Date
k00kza c17f647c64 Обновить REPORT.md 2026-11-16 14:17:49 +03:00
k00kzaAntonovaIra 8bedaa7592 solution 2026-11-16 14:16:01 +03:00
177 changed files with 17065 additions and 430 deletions
+7
View File
@@ -18,3 +18,10 @@ __pycache__
node_modules
evidence
.DS_Store
docs
.agents
.aws
.codex
.pytest_cache
venv
+73 -12
View File
@@ -1,25 +1,86 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
APP_ENV=development
LOG_LEVEL=INFO
MAX_MESSAGES=100000
MAX_IN_FLIGHT=16
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
SESSION_STORE=redis
REDIS_HOST=redis
REDIS_PORT=6379
REDIS_DB=0
REDIS_MAX_CONNECTIONS=20
REDIS_POOL_TIMEOUT_SECONDS=1
REDIS_CONNECT_TIMEOUT_SECONDS=1
REDIS_SOCKET_TIMEOUT_SECONDS=0.5
DB_HOST=postgres
DB_PORT=5432
DB_NAME=messenger
DB_USER=messenger
DB_POOL_SIZE=5
DB_MAX_OVERFLOW=10
DB_POOL_TIMEOUT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
API_REPLICAS=2
API_IMAGE=messenger:local
API_BUILD_REVISION=development
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
SHUTDOWN_DRAIN_SECONDS=1
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5
READINESS_TIMEOUT_SECONDS=2
LB_HOST_PORT=18081
DB_HOST_PORT=25432
REDIS_HOST_PORT=26379
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
S3_CONNECT_TIMEOUT_SECONDS=2
S3_READ_TIMEOUT_SECONDS=10
S3_UPLOAD_TIMEOUT_SECONDS=30
S3_DOWNLOAD_TIMEOUT_SECONDS=60
S3_IMAGE_TIMEOUT_SECONDS=10
S3_MAX_UPLOAD_BYTES=10485760
S3_MAX_IMAGE_PIXELS=40000000
S3_CHUNK_BYTES=65536
S3_MAX_CONCURRENT_UPLOADS=2
S3_CLEANUP_GRACE_SECONDS=86400
S3_CLEANUP_INTERVAL_SECONDS=300
POSTGRES_PASSWORD_FILE=/run/secrets/db_password
MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
MINIO_BROWSER=off
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
BROKER_HOST=broker
BROKER_PORT=5672
BROKER_PASSWORD_FILE=/run/secrets/broker_password
BROKER_QUEUE=attachment-jobs
BROKER_USER=messenger
BROKER_VHOST=/
JOB_LEASE_SECONDS=90
JOB_RETRY_SECONDS=5
PUBLISHER_POLL_SECONDS=1
IMAGE_JPEG_QUALITY=85
IMAGE_WEBP_QUALITY=80
IMAGE_WEBP_METHOD=4
IMAGE_MEMORY_MIB=768
IMAGE_CPU_SECONDS=20
JOB_RETRY_MAX_SECONDS=120
FAILED_STAGING_RETENTION_SECONDS=604800
+7
View File
@@ -0,0 +1,7 @@
* text=auto
*.py text eol=lf
*.sh text eol=lf
*.conf text eol=lf
*.yaml text eol=lf
Dockerfile text eol=lf
*.png binary
+5 -4
View File
@@ -1,4 +1,3 @@
# Local/private state
.env
.env.*
!.env.example
@@ -13,7 +12,6 @@ certs/private/
volumes/
data/
backups/
# Runtime/build artifacts
.venv/
__pycache__/
*.py[cod]
@@ -28,5 +26,8 @@ coverage/
.DS_Store
.idea/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
venv/
.pytest_cache/
.ruff_cache/
/docs/
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.12.15-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
RUN useradd --system --uid 10001 --no-create-home appuser
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY alembic.ini ./
COPY migrations ./migrations
COPY app ./app
ARG API_BUILD_REVISION
LABEL org.opencontainers.image.revision=$API_BUILD_REVISION
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
CMD ["python", "-m", "app"]
+63 -19
View File
@@ -1,37 +1,81 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Антонова Ирина Андреевна, БПИ2301.
- Целевая оценка: 4
- Commit/PR: 8bedaa75920c24e2d625956a835e3bb592d8c5a8
- Использованные источники, библиотеки и помощь ИИ: Codex — перенос, очистка, контейнерный запуск проверок
- Стек: Python 3.12.15, FastAPI 0.143.3, SQLAlchemy 2.1.4, asyncpg 0.32.0, Pillow 13.0.0, aio-pika 10.2.2 (`requirements.txt`).
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Python 3.9+ и Docker с Compose v2 (Windows: Docker Desktop, Linux containers, privileged-контейнеры, интернет). Docker Desktop у меня на WSL2, в `%USERPROFILE%\.wslconfig` заданы `memory=10GB`, `processors=6` (после этого `wsl --shutdown` и перезапуск Docker).
Перед проверками остановите обычный стенд (`docker compose --env-file .env down`), затем:
```sh
python scripts/test.py
```
Хостовый Python только управляет Docker: тесты, миграции и black выполняются в Linux-контейнере Python 3.12.15 с вложенным Docker 29.1.3. Секреты и `.env` для этого не нужны. Первый прогон без кеша долгий и молчит: вывод пишется в `evidence/lab5/`. Блок `finally` удаляет временные контейнеры, сеть, тома и образ; общий кеш сборки может остаться.
Обычный запуск приложения:
```sh
python scripts/init_local.py
docker compose --env-file .env up -d --build
docker compose --env-file .env up -d --wait --wait-timeout 240 nginx
```
`init_local.py` создаёт `.env` и `secrets/` (в репозиторий и evidence они не попадают). HTTP: http://127.0.0.1:18081, остальные сервисы наружу не опубликованы. Миграции и bootstrap MinIO запускаются автоматически.
Остановка с сохранением данных: `docker compose --env-file .env down`.
Намеренное удаление учебных данных: `docker compose --env-file .env down --volumes --remove-orphans`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Клиент → Nginx → две API-реплики → PostgreSQL/Redis и приватный MinIO. API кладёт исходник в staging MinIO, затем в одной транзакции PostgreSQL записывает вложение и задание (и запись outbox), и только после этого отвечает `202 queued`. Источник истины — PostgreSQL. Publisher отправляет идентификатор задания в durable-очередь RabbitMQ с publisher confirm; байты и секреты в сообщение не попадают. Брокер не участвует в синхронном приёме, его отказ покрывается outbox.
Worker отдельный. Состояния: `queued → processing → ready/failed`. Ack отправляется после фиксации результата в PostgreSQL. Блокировка, lease 90 с и фиксированные ключи объектов защищают от дублей и от записи устаревшим worker: доставка at least once, результат идемпотентен.
Обработка: original побайтово; JPEG thumbnail ≤256×256 (quality 85), JPEG crop 128×128, WebP ≤1280×1280 (quality 80, method 4); EXIF-ориентация учитывается, EXIF/ICC/XMP удаляются. Лимиты: 10 MiB, 40 млн пикселей, 60 с, 768 MiB и 20 с CPU на дочерний процесс. Concurrency 2, prefetch 1.
Ошибки: постоянные сразу ведут к `failed`; временные повторяются до 5 раз с backoff и jitter (`c = min(120, 5·2^(n−1))`, задержка от `c/2` до `c` с). Failed-задания и аудит replay хранятся в PostgreSQL; replay проверяет исходник, выдаёт новый бюджет попыток и отвергает уже успешную job. Cleanup удаляет ready через сутки, failed через 7 суток, staging активных заданий сохраняет. Из lab4 сохранены аутентификация, чаты, история, идемпотентность сообщений и сессии Redis; изменён upload (201 → 202).
Операторские команды:
```sh
docker compose run --rm --no-deps worker python3 -m app.core.job_admin list-failed
docker compose run --rm --no-deps worker python3 -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
```
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Итог `python scripts/test.py`: **PASS: проверки завершены, временное окружение удалено**; все шаги `00_init`…`15_cleanup` с кодом 0. Среда: Linux x86_64, Docker 29.1.3, 6 CPU, 10 433 806 336 байт RAM; образы закреплены (postgres 16.10, redis 7.4.2, rabbitmq 4.4.0, nginx 1.28.0).
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда | Результат | Файл |
| --- | --- | --- | --- |
| Формат, контракт, OpenAPI | `black --check`; `check_contract.py`; `validate_openapi.py` | 131 файл без изменений; обе проверки PASS | `01`–`03_*.txt` |
| Стенд поднимается, миграции, bootstrap | `up -d --wait nginx` | все сервисы Healthy, `migrate` и `minio-bootstrap` завершились | `04_readiness.txt` |
| Ур. 3: smoke (202 → ready, original, thumbnail) | `tests/smoke.py` | PASS | `07_smoke.txt` |
| Ур. 3–4: регрессия, лимиты, retry, replay, lease, cleanup | `python3 -m pytest -q` | **201 passed**, 111,29 с | `06_pytest.txt` |
| Ур. 3: приватность S3, отказ MinIO | `scripts.check_s3 --with-outage` | PASS, 22,2 с | `08_storage.txt` |
| Ур. 3: worker остановлен → рестарт → ready | `scripts.check_jobs` (`worker_stopped_restart_all`) | PASS | `10_jobs.txt` |
| Ур. 3: сохранность после рестарта | `resilience.py --mode persistence` | данные и сессия сохранены, 50,8 с | `11_persistence.txt` |
| Ур. 4: crop, WebP | `check_outbox_recovery` | варианты `thumbnail`, `crop`, `optimized` | `09_outbox.txt` |
| Ур. 4: crash COMMIT → publish | `scripts.check_outbox_recovery` | SIGKILL publisher: `queued` → `ready`, без повторной загрузки | `09_outbox.txt` |
| Ур. 4: crash после PUT до ack | `check_jobs` | PASS `crash-after-object`, `crash-before-ack` | `10_jobs.txt` |
| Ур. 4: retry, постоянные/временные ошибки, replay | `check_jobs` | PASS `fail-transient`, `fail-permanent` | `10_jobs.txt` |
| Отказ одной API-реплики | `resilience.py --mode failover` | 120 успешных записей, 4 неуспешные попытки, восстановление 2,707 с | `12_failover.txt` |
| Версии, удаление ресурсов | `docker image ls`, `down --volumes` | код 0, ресурсы `messenger-test_*` удалены | `13_*`, `15_cleanup.txt` |
Что делают hooks. restart.sh: останавливает worker, publisher, cleanup; перезапускает postgres, redis, minio, broker (тома сохраняются) и ждёт их healthy; затем перезапускает api и nginx и запускает publisher, cleanup и worker (если он работал). stop-one.sh: находит одну API-реплику по TARGET_INSTANCE_ID (hostname или INSTANCE_ID) и останавливает её командой docker stop --timeout 8; вторая реплика продолжает работать. Вывод hooks в тестах подавляется, поэтому эффект подтверждён и результатами проверок.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Нагрузочные эксперименты (трек 5) не проводились. Внедрённые отказы описаны в таблице выше; потери подтверждённых данных ни в одном сценарии нет. Времена: outbox-сценарий 15,6 с, `check_jobs` 302,7 с, failover до первого успешного ответа 2,707 с.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения
## Ограничения и следующие шаги
Один Docker-хост и одиночные PostgreSQL, MinIO и брокер: потеря хоста или диска не покрывается репликами API. Admission control очереди не реализован, pipeline фиксирован на версии 1. Времена зависят от ресурсов Docker (у меня 6 CPU, ≈9,7 ГиБ).
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Первый прогон упал на `04_build_start`, пока на хосте параллельно работал обычный стенд; точная причина не устанавливалась. После остановки стенда и увеличения памяти Docker до ≈9,7 ГиБ прогон прошёл полностью, поэтому проверки нужно запускать при остановленном стенде.
+38
View File
@@ -0,0 +1,38 @@
[alembic]
script_location = migrations
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
View File
+4
View File
@@ -0,0 +1,4 @@
from app.main import main
if __name__ == "__main__":
main()
View File
+40
View File
@@ -0,0 +1,40 @@
"""Остановка фоновых процессов и безопасные сообщения об отказах."""
import asyncio
import logging
import signal
from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
from sqlalchemy.exc import InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
TRANSIENT_JOB_ERRORS = (
AMQPException,
ChannelInvalidStateError,
InterfaceError,
OperationalError,
PoolTimeoutError,
OSError,
)
logger = logging.getLogger(__name__)
def shutdown_event() -> asyncio.Event:
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for signum in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(signum, stop.set)
return stop
async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None:
try:
async with asyncio.timeout(seconds):
await stop.wait()
except TimeoutError:
pass
def log_background_error(operation: str, error: Exception) -> None:
logger.warning("%s: %s", operation, type(error).__name__)
+66
View File
@@ -0,0 +1,66 @@
"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис."""
import argparse
import asyncio
from dataclasses import asdict
import json
from app.core.config import Settings
from app.core.jobs import JobSettings
from app.core.database import create_db_engine, create_session_factory
from app.core.s3 import S3Settings
from app.domain.attachments import reconcile_attachments
from app.domain.attachment_jobs import cleanup_staging
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
from app.storage.s3 import create_s3_storage
async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool:
completed = True
try:
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
staging = await cleanup_staging(session, storage, jobs, apply=apply)
print(
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
return completed
async def run(once: bool, apply: bool) -> None:
settings = Settings()
s3_settings = S3Settings()
jobs = JobSettings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(s3_settings)
try:
while True:
completed = await cleanup_once(factory, storage, apply, jobs)
if once:
if not completed:
raise SystemExit(1)
break
await asyncio.sleep(s3_settings.cleanup_interval_seconds)
finally:
await storage.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
parser.add_argument("--once", action="store_true")
parser.add_argument(
"--apply",
action="store_true",
help="удалять безопасные кандидаты; по умолчанию dry-run",
)
args = parser.parse_args()
asyncio.run(run(args.once, args.apply))
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
from enum import StrEnum
from pathlib import Path
from typing import Literal, Self
from pydantic import Field, SecretStr, model_validator
from pydantic_settings import (
BaseSettings,
PydanticBaseSettingsSource,
SettingsConfigDict,
)
from sqlalchemy import URL
from app.core.s3 import S3Settings
ASYNC_DRIVER = "postgresql+asyncpg"
SECRETS_DIR = next(
(
path
for path in (
Path("/run/secrets"),
Path(__file__).resolve().parents[2] / "secrets",
)
if (path / "db_password").is_file()
),
None,
)
class LogLevel(StrEnum):
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
CRITICAL = "CRITICAL"
class Settings(BaseSettings):
s3_enabled: bool = True
s3: S3Settings = Field(default_factory=S3Settings)
app_host: str
app_port: int = Field(ge=1, le=65535)
app_env: str
log_level: LogLevel = LogLevel.INFO
max_messages: int = Field(ge=1)
max_in_flight: int = Field(ge=1)
session_ttl_seconds: int = Field(default=3600, ge=1)
session_store: Literal["redis"] = "redis"
redis_host: str = "localhost"
redis_port: int = Field(default=6379, ge=1, le=65535)
redis_db: int = Field(default=0, ge=0)
redis_password: SecretStr | None = None
redis_max_connections: int = Field(default=20, ge=1)
redis_pool_timeout_seconds: float = Field(default=1.0, gt=0)
redis_connect_timeout_seconds: float = Field(default=1.0, gt=0)
redis_socket_timeout_seconds: float = Field(default=0.5, gt=0)
database_url: SecretStr | None = None
db_host: str | None = None
db_port: int = Field(default=5432, ge=1, le=65535)
db_name: str | None = None
db_user: str | None = None
db_password: SecretStr | None = None
db_pool_size: int = Field(default=5, ge=1)
db_max_overflow: int = Field(default=5, ge=0)
db_pool_timeout_seconds: float = Field(default=3.0, gt=0)
db_connect_timeout_seconds: int = Field(default=3, ge=1)
db_statement_timeout_ms: int = Field(default=5000, ge=1)
instance_id: str | None = None
idempotency_ttl_seconds: int = Field(default=86400, ge=1)
shutdown_drain_seconds: float = Field(default=1, ge=0)
graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1)
readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5)
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore",
secrets_dir=SECRETS_DIR,
)
@classmethod
def settings_customise_sources(
cls,
settings_cls: type[BaseSettings],
init_settings: PydanticBaseSettingsSource,
env_settings: PydanticBaseSettingsSource,
dotenv_settings: PydanticBaseSettingsSource,
file_secret_settings: PydanticBaseSettingsSource,
) -> tuple[PydanticBaseSettingsSource, ...]:
return init_settings, file_secret_settings, env_settings, dotenv_settings
@model_validator(mode="after")
def _require_database_source(self) -> Self:
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
raise ValueError(
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
)
if self.database_url is not None:
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
return self
if not (self.db_host and self.db_name and self.db_user and self.db_password):
raise ValueError(
"Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password"
)
return self
def get_database_url(self) -> str:
if self.database_url is not None:
return self.database_url.get_secret_value()
assert self.db_password is not None
url = URL.create(
ASYNC_DRIVER,
username=self.db_user,
password=self.db_password.get_secret_value(),
host=self.db_host,
port=self.db_port,
database=self.db_name,
)
return url.render_as_string(hide_password=False)
+28
View File
@@ -0,0 +1,28 @@
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from app.core.config import Settings
def create_db_engine(settings: Settings) -> AsyncEngine:
return create_async_engine(
settings.get_database_url(),
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout_seconds,
pool_pre_ping=True,
connect_args={
"timeout": settings.db_connect_timeout_seconds,
"server_settings": {
"statement_timeout": str(settings.db_statement_timeout_ms)
},
},
)
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
+16
View File
@@ -0,0 +1,16 @@
"""Качество вариантов и пределы отдельного процесса обработки."""
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class ImageSettings(BaseSettings):
model_config = SettingsConfigDict(
env_prefix="IMAGE_", env_file=".env", extra="ignore"
)
jpeg_quality: int = Field(default=85, ge=1, le=95)
webp_quality: int = Field(default=80, ge=1, le=100)
webp_method: int = Field(default=4, ge=0, le=6)
memory_mib: int = Field(default=768, ge=256, le=1024)
cpu_seconds: int = Field(default=20, ge=1, le=120)
+24
View File
@@ -0,0 +1,24 @@
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
from pathlib import Path
import resource
import sys
from app.domain.image_validation import decode_image
from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code
def main() -> int:
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
path, content_type, maximum = sys.argv[1:]
status = 0
try:
decode_image(Path(path), content_type, int(maximum))
except IMAGE_WORKER_ERRORS as error:
status = image_worker_error_code(error)
return status
if __name__ == "__main__":
raise SystemExit(main())
+38
View File
@@ -0,0 +1,38 @@
"""Идентификатор API-реплики и заголовок X-Instance-Id."""
import os
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
INSTANCE_ID_HEADER = "X-Instance-Id"
def resolve_instance_id(configured: str | None) -> str:
if configured and configured.strip():
return configured.strip()
hostname = os.getenv("HOSTNAME", "").strip()
if hostname:
return hostname
return str(uuid4())
class InstanceIdMiddleware(BaseHTTPMiddleware):
def __init__(self, app, instance_id: str) -> None:
super().__init__(app)
self.instance_id = instance_id
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
response = await call_next(request)
response.headers[INSTANCE_ID_HEADER] = self.instance_id
return response
+82
View File
@@ -0,0 +1,82 @@
"""Операторский интерфейс failed registry. Доступ требует credentials стенда."""
import argparse
import asyncio
import json
from uuid import UUID
from app.core.background import log_background_error
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.job_admin import (
REPLAY_REASONS,
history,
list_failed,
replay_failed_job,
show_job,
)
from app.errors.exceptions import JobOperationError
from app.storage.s3 import create_s3_storage
async def execute(args):
settings = Settings()
engine = create_db_engine(settings)
storage = None
try:
factory = create_session_factory(engine)
async with factory() as session:
if args.command == "list-failed":
result = await list_failed(session, args.limit)
elif args.command == "show":
result = await show_job(session, args.job_id)
elif args.command == "history":
result = await history(session, args.job_id)
else:
storage = create_s3_storage(settings.s3)
result = await replay_failed_job(
session,
storage,
JobSettings(),
args.job_id,
request_id=args.request_id,
actor=args.actor,
reason=args.reason,
)
print(json.dumps(result, default=str), flush=True)
finally:
if storage is not None:
await storage.aclose()
await engine.dispose()
def main():
parser = argparse.ArgumentParser(
description="Реестр failed и явный replay с аудитом"
)
commands = parser.add_subparsers(dest="command", required=True)
listing = commands.add_parser("list-failed")
listing.add_argument(
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
)
for name in ("show", "history", "replay"):
command = commands.add_parser(name)
command.add_argument("job_id", type=UUID)
if name == "replay":
command.add_argument("--request-id", type=UUID, required=True)
command.add_argument("--actor", required=True)
command.add_argument("--reason", choices=REPLAY_REASONS, required=True)
args = parser.parse_args()
try:
asyncio.run(execute(args))
except JobOperationError as error:
print(json.dumps({"error_code": error.code}), flush=True)
raise SystemExit(2) from None
except Exception as error:
log_background_error("Job administration failed", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+169
View File
@@ -0,0 +1,169 @@
"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
import argparse
import asyncio
from dataclasses import dataclass
import json
import logging
import os
import signal
from uuid import UUID, uuid4
from PIL import features
from pydantic import ValidationError
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import process_attachment_job
from app.errors.exceptions import ProcessingError
from app.schemas.job import JobMessage
from app.storage.broker import JobBroker
from app.storage.s3 import create_s3_storage
@dataclass(frozen=True)
class Faults:
crash_after_object: UUID | None = None
crash_before_ack: UUID | None = None
fail_transient: UUID | None = None
fail_permanent: UUID | None = None
object_name: str = "optimized"
def kill(self, attachment_id: UUID, phase: str) -> None:
print(
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
def after_object(self, attachment_id: UUID, name: str) -> None:
if attachment_id == self.crash_after_object and name == self.object_name:
self.kill(attachment_id, "after_object:" + name)
def before_ack(self, attachment_id: UUID) -> None:
if attachment_id == self.crash_before_ack:
self.kill(attachment_id, "before_ack")
def before_processing(self, attachment_id: UUID) -> None:
if attachment_id == self.fail_transient:
raise TimeoutError("injected transient failure")
if attachment_id == self.fail_permanent:
raise ProcessingError("injected_permanent")
async def consume_connected(
factory,
storage,
broker: JobBroker,
settings: JobSettings,
stop: asyncio.Event,
faults: Faults,
) -> None:
consumer = uuid4().hex
while not stop.is_set():
try:
await broker.ensure_ready()
delivery = await broker.receive(consumer)
if delivery is None:
continue
try:
message = JobMessage.model_validate_json(delivery.payload)
except ValidationError:
await broker.reject_invalid(delivery.id)
continue
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session,
storage,
message,
settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
else:
if acknowledge:
faults.before_ack(message.attachment_id)
await broker.acknowledge_job(delivery.id)
else:
await broker.defer_job(delivery.id)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Worker delivery deferred", error)
await broker.aclose()
await wait_or_stop(stop, settings.publisher_poll_seconds)
async def consume(factory, storage, settings, stop, faults) -> None:
broker = JobBroker(settings)
try:
await consume_connected(factory, storage, broker, settings, stop, faults)
finally:
await broker.aclose()
async def run_worker(faults: Faults) -> None:
settings = Settings()
jobs = JobSettings()
logging.basicConfig(level=settings.log_level)
if not features.check("webp"):
logging.error("Pillow requires WebP support for the image worker")
raise RuntimeError("WebP codec is unavailable")
stop = shutdown_event()
engine = create_db_engine(settings)
storage = None
try:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [
group.create_task(consume(factory, storage, jobs, stop, faults))
for _ in range(jobs.worker_concurrency)
]
await stop.wait()
for task in tasks:
task.cancel()
finally:
if storage is not None:
await storage.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description="Image worker с явными fault-injection hooks"
)
modes = parser.add_mutually_exclusive_group()
for flag in (
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
):
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
parser.add_argument(
"--object-name",
choices=("original", "thumbnail", "crop", "optimized"),
default="optimized",
)
args = parser.parse_args()
faults = Faults(**vars(args))
try:
asyncio.run(run_worker(faults))
except Exception as error:
log_background_error("Worker stopped", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+37
View File
@@ -0,0 +1,37 @@
"""Настройки очереди из .env; credentials брокера читаются отдельно из файла."""
from pathlib import Path
from typing import Self
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class JobSettings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
broker_host: str = "broker"
broker_port: int = Field(default=5672, ge=1, le=65535)
broker_password_file: Path = Path("/run/secrets/broker_password")
broker_queue: str = "attachment-jobs"
broker_user: str = "messenger"
broker_vhost: str = "/"
worker_concurrency: int = Field(default=2, ge=1, le=4)
job_max_attempts: int = Field(default=5, ge=1, le=100)
job_timeout_seconds: int = Field(default=60, ge=10, le=300)
job_lease_seconds: int = Field(default=90, ge=20, le=600)
job_retry_seconds: int = Field(default=5, ge=1, le=300)
job_retry_max_seconds: int = Field(default=120, ge=1, le=3600)
failed_staging_retention_seconds: int = Field(default=604800, ge=300)
publisher_poll_seconds: float = Field(default=1, gt=0, le=10)
staging_retention_seconds: int = Field(default=86400, ge=300)
@model_validator(mode="after")
def validate_lease(self) -> Self:
if self.job_lease_seconds < self.job_timeout_seconds + 30:
raise ValueError(
"Lease должен превышать timeout задания минимум на 30 секунд"
)
if self.job_retry_max_seconds < self.job_retry_seconds:
raise ValueError("Retry maximum must be at least the base delay")
return self
+45
View File
@@ -0,0 +1,45 @@
"""Готовность реплики и мягкое завершение сервера."""
import asyncio
from types import FrameType
import uvicorn
class Lifecycle:
def __init__(self) -> None:
self._draining = False
@property
def draining(self) -> bool:
return self._draining
def start_draining(self) -> None:
self._draining = True
class GracefulServer(uvicorn.Server):
def __init__(
self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float
) -> None:
super().__init__(config)
self.lifecycle = lifecycle
self.drain_seconds = drain_seconds
self._drain_task: asyncio.Task[None] | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
if self._drain_task is not None:
if not self._drain_task.done():
self._drain_task.cancel()
super().handle_exit(sig, frame)
return
self.lifecycle.start_draining()
self._drain_task = asyncio.get_running_loop().create_task(
self._finish_draining(sig, frame)
)
async def _finish_draining(self, sig: int, frame: FrameType | None) -> None:
await asyncio.sleep(self.drain_seconds)
super().handle_exit(sig, frame)
+87
View File
@@ -0,0 +1,87 @@
"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI."""
import argparse
import asyncio
import json
import logging
import os
import signal
from uuid import UUID
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
from app.storage.broker import JobBroker
def crash_before_publish(attachment_id: UUID) -> None:
print(
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
settings = Settings()
jobs = JobSettings()
logging.basicConfig(level=settings.log_level)
stop = shutdown_event()
engine = create_db_engine(settings)
broker = None
try:
broker = JobBroker(jobs)
factory = create_session_factory(engine)
while not stop.is_set():
published = False
try:
await broker.ensure_ready()
async with factory() as session:
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session,
broker,
attachment_id=crash_attachment_id,
before_publish=(
crash_before_publish
if crash_attachment_id is not None
else None
),
)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Publisher deferred", error)
if not published:
await wait_or_stop(stop, jobs.publisher_poll_seconds)
finally:
if broker is not None:
await broker.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description="Публикация устойчивых jobs в RabbitMQ"
)
parser.add_argument(
"--crash-before-publish",
type=UUID,
metavar="ATTACHMENT_ID",
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
)
args = parser.parse_args()
try:
asyncio.run(run_publisher(args.crash_before_publish))
except Exception as error:
log_background_error("Publisher stopped", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+24
View File
@@ -0,0 +1,24 @@
from redis.asyncio import BlockingConnectionPool, Redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
)
pool = BlockingConnectionPool(
host=settings.redis_host,
port=settings.redis_port,
db=settings.redis_db,
password=password,
max_connections=settings.redis_max_connections,
timeout=settings.redis_pool_timeout_seconds,
socket_connect_timeout=settings.redis_connect_timeout_seconds,
socket_timeout=settings.redis_socket_timeout_seconds,
retry=Retry(NoBackoff(), retries=1),
decode_responses=True,
)
return Redis.from_pool(pool)
+42
View File
@@ -0,0 +1,42 @@
from collections.abc import AsyncIterator
from fastapi import Request
from app.errors.exceptions import rate_limited, unavailable
class RequestCapacity:
def __init__(self, max_in_flight: int) -> None:
self._max_in_flight = max_in_flight
self._in_flight = 0
@property
def in_flight(self) -> int:
return self._in_flight
def try_acquire(self) -> bool:
if self._in_flight >= self._max_in_flight:
return False
self._in_flight += 1
return True
def release(self) -> None:
self._in_flight -= 1
def create_request_capacity(max_in_flight: int) -> RequestCapacity:
return RequestCapacity(max_in_flight)
async def acquire_request_slot(request: Request) -> AsyncIterator[None]:
if request.app.state.lifecycle.draining:
raise unavailable("Реплика завершает работу")
capacity: RequestCapacity = request.app.state.request_capacity
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
+22
View File
@@ -0,0 +1,22 @@
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
REQUEST_ID_HEADER = "X-Request-Id"
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
request_id = str(uuid4())
request.state.request_id = request_id
response = await call_next(request)
response.headers[REQUEST_ID_HEADER] = request_id
return response
+47
View File
@@ -0,0 +1,47 @@
"""Конфигурация MinIO; значения credentials читаются только из файлов."""
from pathlib import Path
from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
bucket: str = "messenger-attachments"
access_key_id_file: Path = Path("/run/secrets/s3_access_key_id")
secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key")
connect_timeout_seconds: float = Field(default=2, gt=0, le=5)
read_timeout_seconds: float = Field(default=10, gt=0, le=15)
upload_timeout_seconds: float = Field(default=30, gt=0, le=60)
download_timeout_seconds: float = Field(default=60, gt=0, le=120)
image_timeout_seconds: float = Field(default=10, gt=0, le=15)
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144)
max_concurrent_uploads: int = Field(default=2, ge=1, le=2)
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (
url.scheme not in {"http", "https"}
or not url.hostname
or url.username
or url.password
or url.path not in {"", "/"}
or url.query
or url.fragment
):
raise ValueError(
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
)
return self
+56
View File
@@ -0,0 +1,56 @@
"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий."""
import asyncio
import hashlib
import os
import secrets
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
TOKEN_BYTES = 32
_hasher = PasswordHasher()
_executor = ThreadPoolExecutor(
max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)),
thread_name_prefix="argon2",
)
def _hash_password_sync(password: str) -> str:
return _hasher.hash(password)
def _verify_password_sync(password: str, password_hash: str) -> bool:
try:
return _hasher.verify(password_hash, password)
except (VerificationError, InvalidHashError):
return False
_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES))
async def hash_password(password: str) -> str:
loop = asyncio.get_running_loop()
return await loop.run_in_executor(_executor, _hash_password_sync, password)
async def verify_password(password: str, password_hash: str) -> bool:
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
_executor, _verify_password_sync, password, password_hash
)
async def dummy_verify_password(password: str) -> None:
await verify_password(password, _DUMMY_HASH)
def generate_token() -> str:
return secrets.token_urlsafe(TOKEN_BYTES)
def hash_token(token: str) -> str:
return hashlib.sha256(token.encode()).hexdigest()
+32
View File
@@ -0,0 +1,32 @@
"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU."""
from pathlib import Path
import resource
import sys
from app.core.image_processing import ImageSettings
from app.domain.image_pipeline import create_variants
from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code
def main() -> int:
settings = ImageSettings()
memory = settings.memory_mib * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
)
source, directory, metadata, content_type, maximum = sys.argv[1:]
code = 0
try:
result = create_variants(
Path(source), Path(directory), content_type, int(maximum), settings
)
Path(metadata).write_text(result.model_dump_json())
except IMAGE_WORKER_ERRORS as error:
code = pipeline_worker_error_code(error)
return code
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -0,0 +1 @@
"""Бизнес-логика: проверки доступа, транзакции и ошибки API."""
+260
View File
@@ -0,0 +1,260 @@
"""Приём, durable публикация, попытки и фиксация результата обработки."""
import asyncio
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from minio.error import MinioException
from sqlalchemy.ext.asyncio import AsyncSession
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.jobs import JobSettings
from app.domain.job_retry import classify_failure, retry_delay
from app.domain.job_files import fetch_original, render_variants
from app.errors.exceptions import ProcessingError
from app.schemas.attachment import VARIANT_FILENAMES
from app.schemas.job import JobMessage
from app.storage import jobs, outbox
from app.storage.broker import JobBroker
from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class Claim:
work: JobWork | None = None
acknowledge: bool = False
async def enqueue_attachment_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
return await jobs.create_job(session, attachment_id, staging_key)
async def publish_next_job(
session: AsyncSession,
broker: JobBroker,
*,
attachment_id: UUID | None = None,
before_publish: Callable[[UUID], None] | None = None,
) -> bool:
job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
published = job is not None
if job is not None:
if before_publish is not None:
before_publish(job.attachment_id)
await broker.publish_job(
JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
)
outbox.mark_published(job)
await session.commit()
return published
async def claim_job(
session: AsyncSession, message: JobMessage, settings: JobSettings
) -> Claim:
row = await jobs.lock_job(session, message.job_id)
result = Claim()
if row is None:
raise ProcessingError("unknown_job")
job, attachment = row
if (
message.attachment_id != attachment.id
or message.pipeline_version != job.pipeline_version
):
raise ProcessingError("invalid_job")
now = await jobs.database_time(session)
if job.attempt_limit is None:
job.attempt_limit = settings.job_max_attempts
if attachment.status in {"ready", "failed"}:
result = Claim(acknowledge=True)
elif message.attempt <= job.attempt and attachment.status == "queued":
result = Claim(acknowledge=True)
elif message.attempt < job.attempt:
result = Claim(acknowledge=True)
elif message.attempt > job.attempt + 1:
raise ProcessingError("invalid_attempt")
elif attachment.status == "queued" and job.next_attempt_at <= now:
if (
job.attempt >= (job.attempt_limit or settings.job_max_attempts)
or job.staging_key is None
):
jobs.record_failure(
job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
)
result = Claim(acknowledge=True)
else:
result = Claim(
work=jobs.start_job(
job, attachment, uuid4(), now, settings.job_lease_seconds
)
)
await session.commit()
return result
async def save_failure(
session: AsyncSession,
work: JobWork,
settings: JobSettings,
code: str,
*,
permanent: bool,
) -> None:
await session.rollback()
row = await jobs.lock_job(session, work.job_id)
if row is not None:
job, attachment = row
if attachment.status == "processing" and job.lease_token == work.lease_token:
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job,
attachment,
await jobs.database_time(session),
"retry_exhausted" if exhausted and not permanent else code,
terminal=permanent or exhausted,
retry_seconds=retry_delay(
settings, job.attempt - job.cycle_start_attempt
),
)
job.last_error_code = code
job.last_error_permanent = permanent
await session.commit()
async def build_result(
session: AsyncSession,
storage: MinioStorage,
work: JobWork,
after_object: Callable[[UUID, str], None] | None = None,
) -> bool:
with TemporaryDirectory(prefix="messenger-job-") as directory:
source = Path(directory) / "original"
await fetch_original(storage, work, source)
result = await render_variants(storage, work, source, Path(directory))
row = await jobs.lock_job(session, work.job_id)
if row is None:
await session.rollback()
return True
job, attachment = row
now = await jobs.database_time(session)
if attachment.status != "processing" or job.lease_token != work.lease_token:
await session.rollback()
return True
if job.lease_until is None or job.lease_until <= now:
await session.rollback()
return False
await storage.put_file(
work.original_key,
source,
size_bytes=work.size_bytes,
content_type=work.content_type,
sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, "original")
for variant in result.variants:
await storage.put_file(
work.variant_key(variant.name),
Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes,
content_type=variant.content_type,
sha256=variant.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
jobs.complete_job(
job,
attachment,
work,
[variant.model_dump() for variant in result.variants],
await jobs.database_time(session),
)
await session.commit()
return True
async def process_attachment_job(
session: AsyncSession,
storage: MinioStorage,
message: JobMessage,
settings: JobSettings,
*,
after_object: Callable[[UUID, str], None] | None = None,
before_processing: Callable[[UUID], None] | None = None,
) -> bool:
claim = await claim_job(session, message, settings)
acknowledge = claim.acknowledge
if claim.work is not None:
work = claim.work
try:
async with asyncio.timeout(settings.job_timeout_seconds):
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object)
except (
ProcessingError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
acknowledge = True
return acknowledge
async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int:
rows = await jobs.expired_jobs(session)
now = await jobs.database_time(session)
for job, attachment in rows:
if job.attempt_limit is None:
job.attempt_limit = settings.job_max_attempts
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job,
attachment,
now,
"retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
)
job.last_error_code = "worker_timeout"
job.last_error_permanent = False
await session.commit()
return len(rows)
async def cleanup_staging(
session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
) -> int:
rows = await jobs.terminal_staging(
session,
settings.staging_retention_seconds,
settings.failed_staging_retention_seconds,
)
if apply:
for job, attachment in rows:
if job.staging_key != attachment.object_key:
await storage.delete_object(job.staging_key)
jobs.clear_staging(job)
await session.commit()
else:
await session.rollback()
return len(rows)
+337
View File
@@ -0,0 +1,337 @@
"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
from collections.abc import AsyncIterator, Iterator
from contextlib import contextmanager, suppress
from dataclasses import dataclass
import hashlib
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from anyio import CancelScope
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.request_capacity import RequestCapacity
from app.domain.chats import ensure_chat_member
from app.domain.attachment_jobs import enqueue_attachment_job
from app.domain.images import validate_filename, validate_image
from app.errors.exceptions import (
conflict,
invalid_request,
not_found,
payload_too_large,
rate_limited,
unavailable,
unsupported_media_type,
)
from app.errors.handlers import (
RECOVERABLE_STORAGE_ERRORS,
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment, VARIANT_FILENAMES
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class CleanupResult:
examined: int
candidates: int
deleted: int
@dataclass(frozen=True)
class OriginalFile:
path: Path
size_bytes: int
sha256: str
def require_storage(storage: MinioStorage | None) -> MinioStorage:
if storage is None:
raise unavailable("Объектное хранилище не настроено")
return storage
def validate_declared_size(value: str | None, maximum: int) -> None:
if value is not None:
if not value.isascii() or not value.isdecimal() or len(value) > 20:
raise invalid_request("Некорректный Content-Length")
if int(value) > maximum:
raise payload_too_large()
@contextmanager
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
async def receive_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
chat_id: UUID,
*,
capacity: RequestCapacity,
filename: str,
content_type: str,
declared_length: str | None,
chunks: AsyncIterator[bytes],
) -> Attachment:
storage = require_storage(storage)
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
timeout_error = invalid_request("Истекло время загрузки")
with upload_slot(capacity):
async with operation_timeout(
storage.settings.upload_timeout_seconds, timeout_error
):
result = await upload_attachment(
session,
storage,
user,
chat_id,
filename=filename,
content_type=content_type,
chunks=chunks,
)
return result
async def read_original(
chunks: AsyncIterator[bytes],
path: Path,
maximum: int,
chunk_bytes: int,
) -> OriginalFile:
checksum = hashlib.sha256()
size = 0
with path.open("wb") as target:
async for chunk in chunks:
size += len(chunk)
if size > maximum:
raise payload_too_large()
checksum.update(chunk)
view = memoryview(chunk)
for offset in range(0, len(view), chunk_bytes):
target.write(view[offset : offset + chunk_bytes])
if size == 0:
raise unsupported_media_type("Пустой файл")
return OriginalFile(path, size, checksum.hexdigest())
async def compensate_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
await session.rollback()
intent = await repository.lock_upload(session, id)
if intent is not None:
if await repository.get_by_key(session, intent.object_key) is None:
await storage.delete_object(intent.object_key)
await repository.finish_upload(session, id)
await session.commit()
async def recover_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
try:
await compensate_upload(session, storage, id)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Upload compensation", error)
with suppress(*RECOVERABLE_STORAGE_ERRORS):
await session.rollback()
async def stage_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
original: OriginalFile,
filename: str,
content_type: str,
) -> Attachment:
id = uuid4()
key = f"staging/{uuid4().hex}"
await repository.reserve_upload(session, id, key)
await session.commit()
try:
if await repository.lock_upload(session, id) is None:
raise unavailable("Загрузка отменена до записи оригинала")
await storage.put_file(
key,
original.path,
size_bytes=original.size_bytes,
content_type=content_type,
sha256=original.sha256,
)
record = await repository.create_attachment(
session,
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=f"originals/{id.hex}/v1",
status="queued",
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await enqueue_attachment_job(session, id, key)
await repository.finish_upload(session, id)
result = Attachment.model_validate(record)
await session.commit()
except BaseException:
with CancelScope(shield=True):
await recover_upload(session, storage, id)
raise
return result
async def upload_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
filename: str,
content_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
await ensure_chat_member(session, user, chat_id)
validate_filename(filename)
if content_type not in {"image/png", "image/jpeg"}:
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
await session.rollback()
settings = storage.settings
with TemporaryDirectory(prefix="messenger-upload-") as directory:
original = await read_original(
chunks,
Path(directory) / "original",
settings.max_upload_bytes,
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await stage_attachment(
session,
storage,
user,
chat_id,
original=original,
filename=filename,
content_type=content_type,
)
return result
async def get_attachment(
session: AsyncSession, user: User, attachment_id: UUID
) -> Attachment:
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
await ensure_chat_member(session, user, record.chat_id)
return Attachment.model_validate(record)
@dataclass(frozen=True)
class DownloadInfo:
filename: str
content_type: str
size_bytes: int
async def download_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[DownloadInfo, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if result.status != "ready":
raise conflict("Вложение пока недоступно для скачивания")
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
if variant == "original":
key = record.object_key
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
else:
selected = next(
(item for item in result.variants if item.name == variant), None
)
keys = {
"thumbnail": record.thumbnail_key,
"crop": record.crop_key,
"optimized": record.optimized_key,
}
key = keys.get(variant)
if selected is None or key is None:
raise not_found()
info = DownloadInfo(
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
)
await session.rollback()
stored = await storage.stat_object(key)
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
raise unavailable("Файл временно недоступен")
stream = await storage.open_object(key)
return info, stream
async def validate_message_attachments(
session: AsyncSession,
user: User,
chat_id: UUID,
attachment_ids: list[UUID],
) -> None:
await ensure_chat_member(session, user, chat_id)
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status == "failed" for record in records):
raise conflict("Обработка вложения завершилась ошибкой")
async def reconcile_attachments(
session: AsyncSession,
storage: MinioStorage,
*,
batch_size: int = 20,
dry_run: bool = True,
) -> CleanupResult:
if not 1 <= batch_size <= 100:
raise invalid_request("batch_size должен быть от 1 до 100")
intents = await repository.stale_uploads(
session, storage.settings.cleanup_grace_seconds, batch_size
)
candidates = deleted = 0
for intent in intents:
if await repository.get_by_key(session, intent.object_key) is None:
candidates += 1
if not dry_run:
await storage.delete_object(intent.object_key)
deleted += 1
if not dry_run:
await repository.finish_upload(session, intent.id)
if dry_run:
await session.rollback()
else:
await session.commit()
return CleanupResult(len(intents), candidates, deleted)
+81
View File
@@ -0,0 +1,81 @@
import re
from datetime import UTC, datetime, timedelta
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.domain.users import to_user_schema
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.session import AuthSession
from app.schemas.user import (
PASSWORD_MAX_LENGTH,
PASSWORD_MIN_LENGTH,
USERNAME_PATTERN,
User,
)
from app.storage import sessions as sessions_storage
from app.storage import users as users_storage
USERNAME_RE = re.compile(USERNAME_PATTERN)
async def create_session(
session: AsyncSession,
redis: Redis,
username: str,
password: str,
ttl_seconds: int,
) -> AuthSession:
if (
USERNAME_RE.fullmatch(username) is None
or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH
or not password.strip()
):
raise unauthorized_basic()
user = await users_storage.get_user_by_username(session, username)
await session.commit()
if user is None:
await security.dummy_verify_password(password)
raise unauthorized_basic()
if not await security.verify_password(password, user.password_hash):
raise unauthorized_basic()
token = security.generate_token()
expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
await sessions_storage.create_session(
redis,
token_hash=security.hash_token(token),
user_id=user.id,
expires_at=expires_at,
)
auth_session = AuthSession(
token=token, expires_at=expires_at, user=to_user_schema(user)
)
return auth_session
async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User:
user_id = await sessions_storage.get_session_user_id(
redis, security.hash_token(token)
)
if user_id is None:
raise unauthorized_bearer()
user = await users_storage.get_user_by_id(session, user_id)
if user is None:
raise unauthorized_bearer()
current_user = to_user_schema(user)
return current_user
async def revoke_session(redis: Redis, token: str) -> None:
is_deleted = await sessions_storage.delete_session(
redis, security.hash_token(token)
)
if not is_deleted:
raise unauthorized_bearer()
+98
View File
@@ -0,0 +1,98 @@
from uuid import UUID
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.pagination import (
KIND_CHATS,
build_next_cursor,
decode_cursor,
split_page,
)
from app.errors.exceptions import invalid_request, not_found
from app.models.chat import Chat as ChatModel
from app.schemas.chat import Chat, ChatPage, CreateChat
from app.schemas.user import User
from app.storage import chats as chats_storage
from app.storage import users as users_storage
from app.storage.types import Position
UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя"
def to_chat_schema(chat: ChatModel) -> Chat:
chat_schema = Chat(
id=chat.id,
title=chat.title,
member_ids=[member.user_id for member in chat.members],
created_at=chat.created_at,
)
return chat_schema
def validate_new_chat(creator: User, new_chat: CreateChat) -> None:
member_ids = new_chat.member_ids
if creator.id not in member_ids:
raise invalid_request("Создатель чата должен входить в member_ids")
if len(set(member_ids)) != len(member_ids):
raise invalid_request("member_ids содержит дубликаты")
async def create_chat(
session: AsyncSession, creator: User, new_chat: CreateChat
) -> Chat:
validate_new_chat(creator, new_chat)
existing_ids = await users_storage.get_existing_user_ids(
session, new_chat.member_ids
)
if existing_ids != set(new_chat.member_ids):
raise invalid_request(UNKNOWN_MEMBER_MESSAGE)
try:
chat = await chats_storage.create_chat(
session, title=new_chat.title, member_ids=new_chat.member_ids
)
await session.commit()
except IntegrityError:
await session.rollback()
raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None
created_chat = to_chat_schema(chat)
return created_chat
async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat:
chat = await chats_storage.get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise not_found()
found_chat = to_chat_schema(chat)
return found_chat
async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None:
if not await chats_storage.is_chat_member(session, chat_id, user.id):
raise not_found()
async def list_chats(
session: AsyncSession,
user: User,
cursor: str | None,
limit: int,
) -> ChatPage:
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_CHATS, user.id)
rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
items = [to_chat_schema(chat) for chat in page_rows]
page = ChatPage(
items=items,
next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next),
)
return page
+36
View File
@@ -0,0 +1,36 @@
"""Условия готовности реплики и ограниченная проверка зависимостей."""
import asyncio
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncEngine
from app.core.lifecycle import Lifecycle
from app.errors.exceptions import unavailable
from app.errors.handlers import operation_timeout
from app.schemas.base import Health
from app.storage.health import check_database, check_redis
from app.storage.s3 import MinioStorage
def liveness() -> Health:
return Health(status="ok")
async def readiness(
engine: AsyncEngine,
redis: Redis,
storage: MinioStorage | None,
lifecycle: Lifecycle,
timeout_seconds: float,
) -> Health:
if lifecycle.draining:
raise unavailable()
checks = [check_database(engine), check_redis(redis)]
if storage is not None:
checks.append(storage.check_available())
async with operation_timeout(timeout_seconds, unavailable()):
dependencies = await asyncio.gather(*checks, return_exceptions=True)
if not all(result is True for result in dependencies):
raise unavailable()
return Health(status="ok")
+75
View File
@@ -0,0 +1,75 @@
"""Атомарная отправка сообщения с защитой от повторов."""
import hashlib
import json
import re
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.attachments import validate_message_attachments
from app.errors.exceptions import conflict, invalid_request, unavailable
from app.schemas.message import CreateMessage, Message
from app.schemas.user import User
from app.storage import idempotency as key_storage
from app.storage import messages as message_storage
from app.storage import attachments as attachment_storage
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def validate_idempotency_key(key: str | None) -> str:
if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None:
raise invalid_request("Требуется корректный Idempotency-Key")
return key
def request_fingerprint(new_message: CreateMessage) -> str:
body = new_message.model_dump(mode="json")
if not body["attachment_ids"]:
body.pop("attachment_ids")
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
return hashlib.sha256(canonical.encode()).hexdigest()
async def send_message_idempotent(
session: AsyncSession,
sender: User,
chat_id: UUID,
new_message: CreateMessage,
idempotency_key: str,
max_messages: int,
ttl_seconds: int,
) -> Message:
await ensure_chat_member(session, sender, chat_id)
scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key)
fingerprint = request_fingerprint(new_message)
await key_storage.lock_key(session, scope)
previous = await key_storage.get_active_key(session, scope, ttl_seconds)
if previous is not None:
if previous.request_hash != fingerprint:
raise conflict("Idempotency-Key уже использован для другого сообщения")
message = await message_storage.get_message(session, previous.message_id)
else:
await validate_message_attachments(
session, sender, chat_id, new_message.attachment_ids
)
if not await message_storage.reserve_message_slot(session, max_messages):
raise unavailable("Достигнут лимит количества сообщений")
message = await message_storage.create_message(
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
)
await attachment_storage.bind_attachments(
session, message.id, new_message.attachment_ids
)
await key_storage.save_result(session, scope, fingerprint, message.id)
links = await attachment_storage.message_attachment_ids(session, [message.id])
result = Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]}
)
await session.commit()
return result
+101
View File
@@ -0,0 +1,101 @@
"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка."""
import hashlib
from pathlib import Path
from PIL import Image, ImageOps
from app.core.image_processing import ImageSettings
from app.domain.image_validation import decode_image
from app.schemas.attachment import PipelineResult, Variant
def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
longest = max(size)
result = size
if longest > maximum:
result = (
max(1, size[0] * maximum // longest),
max(1, size[1] * maximum // longest),
)
return result
def describe_variant(
name: str, content_type: str, path: Path, size: tuple[int, int]
) -> Variant:
checksum = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(65536), b""):
checksum.update(chunk)
return Variant(
name=name,
content_type=content_type,
size_bytes=path.stat().st_size,
width=size[0],
height=size[1],
sha256=checksum.hexdigest(),
)
def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None:
with Image.new("RGB", image.size, "white") as clean:
with image.getchannel("A") as alpha:
clean.paste(image, mask=alpha)
clean.save(destination, format="JPEG", quality=quality, exif=b"")
def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant:
size = fitted_size(image.size, 256)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
save_jpeg(resized, destination, quality)
return describe_variant("thumbnail", "image/jpeg", destination, size)
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
with ImageOps.fit(
image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
) as crop:
save_jpeg(crop, destination, quality)
return describe_variant("crop", "image/jpeg", destination, (128, 128))
def create_optimized(
image: Image.Image, destination: Path, settings: ImageSettings
) -> Variant:
size = fitted_size(image.size, 1280)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
with Image.new("RGBA", size) as clean:
clean.paste(resized)
clean.save(
destination,
format="WEBP",
lossless=False,
quality=settings.webp_quality,
method=settings.webp_method,
exif=b"",
icc_profile=b"",
xmp=b"",
)
return describe_variant("optimized", "image/webp", destination, size)
def create_variants(
source: Path,
directory: Path,
content_type: str,
max_pixels: int,
settings: ImageSettings,
) -> PipelineResult:
decode_image(source, content_type, max_pixels)
with Image.open(source, formats=["PNG", "JPEG"]) as image:
ImageOps.exif_transpose(image, in_place=True)
with image.convert("RGBA") as pixels:
variants = [
create_thumbnail(
pixels, directory / "thumbnail.jpg", settings.jpeg_quality
),
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
create_optimized(pixels, directory / "optimized.webp", settings),
]
return PipelineResult(variants=variants)
+23
View File
@@ -0,0 +1,23 @@
"""Синхронная проверка изображения; вызывается только из ограниченного процесса."""
from pathlib import Path
import warnings
from PIL import Image
from app.errors.exceptions import payload_too_large, unsupported_media_type
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
Image.MAX_IMAGE_PIXELS = max_pixels
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(path, formats=["PNG", "JPEG"]) as image:
if image.width * image.height > max_pixels:
raise payload_too_large("Превышен лимит пикселей изображения")
image_format = image.format
if image_format is None or Image.MIME.get(image_format) != content_type:
raise unsupported_media_type("Содержимое не соответствует Content-Type")
image.verify()
with Image.open(path, formats=["PNG", "JPEG"]) as image:
image.load()
+48
View File
@@ -0,0 +1,48 @@
"""Проверка имени и запуск ограниченного декодера."""
import asyncio
from pathlib import Path
import sys
import unicodedata
from anyio import CancelScope
from app.core.s3 import S3Settings
from app.errors.exceptions import (
image_validation_error,
invalid_request,
payload_too_large,
)
from app.errors.handlers import operation_timeout
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or any(
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
for char in filename
):
raise invalid_request("Недопустимое имя файла")
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.core.image_worker",
str(path),
content_type,
str(settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
error = payload_too_large("Превышен бюджет проверки изображения")
try:
async with operation_timeout(settings.image_timeout_seconds, error):
code = await process.wait()
finally:
if process.returncode is None:
process.kill()
with CancelScope(shield=True):
await process.wait()
if code != 0:
raise image_validation_error(code)
+157
View File
@@ -0,0 +1,157 @@
"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
import asyncio
import re
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import uuid4
from minio.error import S3Error
from app.domain.job_files import fetch_original
from app.errors.exceptions import JobOperationError, ProcessingError
from app.storage import job_admin, jobs
from app.storage.jobs import JobWork
REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
def snapshot(job, attachment):
return dict(
job_id=str(job.id),
attachment_id=str(attachment.id),
status=attachment.status,
attempt=job.attempt,
attempt_limit=job.attempt_limit,
cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code,
last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent,
staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at,
lease_until=job.lease_until,
finished_at=job.finished_at,
)
async def list_failed(session, limit):
return [
snapshot(job, attachment)
for job, attachment in await job_admin.failed_jobs(session, limit)
]
async def show_job(session, job_id):
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
result = snapshot(*row)
await session.rollback()
return result
async def history(session, job_id):
return [
dict(
request_id=str(row.id),
actor=row.actor,
reason=row.reason,
previous_attempt=row.previous_attempt,
attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code,
created_at=row.created_at,
)
for row in await job_admin.replay_history(session, job_id)
]
def same_request(record, job_id, actor, reason):
if record.job_id != job_id or record.actor != actor or record.reason != reason:
raise JobOperationError("request_conflict")
return {
"request_id": str(record.id),
"job_id": str(job_id),
"already_applied": True,
}
def validate_replay_request(actor: str, reason: str) -> None:
if (
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
or reason not in REPLAY_REASONS
):
raise JobOperationError("invalid_audit_fields")
def validate_replay_job(job, attachment) -> None:
if attachment.status != "failed":
raise JobOperationError("job_not_failed")
if job.staging_key is None:
raise JobOperationError("source_expired")
async def verify_replay_source(storage, settings, job, attachment) -> None:
work = JobWork(
job.id,
attachment.id,
uuid4(),
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
try:
async with asyncio.timeout(settings.job_timeout_seconds):
with TemporaryDirectory(prefix="replay-source-") as directory:
await fetch_original(storage, work, Path(directory) / "original")
except ProcessingError as error:
raise JobOperationError(error.code) from None
except S3Error as error:
if error.code in {"NoSuchKey", "NoSuchObject"}:
raise JobOperationError("source_missing") from None
raise
async def replay_failed_job(
session, storage, settings, job_id, *, request_id, actor, reason
):
validate_replay_request(actor, reason)
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
record = await job_admin.replay_record(session, request_id)
if record is not None:
result = same_request(record, job_id, actor, reason)
await session.rollback()
return result
job, attachment = row
validate_replay_job(job, attachment)
await verify_replay_source(storage, settings, job, attachment)
limit = job.attempt + settings.job_max_attempts
inserted = await job_admin.insert_replay(
session,
id=request_id,
job_id=job_id,
actor=actor,
reason=reason,
previous_attempt=job.attempt,
attempt_limit=limit,
previous_error_code=job.last_error_code,
)
if not inserted:
result = same_request(
await job_admin.replay_record(session, request_id), job_id, actor, reason
)
await session.rollback()
return result
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
await session.commit()
return {
"request_id": str(request_id),
"job_id": str(job_id),
"already_applied": False,
"status": "queued",
"attempt_limit": limit,
}
+72
View File
@@ -0,0 +1,72 @@
"""Получение исходника и запуск изолированного процесса обработки вариантов."""
import asyncio
import errno
import hashlib
from pathlib import Path
import sys
from anyio import CancelScope
from app.errors.exceptions import ProcessingError
from app.schemas.attachment import PipelineResult
from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
async def fetch_original(
storage: MinioStorage, work: JobWork, destination: Path
) -> None:
if work.size_bytes > storage.settings.max_upload_bytes:
raise ProcessingError("image_limits")
checksum = hashlib.sha256()
size = 0
stream = await storage.open_object(work.staging_key)
try:
with destination.open("wb") as target:
async for chunk in stream:
size += len(chunk)
if size > work.size_bytes:
raise ProcessingError("source_mismatch")
target.write(chunk)
checksum.update(chunk)
finally:
await stream.aclose()
if size != work.size_bytes or checksum.hexdigest() != work.sha256:
raise ProcessingError("source_mismatch")
async def render_variants(
storage: MinioStorage, work: JobWork, source: Path, directory: Path
) -> PipelineResult:
metadata = directory / "variants.json"
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.core.thumbnail_worker",
str(source),
str(directory),
str(metadata),
work.content_type,
str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
try:
code = await process.wait()
finally:
if process.returncode is None:
process.kill()
with CancelScope(shield=True):
await process.wait()
if code == 2:
raise ProcessingError("invalid_image")
if code == 3 or code < 0:
raise ProcessingError("image_limits")
if code == 4:
raise OSError(errno.EIO, "Pipeline filesystem unavailable")
if code == 5:
raise ProcessingError("processing_configuration")
if code != 0:
raise RuntimeError("Image pipeline subprocess failed")
return PipelineResult.model_validate_json(metadata.read_text())
+51
View File
@@ -0,0 +1,51 @@
"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой."""
import errno
import random
from minio.error import S3Error
from app.core.jobs import JobSettings
from app.errors.exceptions import ProcessingError
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
ceiling = min(
settings.job_retry_max_seconds,
settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
)
return random.uniform(ceiling / 2, ceiling)
def classify_failure(error: Exception) -> tuple[str, bool]:
if isinstance(error, ProcessingError):
return error.code, True
if isinstance(error, S3Error):
if error.code in {"NoSuchKey", "NoSuchObject"}:
return "source_missing", True
if error.code in {
"AccessDenied",
"InvalidAccessKeyId",
"SignatureDoesNotMatch",
"NoSuchBucket",
}:
return "storage_configuration", True
if error.code in {
"InternalError",
"SlowDown",
"ServiceUnavailable",
"RequestTimeout",
"Throttling",
}:
return "storage_unavailable", False
return "storage_rejected", True
if isinstance(error, TimeoutError):
return "processing_timeout", False
if isinstance(error, OSError) and error.errno in {
errno.EACCES,
errno.EPERM,
errno.EROFS,
errno.ENOENT,
}:
return "processing_configuration", True
return "processing_unavailable", False
+47
View File
@@ -0,0 +1,47 @@
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.pagination import (
KIND_MESSAGES,
build_next_cursor,
decode_cursor,
split_page,
)
from app.schemas.message import Message, MessagePage
from app.schemas.user import User
from app.storage import messages as messages_storage
from app.storage import attachments as attachments_storage
from app.storage.types import Position
async def list_messages(
session: AsyncSession,
user: User,
chat_id: UUID,
cursor: str | None,
limit: int,
) -> MessagePage:
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_MESSAGES, chat_id)
await ensure_chat_member(session, user, chat_id)
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
links = await attachments_storage.message_attachment_ids(
session, [message.id for message in page_rows]
)
items = [
Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
)
for message in page_rows
]
page = MessagePage(
items=items,
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
)
return page
+78
View File
@@ -0,0 +1,78 @@
import base64
import json
from collections.abc import Sequence
from datetime import datetime
from typing import Protocol, TypeVar
from uuid import UUID
from app.errors.exceptions import invalid_request
from app.storage.types import Position
KIND_CHATS = "chats"
KIND_MESSAGES = "messages"
T = TypeVar("T")
class Positioned(Protocol):
@property
def id(self) -> UUID: ...
@property
def created_at(self) -> datetime: ...
def encode_cursor(
kind: str,
resource_id: UUID,
created_at: datetime,
item_id: UUID,
) -> str:
payload = {
"kind": kind,
"resource_id": str(resource_id),
"created_at": created_at.isoformat(),
"item_id": str(item_id),
}
raw = json.dumps(payload).encode()
return base64.urlsafe_b64encode(raw).decode()
def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position:
try:
payload = json.loads(base64.urlsafe_b64decode(cursor))
cursor_kind = payload["kind"]
cursor_resource_id = UUID(payload["resource_id"])
created_at = datetime.fromisoformat(payload["created_at"])
item_id = UUID(payload["item_id"])
except (ValueError, KeyError, TypeError, AttributeError):
raise invalid_request("Некорректный cursor") from None
if (
cursor_kind != kind
or cursor_resource_id != resource_id
or created_at.tzinfo is None
):
raise invalid_request("Некорректный cursor")
return created_at, item_id
def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]:
has_next = len(rows) > limit
items = rows[:limit]
return items, has_next
def build_next_cursor(
kind: str,
resource_id: UUID,
items: Sequence[Positioned],
has_next: bool,
) -> str | None:
if not has_next:
return None
last_item = items[-1]
return encode_cursor(kind, resource_id, last_item.created_at, last_item.id)
+39
View File
@@ -0,0 +1,39 @@
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.errors.exceptions import conflict
from app.models.user import User as UserModel
from app.schemas.user import CreateUser, User
from app.storage import users as users_storage
USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято"
def to_user_schema(user: UserModel) -> User:
user_schema = User.model_validate(user)
return user_schema
async def register_user(session: AsyncSession, new_user: CreateUser) -> User:
existing_user = await users_storage.get_user_by_username(session, new_user.username)
await session.commit()
if existing_user is not None:
raise conflict(USERNAME_TAKEN_MESSAGE)
password_hash = await security.hash_password(new_user.password)
try:
user = await users_storage.create_user(
session,
username=new_user.username,
display_name=new_user.display_name,
password_hash=password_hash,
)
await session.commit()
except IntegrityError:
await session.rollback()
raise conflict(USERNAME_TAKEN_MESSAGE) from None
created_user = to_user_schema(user)
return created_user
View File
+74
View File
@@ -0,0 +1,74 @@
"""HTTP-параметры и упаковка ответов; сценарии находятся в domain."""
from typing import Annotated
from urllib.parse import quote
from uuid import UUID
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from starlette.background import BackgroundTask
from app.domain import attachments
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
from app.schemas.attachment import Attachment
router = APIRouter(prefix="/api/v1", tags=["attachments"])
@router.post("/chats/{chat_id}/attachments", status_code=202)
async def upload(
request: Request,
chat_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
capacity: UploadCapacity,
filename: Annotated[str, Query(min_length=1, max_length=255)],
) -> Attachment:
return await attachments.receive_attachment(
session,
storage,
user,
chat_id,
capacity=capacity,
filename=filename,
content_type=request.headers.get("content-type", "").lower(),
declared_length=request.headers.get("content-length"),
chunks=request.stream(),
)
@router.get("/attachments/{attachment_id}")
async def metadata(
attachment_id: UUID, user: CurrentUser, session: DbSession
) -> Attachment:
return await attachments.get_attachment(session, user, attachment_id)
@router.get("/attachments/{attachment_id}/content")
async def content(
attachment_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
variant: Annotated[str, Query()] = "original",
) -> StreamingResponse:
record, stream = await attachments.download_attachment(
session,
storage,
user,
attachment_id,
variant=variant,
)
return StreamingResponse(
stream,
media_type=record.content_type,
headers={
"Content-Length": str(record.size_bytes),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(record.filename, safe=""),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
},
background=BackgroundTask(stream.aclose),
)
+32
View File
@@ -0,0 +1,32 @@
from fastapi import APIRouter, Response
from app.domain.auth import create_session, revoke_session
from app.endpoints.dependencies import (
AppSettings,
BasicCredentials,
BearerToken,
DbSession,
RedisClient,
)
from app.schemas.session import AuthSession
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
@router.post("/sessions", status_code=201)
async def login(
credentials: BasicCredentials,
session: DbSession,
redis: RedisClient,
settings: AppSettings,
) -> AuthSession:
username, password = credentials
return await create_session(
session, redis, username, password, settings.session_ttl_seconds
)
@router.delete("/sessions/current", status_code=204)
async def logout(token: BearerToken, redis: RedisClient) -> Response:
await revoke_session(redis, token)
return Response(status_code=204)
+32
View File
@@ -0,0 +1,32 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Query
from app.domain.chats import create_chat, get_chat, list_chats
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.chat import Chat, ChatPage, CreateChat
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
@router.post("", status_code=201)
async def create_new_chat(
new_chat: CreateChat, creator: CurrentUser, session: DbSession
) -> Chat:
return await create_chat(session, creator, new_chat)
@router.get("")
async def list_user_chats(
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> ChatPage:
return await list_chats(session, user, cursor, limit)
@router.get("/{chat_id}")
async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat:
return await get_chat(session, user, chat_id)
+109
View File
@@ -0,0 +1,109 @@
import base64
import binascii
from collections.abc import AsyncIterator
from typing import Annotated
from fastapi import Depends, Request, Security
from fastapi.openapi.models import HTTPBase, HTTPBearer
from fastapi.security.base import SecurityBase
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import Settings
from app.core.request_capacity import RequestCapacity
from app.domain.auth import authenticate_token
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.user import User
from app.storage.s3 import MinioStorage
async def get_settings(request: Request) -> Settings:
settings: Settings = request.app.state.settings
return settings
async def get_db(request: Request) -> AsyncIterator[AsyncSession]:
session_factory = request.app.state.session_factory
async with session_factory() as session:
yield session
async def get_redis(request: Request) -> Redis:
redis: Redis = request.app.state.redis
return redis
def get_s3(request: Request) -> MinioStorage | None:
return request.app.state.s3
def get_upload_capacity(request: Request) -> RequestCapacity:
return request.app.state.upload_capacity
class AuthorizationHeader(SecurityBase):
def __init__(self, model: HTTPBase, scheme_name: str) -> None:
self.model = model
self.scheme_name = scheme_name
async def __call__(self, request: Request) -> str | None:
return request.headers.get("Authorization")
basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth")
bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth")
def _split_authorization(authorization: str | None) -> tuple[str, str] | None:
if authorization is None:
return None
scheme, _, value = authorization.partition(" ")
if not value.strip():
return None
return scheme.lower(), value.strip()
async def get_bearer_token(
authorization: Annotated[str | None, Security(bearer_auth)],
) -> str:
parts = _split_authorization(authorization)
if parts is None or parts[0] != "bearer":
raise unauthorized_bearer()
return parts[1]
async def get_basic_credentials(
authorization: Annotated[str | None, Security(basic_auth)],
) -> tuple[str, str]:
parts = _split_authorization(authorization)
if parts is None or parts[0] != "basic":
raise unauthorized_basic()
try:
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError, ValueError):
raise unauthorized_basic() from None
username, separator, password = decoded.partition(":")
if not separator or not username:
raise unauthorized_basic()
return username, password
DbSession = Annotated[AsyncSession, Depends(get_db)]
RedisClient = Annotated[Redis, Depends(get_redis)]
AppSettings = Annotated[Settings, Depends(get_settings)]
S3Storage = Annotated[MinioStorage | None, Depends(get_s3)]
UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)]
BearerToken = Annotated[str, Depends(get_bearer_token)]
BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)]
async def get_current_user(
token: BearerToken, session: DbSession, redis: RedisClient
) -> User:
return await authenticate_token(session, redis, token)
CurrentUser = Annotated[User, Depends(get_current_user)]
+23
View File
@@ -0,0 +1,23 @@
from fastapi import APIRouter, Request
from app.domain import health
from app.schemas.base import Health
router = APIRouter(prefix="/health", tags=["health"])
@router.get("/live")
async def liveness() -> Health:
return health.liveness()
@router.get("/ready")
async def readiness(request: Request) -> Health:
state = request.app.state
return await health.readiness(
state.engine,
state.redis,
state.s3,
state.lifecycle,
state.settings.readiness_timeout_seconds,
)
+42
View File
@@ -0,0 +1,42 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Header, Query
from app.domain.idempotency import send_message_idempotent, validate_idempotency_key
from app.domain.messages import list_messages
from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession
from app.schemas.message import CreateMessage, Message, MessagePage
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
@router.post("/{chat_id}/messages", status_code=201)
async def send_new_message(
chat_id: UUID,
new_message: CreateMessage,
sender: CurrentUser,
session: DbSession,
settings: AppSettings,
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
) -> Message:
return await send_message_idempotent(
session,
sender,
chat_id,
new_message,
validate_idempotency_key(idempotency_key),
settings.max_messages,
settings.idempotency_ttl_seconds,
)
@router.get("/{chat_id}/messages")
async def get_history_messages(
chat_id: UUID,
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> MessagePage:
return await list_messages(session, user, chat_id, cursor, limit)
+17
View File
@@ -0,0 +1,17 @@
from fastapi import APIRouter
from app.domain.users import register_user
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.user import CreateUser, User
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", status_code=201)
async def create_user(new_user: CreateUser, session: DbSession) -> User:
return await register_user(session, new_user)
@router.get("/me")
async def get_me(user: CurrentUser) -> User:
return user
View File
+131
View File
@@ -0,0 +1,131 @@
import errno
from collections.abc import Mapping
from PIL import Image, UnidentifiedImageError
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(
self,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers
def invalid_request(message: str = "Неверный запрос") -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(
message: str = "Нет действительной аутентификации",
www_authenticate: str | None = None,
) -> ApiError:
headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None
return ApiError(401, "unauthorized", message, headers)
def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError:
return unauthorized(message, BASIC_CHALLENGE)
def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError:
return unauthorized(message, BEARER_CHALLENGE)
def not_found(message: str = "Объект не найден") -> ApiError:
return ApiError(404, "not_found", message)
def conflict(message: str = "Конфликт") -> ApiError:
return ApiError(409, "conflict", message)
def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError:
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(
message: str = "Требуется Content-Type: application/json",
) -> ApiError:
return ApiError(415, "unsupported_media_type", message)
def rate_limited(retry_after: int = 1) -> ApiError:
return ApiError(
429,
"rate_limited",
"Слишком много запросов",
{"Retry-After": str(max(1, retry_after))},
)
def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
return ApiError(503, "unavailable", message)
def internal_error() -> ApiError:
return ApiError(500, "internal_error", "Внутренняя ошибка сервера")
IMAGE_WORKER_ERRORS = (
ApiError,
Image.DecompressionBombError,
Image.DecompressionBombWarning,
MemoryError,
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
)
def image_worker_error_code(error: Exception) -> int:
code = 2
resource_error = isinstance(
error,
(Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
)
declared_limit = isinstance(error, ApiError) and error.status_code == 413
if resource_error or declared_limit:
code = 3
return code
def image_validation_error(code: int) -> ApiError:
error = unsupported_media_type("Повреждённое изображение или несовпадение MIME")
if code == 3 or code < 0:
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
return error
class ProcessingError(Exception):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
class JobOperationError(Exception):
def __init__(self, code: str):
self.code = code
super().__init__(code)
def pipeline_worker_error_code(error: Exception) -> int:
if isinstance(error, OSError) and error.errno is not None:
if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
return 5
return 4
return image_worker_error_code(error)
+234
View File
@@ -0,0 +1,234 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from collections.abc import Mapping
from uuid import uuid4
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from minio.error import MinioException
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import TimeoutError as RedisTimeoutError
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.requests import ClientDisconnect
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.instance import INSTANCE_ID_HEADER
from app.core.request_id import REQUEST_ID_HEADER
from app.errors.exceptions import (
ApiError,
internal_error,
invalid_request,
not_found,
unavailable,
)
logger = logging.getLogger(__name__)
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
TRANSIENT_SQLSTATES = frozenset({"55P03"})
RECOVERABLE_STORAGE_ERRORS = (
ApiError,
SQLAlchemyError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
)
@asynccontextmanager
async def operation_timeout(seconds: float, error: ApiError):
try:
async with asyncio.timeout(seconds):
yield
except TimeoutError:
raise error from None
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
return await api_error_handler(
request, invalid_request("Загрузка прервана клиентом")
)
def log_recoverable_error(operation: str, error: Exception) -> None:
logger.warning("%s deferred: %s", operation, type(error).__name__)
def _get_request_id(request: Request) -> str:
request_id = getattr(request.state, "request_id", None)
if request_id is None:
request_id = str(uuid4())
request.state.request_id = request_id
return request_id
def _build_response(
request: Request,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> JSONResponse:
request_id = _get_request_id(request)
response_headers = {
**(headers or {}),
REQUEST_ID_HEADER: request_id,
}
instance_id = getattr(request.app.state, "instance_id", None)
if instance_id is not None:
response_headers[INSTANCE_ID_HEADER] = instance_id
body = {
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
}
return JSONResponse(
status_code=status_code,
content=body,
headers=response_headers,
)
async def api_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, ApiError)
return _build_response(
request,
exc.status_code,
exc.code,
exc.message,
exc.headers,
)
async def routing_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, StarletteHTTPException)
error = invalid_request()
if exc.status_code == 404:
error = not_found()
return _build_response(
request,
exc.status_code,
error.code,
error.message,
exc.headers,
)
async def validation_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, RequestValidationError)
fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()})
message = "Неверный запрос"
if fields:
message = f"{message}: {', '.join(fields)}"
return _build_response(
request,
400,
"invalid_request",
message,
)
async def dependency_unavailable_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
request_id = _get_request_id(request)
logger.error(
"Dependency unavailable request_id=%s: %s",
request_id,
type(exc).__name__,
)
error = unavailable()
return _build_response(
request,
error.status_code,
error.code,
error.message,
{"Retry-After": "1"},
)
def _is_transient_database_error(exc: DBAPIError) -> bool:
sqlstate = getattr(exc.orig, "sqlstate", None) or ""
return (
sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES)
or sqlstate in TRANSIENT_SQLSTATES
)
async def database_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, DBAPIError)
handler = unhandled_error_handler
if _is_transient_database_error(exc):
handler = dependency_unavailable_handler
return await handler(request, exc)
async def unhandled_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
request_id = _get_request_id(request)
logger.error(
"Unhandled error request_id=%s",
request_id,
exc_info=exc,
)
error = internal_error()
return _build_response(
request,
error.status_code,
error.code,
error.message,
)
def register_exception_handlers(app: FastAPI) -> None:
app.add_exception_handler(ApiError, api_error_handler)
app.add_exception_handler(ClientDisconnect, client_disconnect_handler)
app.add_exception_handler(MinioException, dependency_unavailable_handler)
app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler)
app.add_exception_handler(StarletteHTTPException, routing_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(OperationalError, dependency_unavailable_handler)
app.add_exception_handler(InterfaceError, dependency_unavailable_handler)
app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(DBAPIError, database_error_handler)
app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler)
app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(OSError, dependency_unavailable_handler)
app.add_exception_handler(Exception, unhandled_error_handler)
+94
View File
@@ -0,0 +1,94 @@
import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
import uvicorn
from fastapi import Depends, FastAPI
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.instance import InstanceIdMiddleware, resolve_instance_id
from app.core.lifecycle import GracefulServer, Lifecycle
from app.core.redis import create_redis_client
from app.core.request_capacity import acquire_request_slot, create_request_capacity
from app.core.request_id import RequestIdMiddleware
from app.endpoints.auth import router as auth_router
from app.endpoints.attachments import router as attachments_router
from app.endpoints.chats import router as chats_router
from app.endpoints.health import router as health_router
from app.endpoints.messages import router as messages_router
from app.endpoints.users import router as users_router
from app.errors.handlers import register_exception_handlers
from app.storage.s3 import create_s3_storage
def configure_logging(settings: Settings) -> None:
logging.basicConfig(
level=settings.log_level,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings()
configure_logging(settings)
engine = create_db_engine(settings)
redis = create_redis_client(settings)
s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None
@asynccontextmanager
async def lifespan(_: FastAPI) -> AsyncGenerator[None]:
yield
if s3 is not None:
await s3.aclose()
await redis.aclose()
await engine.dispose()
application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan)
application.state.settings = settings
application.state.engine = engine
application.state.session_factory = create_session_factory(engine)
application.state.redis = redis
application.state.s3 = s3
application.state.upload_capacity = create_request_capacity(
settings.s3.max_concurrent_uploads
)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
application.state.lifecycle = Lifecycle()
application.add_middleware(RequestIdMiddleware)
application.add_middleware(
InstanceIdMiddleware, instance_id=application.state.instance_id
)
register_exception_handlers(application)
limited = [Depends(acquire_request_slot)]
application.include_router(health_router)
application.include_router(users_router, dependencies=limited)
application.include_router(auth_router, dependencies=limited)
application.include_router(chats_router, dependencies=limited)
application.include_router(messages_router, dependencies=limited)
application.include_router(attachments_router, dependencies=limited)
return application
def main() -> None:
settings = Settings()
application = create_app(settings)
config = uvicorn.Config(
application,
host=settings.app_host,
port=settings.app_port,
log_level=settings.log_level.lower(),
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
)
GracefulServer(
config, application.state.lifecycle, settings.shutdown_drain_seconds
).run()
if __name__ == "__main__":
main()
+25
View File
@@ -0,0 +1,25 @@
from app.models.job_replay import JobReplay
from app.models.base import Base
from app.models.job import AttachmentJob
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.models.counter import Counter
from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = [
"JobReplay",
"AttachmentJob",
"Base",
"Attachment",
"MessageAttachment",
"UploadIntent",
"Chat",
"ChatMember",
"Counter",
"IdempotencyKey",
"Message",
"User",
]
+70
View File
@@ -0,0 +1,70 @@
"""Метаданные оригиналов и журнал операций до записи в S3."""
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Attachment(Base):
__tablename__ = "attachments"
__table_args__ = (
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint(
"status IN ('queued', 'processing', 'ready', 'failed')", name="status"
),
CheckConstraint(
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
name="error_code",
),
Index("ix_attachments_chat_id", "chat_id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
object_key: Mapped[str] = mapped_column(String(128), unique=True)
filename: Mapped[str] = mapped_column(String(255))
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int]
sha256: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16), server_default="queued")
error_code: Mapped[str | None] = mapped_column(String(64))
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
crop_key: Mapped[str | None] = mapped_column(String(128))
optimized_key: Mapped[str | None] = mapped_column(String(128))
variants: Mapped[list[dict]] = mapped_column(
JSONB, default=list, server_default=text("'[]'::jsonb")
)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int]
class UploadIntent(Base):
__tablename__ = "upload_intents"
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
object_key: Mapped[str] = mapped_column(String(128), unique=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+26
View File
@@ -0,0 +1,26 @@
import uuid
from datetime import datetime
from typing import Any, ClassVar
from sqlalchemy import DateTime, MetaData
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.types import TypeEngine
NAMING_CONVENTION = {
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
class Base(DeclarativeBase):
metadata = MetaData(naming_convention=NAMING_CONVENTION)
type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = {
uuid.UUID: PG_UUID(as_uuid=True),
datetime: DateTime(timezone=True),
}
+31
View File
@@ -0,0 +1,31 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class Chat(Base):
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
title: Mapped[str] = mapped_column(String(100))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
members: Mapped[list["ChatMember"]] = relationship(
back_populates="chat",
cascade="all, delete-orphan",
passive_deletes=True,
lazy="selectin",
)
messages: Mapped[list["Message"]] = relationship(
back_populates="chat", lazy="raise"
)
+25
View File
@@ -0,0 +1,25 @@
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class ChatMember(Base):
__tablename__ = "chat_members"
__table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),)
chat_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise")
user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise")
+12
View File
@@ -0,0 +1,12 @@
from sqlalchemy import BigInteger, String
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Counter(Base):
__tablename__ = "counters"
name: Mapped[str] = mapped_column(String(64), primary_key=True)
value: Mapped[int] = mapped_column(BigInteger)
+23
View File
@@ -0,0 +1,23 @@
"""Сохранённый результат отправки сообщения."""
import uuid
from datetime import datetime
from sqlalchemy import ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class IdempotencyKey(Base):
__tablename__ = "idempotency_keys"
__table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
method: Mapped[str] = mapped_column(String(8), primary_key=True)
path: Mapped[str] = mapped_column(String(255), primary_key=True)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
request_hash: Mapped[str] = mapped_column(String(64))
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id"))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+40
View File
@@ -0,0 +1,40 @@
"""Устойчивая работа и намерение публикации; один pipeline на вложение."""
from datetime import datetime
from uuid import UUID, uuid4
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class AttachmentJob(Base):
__tablename__ = "attachment_jobs"
__table_args__ = (
CheckConstraint("pipeline_version = 1", name="pipeline_version"),
CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"),
Index("ix_attachment_jobs_due", "next_attempt_at"),
Index("ix_attachment_jobs_lease", "lease_until"),
)
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey("attachments.id"),
unique=True,
)
staging_key: Mapped[str | None] = mapped_column(String(128))
pipeline_version: Mapped[int] = mapped_column(server_default="1")
attempt: Mapped[int] = mapped_column(server_default="0")
cycle_start_attempt: Mapped[int] = mapped_column(server_default="0")
attempt_limit: Mapped[int | None]
last_error_permanent: Mapped[bool | None]
published_attempt: Mapped[int] = mapped_column(server_default="0")
next_attempt_at: Mapped[datetime] = mapped_column(
server_default=func.clock_timestamp()
)
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None]
last_error_code: Mapped[str | None] = mapped_column(String(64))
finished_at: Mapped[datetime | None]
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+22
View File
@@ -0,0 +1,22 @@
"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class JobReplay(Base):
__tablename__ = "job_replays"
id: Mapped[UUID] = mapped_column(primary_key=True)
job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True)
actor: Mapped[str] = mapped_column(String(64))
reason: Mapped[str] = mapped_column(String(32))
previous_attempt: Mapped[int]
attempt_limit: Mapped[int]
previous_error_code: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+29
View File
@@ -0,0 +1,29 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index, Text, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class Message(Base):
__tablename__ = "messages"
__table_args__ = (
Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
text: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise")
sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise")
+30
View File
@@ -0,0 +1,30 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
username: Mapped[str] = mapped_column(String(32), unique=True)
display_name: Mapped[str] = mapped_column(String(100))
password_hash: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
memberships: Mapped[list["ChatMember"]] = relationship(
back_populates="user", lazy="raise"
)
messages: Mapped[list["Message"]] = relationship(
back_populates="sender", lazy="raise"
)
+6
View File
@@ -0,0 +1,6 @@
from app.schemas.chat import Chat
from app.schemas.message import Message
from app.schemas.session import AuthSession
from app.schemas.user import User
__all__ = ["AuthSession", "Chat", "Message", "User"]
+53
View File
@@ -0,0 +1,53 @@
"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются."""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator
from app.schemas.base import Base
class Variant(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
name: Literal["thumbnail", "crop", "optimized"]
content_type: Literal["image/jpeg", "image/png", "image/webp"]
size_bytes: int = Field(ge=1)
width: int = Field(ge=1)
height: int = Field(ge=1)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
class Attachment(Base):
chat_id: UUID
owner_id: UUID
filename: str = Field(min_length=1, max_length=255)
content_type: Literal["image/png", "image/jpeg"]
size_bytes: int = Field(ge=1, le=10_485_760)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
status: Literal["queued", "processing", "ready", "failed"]
variants: list[Variant] = Field(default_factory=list)
error_code: str | None = None
created_at: datetime
updated_at: datetime
VARIANT_FILENAMES = {
"thumbnail": "thumbnail.jpg",
"crop": "crop.jpg",
"optimized": "optimized.webp",
}
class PipelineResult(BaseModel):
model_config = ConfigDict(extra="forbid")
variants: list[Variant] = Field(min_length=3, max_length=3)
@field_validator("variants")
@classmethod
def require_all_variants(cls, variants: list[Variant]) -> list[Variant]:
if {item.name for item in variants} != set(VARIANT_FILENAMES):
raise ValueError("Нужны thumbnail, crop и optimized без повторов")
return variants
+34
View File
@@ -0,0 +1,34 @@
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, field_validator
class Base(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
id: UUID
class Health(BaseModel):
model_config = ConfigDict(extra="forbid")
status: Literal["ok"]
class BaseCreate(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
if isinstance(value, str):
if not value.strip():
raise ValueError("Строка не может состоять только из пробелов")
if "\x00" in value:
raise ValueError("Строка не может содержать символ NUL")
return value
class BasePage(BaseModel):
model_config = ConfigDict(extra="forbid")
+22
View File
@@ -0,0 +1,22 @@
from datetime import datetime
from uuid import UUID
from pydantic import Field
from app.schemas.base import Base, BaseCreate, BasePage
class Chat(Base):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
created_at: datetime
class CreateChat(BaseCreate):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
class ChatPage(BasePage):
items: list[Chat]
next_cursor: str | None
+22
View File
@@ -0,0 +1,22 @@
"""Сообщение брокера: только идентификаторы и служебные поля контракта v1."""
from typing import Literal
from uuid import UUID
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
class JobMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1] = 1
type: Literal["attachment.process.v1"] = "attachment.process.v1"
job_id: UUID
attachment_id: UUID
pipeline_version: int = Field(default=1, ge=1)
attempt: int = Field(ge=1)
created_at: AwareDatetime
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
+41
View File
@@ -0,0 +1,41 @@
from datetime import datetime
from typing import Self
from uuid import UUID
from pydantic import Field, field_validator, model_validator
from app.schemas.base import Base, BaseCreate, BasePage
class Message(Base):
chat_id: UUID
sender_id: UUID
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
created_at: datetime
class CreateMessage(BaseCreate):
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
if isinstance(value, str):
if "\x00" in value or (value != "" and not value.strip()):
raise ValueError("Некорректная строка сообщения")
return value
@model_validator(mode="after")
def require_content(self) -> Self:
if not self.text and not self.attachment_ids:
raise ValueError("Нужен текст или вложение")
if len(self.attachment_ids) != len(set(self.attachment_ids)):
raise ValueError("attachment_ids содержит дубликаты")
return self
class MessagePage(BasePage):
items: list[Message]
next_cursor: str | None
+16
View File
@@ -0,0 +1,16 @@
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from app.schemas.user import User
class AuthSession(BaseModel):
model_config = ConfigDict(extra="forbid")
token: str = Field(min_length=32)
token_type: Literal["Bearer"] = "Bearer"
expires_at: datetime
user: User
+24
View File
@@ -0,0 +1,24 @@
from datetime import datetime
from pydantic import Field
from app.schemas.base import Base, BaseCreate
USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$"
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
class User(Base):
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
created_at: datetime
class CreateUser(BaseCreate):
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(
min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False
)
+1
View File
@@ -0,0 +1 @@
"""Доступ к данным PostgreSQL и Redis."""
+99
View File
@@ -0,0 +1,99 @@
"""SQL-запросы метаданных и журнала; commit выполняется в domain."""
from datetime import timedelta
from uuid import UUID
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
async def create_attachment(session: AsyncSession, **values) -> Attachment:
record = Attachment(**values)
session.add(record)
await session.flush()
return record
async def get_attachment(
session: AsyncSession, attachment_id: UUID
) -> Attachment | None:
return await session.get(Attachment, attachment_id)
async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
async def get_for_chat(
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
) -> list[Attachment]:
records = []
if attachment_ids:
records = list(
await session.scalars(
select(Attachment).where(
Attachment.id.in_(attachment_ids),
Attachment.chat_id == chat_id,
)
)
)
return records
async def bind_attachments(
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
) -> None:
session.add_all(
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
for position, id in enumerate(attachment_ids)
)
await session.flush()
async def message_attachment_ids(
session: AsyncSession, message_ids: list[UUID]
) -> dict[UUID, list[UUID]]:
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
if not message_ids:
return result
rows = await session.execute(
select(MessageAttachment)
.where(MessageAttachment.message_id.in_(message_ids))
.order_by(MessageAttachment.position)
)
for link in rows.scalars():
result[link.message_id].append(link.attachment_id)
return result
async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
session.add(UploadIntent(id=id, object_key=key))
await session.flush()
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
return await session.scalar(
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
)
async def finish_upload(session: AsyncSession, id: UUID) -> None:
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
async def stale_uploads(
session: AsyncSession, grace_seconds: int, limit: int
) -> list[UploadIntent]:
result = await session.scalars(
select(UploadIntent)
.where(
UploadIntent.created_at
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
)
.order_by(UploadIntent.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
return list(result)
+93
View File
@@ -0,0 +1,93 @@
"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack."""
import asyncio
import logging
from dataclasses import dataclass
import aio_pika
from aio_pika.abc import AbstractIncomingMessage
from app.core.jobs import JobSettings
from app.schemas.job import JobMessage
@dataclass(frozen=True)
class Delivery:
id: AbstractIncomingMessage
payload: bytes
class JobBroker:
def __init__(self, settings: JobSettings) -> None:
self.settings = settings
self.connection = self.channel = self.queue = self.iterator = None
logging.getLogger("aio_pika").setLevel(logging.CRITICAL)
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
async def ensure_ready(self) -> None:
if (
self.connection is None
or self.connection.is_closed
or self.channel is None
or self.channel.is_closed
):
await self.aclose()
self.connection = await aio_pika.connect(
host=self.settings.broker_host,
port=self.settings.broker_port,
login=self.settings.broker_user,
password=self.settings.broker_password_file.read_text().strip(),
virtualhost=self.settings.broker_vhost,
timeout=5,
heartbeat=30,
)
self.channel = await self.connection.channel(
publisher_confirms=True, on_return_raises=True
)
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(
self.settings.broker_queue, durable=True
)
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
),
routing_key=self.settings.broker_queue,
mandatory=True,
timeout=10,
)
async def receive(self, consumer: str) -> Delivery:
await self.ensure_ready()
if self.iterator is None:
self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer)
await self.iterator.__aenter__()
message = await self.iterator.__anext__()
return Delivery(message, message.body)
async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None:
await delivery_id.ack()
async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None:
await asyncio.sleep(self.settings.publisher_poll_seconds)
await delivery_id.nack(requeue=True)
async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None:
logging.getLogger(__name__).warning("Rejected invalid job envelope")
await delivery_id.reject(requeue=False)
async def aclose(self) -> None:
connection = self.connection
self.connection = self.channel = self.queue = self.iterator = None
if connection is not None and not connection.is_closed:
await connection.close()
+67
View File
@@ -0,0 +1,67 @@
from collections.abc import Sequence
from uuid import UUID
from sqlalchemy import exists, select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.storage.types import Position
async def create_chat(
session: AsyncSession,
*,
title: str,
member_ids: Sequence[UUID],
) -> Chat:
chat = Chat(
title=title,
members=[ChatMember(user_id=user_id) for user_id in member_ids],
)
session.add(chat)
await session.flush()
return chat
async def get_chat_for_member(
session: AsyncSession,
chat_id: UUID,
user_id: UUID,
) -> Chat | None:
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
chat = result.scalar_one_or_none()
return chat
async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool:
stmt = select(
exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
is_member = bool(result.scalar_one())
return is_member
async def list_chats_for_user(
session: AsyncSession,
user_id: UUID,
after: Position | None,
limit: int,
) -> list[Chat]:
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(ChatMember.user_id == user_id)
)
if after is not None:
stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit)
result = await session.execute(stmt)
chats = list(result.scalars().all())
return chats
+15
View File
@@ -0,0 +1,15 @@
"""Простые проверки соединения с общими хранилищами."""
from redis.asyncio import Redis
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text("SELECT 1"))
return True
async def check_redis(client: Redis) -> bool:
return bool(await client.ping())
+67
View File
@@ -0,0 +1,67 @@
"""Хранение и блокировка ключей в PostgreSQL."""
from datetime import timedelta
import hashlib
import json
from uuid import UUID
from sqlalchemy import BigInteger, func, literal, select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.idempotency_key import IdempotencyKey
Scope = tuple[UUID, str, str, str]
async def lock_key(session: AsyncSession, scope: Scope) -> None:
raw = json.dumps([str(scope[0]), *scope[1:]]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
await session.execute(
select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger)))
)
async def get_active_key(
session: AsyncSession, scope: Scope, ttl_seconds: int
) -> IdempotencyKey | None:
stmt = select(IdempotencyKey).where(
tuple_(
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
)
== tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
return await session.scalar(stmt)
async def save_result(
session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID
) -> None:
stmt = insert(IdempotencyKey).values(
user_id=scope[0],
method=scope[1],
path=scope[2],
key=scope[3],
request_hash=request_hash,
message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
"message_id": message_id,
"created_at": func.clock_timestamp(),
},
)
)
+59
View File
@@ -0,0 +1,59 @@
"""Операторские выборки и журнал replay; транзакция принадлежит domain."""
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.models.job_replay import JobReplay
async def failed_jobs(session, limit: int):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "failed")
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
.limit(limit)
)
return result.all()
async def replay_record(session, request_id):
return await session.get(JobReplay, request_id)
async def insert_replay(session, **values) -> bool:
result = await session.scalar(
insert(JobReplay)
.values(**values)
.on_conflict_do_nothing(index_elements=[JobReplay.id])
.returning(JobReplay.id)
)
return result is not None
async def replay_history(session, job_id):
return (
await session.scalars(
select(JobReplay)
.where(JobReplay.job_id == job_id)
.order_by(JobReplay.created_at)
.limit(100)
)
).all()
def reset_failed(job, attachment, now, attempt_limit):
job.cycle_start_attempt = job.attempt
job.attempt_limit = attempt_limit
job.published_attempt = job.attempt
job.next_attempt_at = now
job.lease_token = None
job.lease_until = None
job.finished_at = None
job.last_error_code = None
job.last_error_permanent = None
attachment.status = "queued"
attachment.error_code = None
attachment.updated_at = now
+160
View File
@@ -0,0 +1,160 @@
"""SQL и изменения jobs/attachments; commit остаётся в domain."""
from dataclasses import dataclass
from datetime import datetime, timedelta
from uuid import UUID
from sqlalchemy import and_, or_, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.schemas.attachment import VARIANT_FILENAMES
@dataclass(frozen=True)
class JobWork:
job_id: UUID
attachment_id: UUID
lease_token: UUID
pipeline_version: int
staging_key: str
content_type: str
size_bytes: int
sha256: str
@property
def original_key(self) -> str:
return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
def variant_key(self, name: str) -> str:
filename = VARIANT_FILENAMES[name]
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
return job.id
async def lock_job(session: AsyncSession, job_id: UUID):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(AttachmentJob.id == job_id)
.with_for_update(of=AttachmentJob)
.execution_options(populate_existing=True)
)
return result.one_or_none()
async def database_time(session: AsyncSession) -> datetime:
return await session.scalar(select(func.clock_timestamp()))
async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
Attachment.status == "processing",
AttachmentJob.lease_until < func.clock_timestamp(),
)
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def start_job(
job, attachment, token: UUID, now: datetime, lease_seconds: int
) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id,
attachment.id,
token,
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
def record_failure(
job,
attachment,
now: datetime,
error_code: str,
*,
terminal: bool,
retry_seconds: float,
) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
job.next_attempt_at = now + timedelta(seconds=retry_seconds)
job.finished_at = now if terminal else None
attachment.status = "failed" if terminal else "queued"
attachment.error_code = error_code if terminal else None
attachment.updated_at = now
def complete_job(
job, attachment, work: JobWork, variants: list[dict], now: datetime
) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
attachment.optimized_key = work.variant_key("optimized")
attachment.variants = variants
attachment.status = "ready"
attachment.error_code = None
attachment.updated_at = now
job.lease_token = None
job.lease_until = None
job.last_error_code = None
job.last_error_permanent = None
job.finished_at = now
async def terminal_staging(
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(
Attachment.status == "ready",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=retention),
),
and_(
Attachment.status == "failed",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=failed_retention),
),
),
)
.order_by(AttachmentJob.finished_at)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def clear_staging(job) -> None:
job.staging_key = None
+59
View File
@@ -0,0 +1,59 @@
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.counter import Counter
from app.models.message import Message
from app.storage.types import Position
MESSAGES_COUNTER = "messages"
async def get_message(session: AsyncSession, message_id: UUID) -> Message | None:
return await session.get(Message, message_id)
async def create_message(
session: AsyncSession,
*,
chat_id: UUID,
sender_id: UUID,
text: str,
) -> Message:
message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
session.add(message)
await session.flush()
return message
async def list_messages(
session: AsyncSession,
chat_id: UUID,
after: Position | None,
limit: int,
) -> list[Message]:
stmt = select(Message).where(Message.chat_id == chat_id)
if after is not None:
stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
stmt = stmt.order_by(Message.created_at, Message.id).limit(limit)
result = await session.execute(stmt)
messages = list(result.scalars().all())
return messages
async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool:
stmt = (
insert(Counter)
.values(name=MESSAGES_COUNTER, value=1)
.on_conflict_do_update(
index_elements=[Counter.name],
set_={"value": Counter.value + 1},
where=Counter.value < max_messages,
)
.returning(Counter.value)
)
result = await session.execute(stmt)
is_reserved = result.scalar_one_or_none() is not None
return is_reserved
+33
View File
@@ -0,0 +1,33 @@
"""Jobs одновременно хранят намерение публикации очередной попытки."""
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
async def next_unpublished_job(
session: AsyncSession, *, attachment_id: UUID | None = None
) -> AttachmentJob | None:
statement = (
select(AttachmentJob)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
Attachment.status == "queued",
AttachmentJob.next_attempt_at <= func.clock_timestamp(),
AttachmentJob.published_attempt <= AttachmentJob.attempt,
)
.order_by(AttachmentJob.next_attempt_at, AttachmentJob.id)
.limit(1)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
if attachment_id is not None:
statement = statement.where(AttachmentJob.attachment_id == attachment_id)
return await session.scalar(statement)
def mark_published(job: AttachmentJob) -> None:
job.published_attempt = job.attempt + 1
+180
View File
@@ -0,0 +1,180 @@
"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
import asyncio
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import TypeVar
import time
from urllib.parse import urlsplit
from anyio import CancelScope
from minio import Minio
import urllib3
from app.core.s3 import S3Settings
T = TypeVar("T")
async def run_sync(
operation: Callable[..., T], *args, discard_result=None, **kwargs
) -> T:
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
try:
result = await asyncio.shield(task)
except asyncio.CancelledError:
with CancelScope(shield=True):
await settle_cancelled_call(task, discard_result)
raise
return result
async def settle_cancelled_call(
task: asyncio.Task, discard_result: Callable | None
) -> None:
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
while not task.done():
with suppress(asyncio.CancelledError):
await asyncio.wait([task])
completed = not task.cancelled() and task.exception() is None
if completed and discard_result is not None:
discard_result(task.result())
@dataclass(frozen=True)
class ObjectInfo:
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
key: str
size_bytes: int | None
content_type: str | None
last_modified: datetime | None
def close_response(response) -> None:
response.close()
response.release_conn()
class ObjectStream:
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
def __init__(self, storage, response):
self.storage = storage
self.response = response
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
self.closed = False
def __aiter__(self):
return self
async def __anext__(self):
if self.closed:
raise StopAsyncIteration
try:
remaining = self.deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError
async with asyncio.timeout(remaining):
chunk = await self.storage._call(
lambda: self.response.read(self.storage.settings.chunk_bytes)
)
if not chunk:
await self.aclose()
raise StopAsyncIteration
except BaseException:
await self.aclose()
raise
return chunk
async def aclose(self):
if not self.closed:
self.closed = True
close_response(self.response)
class MinioStorage:
def __init__(self, settings: S3Settings) -> None:
self.settings = settings
url = urlsplit(settings.endpoint)
self._http = urllib3.PoolManager(
timeout=urllib3.Timeout(
connect=settings.connect_timeout_seconds,
read=settings.read_timeout_seconds,
),
retries=urllib3.Retry(total=0),
maxsize=32,
)
self._health_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=0.25, read=0.5),
retries=urllib3.Retry(total=0),
)
self._client = Minio(
url.netloc,
secure=url.scheme == "https",
region=settings.region,
access_key=settings.access_key_id_file.read_text().strip(),
secret_key=settings.secret_access_key_file.read_text().strip(),
http_client=self._http,
)
async def _call(self, operation: Callable, *, discard_result=None):
return await run_sync(operation, discard_result=discard_result)
async def put_file(
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
) -> None:
def put():
with path.open("rb") as source:
self._client.put_object(
self.settings.bucket,
key,
source,
size_bytes,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10_485_760,
num_parallel_uploads=1,
)
await self._call(put)
async def open_object(self, key: str) -> ObjectStream:
response = await self._call(
lambda: self._client.get_object(self.settings.bucket, key),
discard_result=close_response,
)
return ObjectStream(self, response)
async def stat_object(self, key: str) -> ObjectInfo:
def stat():
info = self._client.stat_object(self.settings.bucket, key)
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
return await self._call(stat)
async def delete_object(self, key: str) -> None:
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
async def check_available(self) -> bool:
status = await self._call(
lambda: self._health_http.request(
"GET",
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status
)
return status == 200
async def aclose(self) -> None:
self._http.clear()
self._health_http.clear()
def create_s3_storage(settings: S3Settings) -> MinioStorage:
return MinioStorage(settings)
+34
View File
@@ -0,0 +1,34 @@
from datetime import datetime
from uuid import UUID
from redis.asyncio import Redis
SESSION_KEY_PREFIX = "session:"
def session_key(token_hash: str) -> str:
return f"{SESSION_KEY_PREFIX}{token_hash}"
async def create_session(
redis: Redis,
*,
token_hash: str,
user_id: UUID,
expires_at: datetime,
) -> None:
await redis.set(session_key(token_hash), str(user_id), pxat=expires_at)
async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None:
raw_user_id = await redis.get(session_key(token_hash))
if isinstance(raw_user_id, bytes):
raw_user_id = raw_user_id.decode()
user_id = UUID(raw_user_id) if raw_user_id else None
return user_id
async def delete_session(redis: Redis, token_hash: str) -> bool:
deleted_count = await redis.delete(session_key(token_hash))
is_deleted = deleted_count > 0
return is_deleted
+6
View File
@@ -0,0 +1,6 @@
"""Общие типы слоя хранения."""
from datetime import datetime
from uuid import UUID
Position = tuple[datetime, UUID]
+43
View File
@@ -0,0 +1,43 @@
from collections.abc import Collection
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.user import User
async def create_user(
session: AsyncSession,
*,
username: str,
display_name: str,
password_hash: str,
) -> User:
user = User(
username=username, display_name=display_name, password_hash=password_hash
)
session.add(user)
await session.flush()
return user
async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
user = await session.get(User, user_id)
return user
async def get_user_by_username(session: AsyncSession, username: str) -> User | None:
stmt = select(User).where(User.username == username)
result = await session.execute(stmt)
user = result.scalar_one_or_none()
return user
async def get_existing_user_ids(
session: AsyncSession, user_ids: Collection[UUID]
) -> set[UUID]:
stmt = select(User.id).where(User.id.in_(user_ids))
result = await session.execute(stmt)
existing_ids = set(result.scalars().all())
return existing_ids
+7
View File
@@ -0,0 +1,7 @@
services:
postgres:
ports:
- "127.0.0.1:25432:5432"
redis:
ports:
- "127.0.0.1:26379:6379"
+417
View File
@@ -0,0 +1,417 @@
name: messenger
x-api-image: &api-image
image: ${API_IMAGE:?Set API_IMAGE in .env}
pull_policy: never
x-base-env: &base-env
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
DB_HOST: ${DB_HOST:?Set DB_HOST in .env}
DB_PORT: ${DB_PORT:?Set DB_PORT in .env}
DB_NAME: ${DB_NAME:?Set DB_NAME in .env}
DB_USER: ${DB_USER:?Set DB_USER in .env}
DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env}
DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env}
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env}
SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env}
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env}
READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env}
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_REGION: ${S3_REGION:?Set S3_REGION in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env}
S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env}
S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env}
S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env}
S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env}
S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env}
S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env}
S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env}
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
x-jobs-env: &jobs-env
BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env}
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env}
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env}
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env}
JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env}
PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env}
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env}
IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env}
IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env}
IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env}
IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env}
services:
minio:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: server
image: messenger-minio:local
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
restart: unless-stopped
environment:
MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
MINIO_BROWSER: ${MINIO_BROWSER:?Set MINIO_BROWSER in .env}
secrets:
- source: minio_root_user
target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
- source: minio_root_password
target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
volumes:
- miniodata:/data
networks: [objects]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
minio-bootstrap:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: bootstrap
image: messenger-minio-bootstrap:local
environment:
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
secrets:
- source: minio_root_user
target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
- source: minio_root_password
target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [objects]
restart: "no"
depends_on:
minio:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
postgres:
image: postgres:16.10-bookworm
restart: unless-stopped
environment:
POSTGRES_USER: ${DB_USER:?Set DB_USER in .env}
POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env}
POSTGRES_PASSWORD_FILE: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
secrets:
- source: db_password
target: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
redis:
image: redis:7.4.2-bookworm
restart: unless-stopped
entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"]
secrets:
- redis_password
volumes:
- ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
- ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro
- redisdata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
broker:
image: rabbitmq:4.4.0
hostname: broker
restart: unless-stopped
entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"]
environment:
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
secrets:
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
volumes:
- ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 20s
deploy:
resources:
limits:
cpus: "1.0"
memory: 768M
publisher:
<<: *api-image
init: true
command: ["python", "-m", "app.core.outbox_publisher"]
restart: unless-stopped
environment:
<<: [*base-env, *jobs-env]
secrets:
- db_password
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
healthcheck:
disable: true
stop_grace_period: 15s
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
worker:
<<: *api-image
init: true
command: ["python", "-m", "app.core.job_worker"]
restart: unless-stopped
environment:
<<: [*base-env, *s3-env, *jobs-env]
secrets:
- db_password
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m,mode=1777
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
minio:
condition: service_healthy
minio-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
healthcheck:
disable: true
stop_signal: SIGTERM
stop_grace_period: 20s
deploy:
resources:
limits:
cpus: "2.0"
memory: 2048M
migrate:
<<: *api-image
command: ["alembic", "upgrade", "head"]
environment: *base-env
secrets:
- db_password
depends_on:
postgres:
condition: service_healthy
restart: "no"
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
api:
<<: *api-image
build:
context: .
args:
API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env}
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env}
REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env}
REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env}
REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env}
REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env}
REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env}
REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env}
SESSION_STORE: ${SESSION_STORE:?Set SESSION_STORE in .env}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env}
secrets:
- db_password
- redis_password
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
deploy:
replicas: ${API_REPLICAS:?Set API_REPLICAS in .env}
resources:
limits:
cpus: "1.0"
memory: 1536M
healthcheck:
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
interval: 3s
timeout: 3s
start_period: 10s
retries: 2
stop_signal: SIGTERM
stop_grace_period: 8s
depends_on:
minio:
condition: service_healthy
minio-bootstrap:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
cleanup:
<<: *api-image
command: ["python", "-m", "app.core.cleanup", "--apply"]
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
secrets:
- db_password
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
minio-bootstrap:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
nginx:
image: nginx:1.28.0-alpine
restart: unless-stopped
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"]
interval: 5s
timeout: 3s
retries: 5
depends_on:
api:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
volumes:
pgdata:
redisdata:
miniodata:
rabbitmqdata:
networks:
default:
objects:
internal: true
secrets:
broker_password:
file: ./secrets/broker_password
db_password:
file: ./secrets/db_password
redis_password:
file: ./secrets/redis_password
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+11
View File
@@ -0,0 +1,11 @@
$ python3 scripts/init_local.py
Создан файл secrets/db_password
Создан файл secrets/redis_password
Создан файл secrets/broker_password
Создан файл secrets/minio_root_user
Создан файл secrets/minio_root_password
Создан файл secrets/s3_access_key_id
Создан файл secrets/s3_secret_access_key
Настройки готовы; существующие значения сохранены
exit_code=0; duration_seconds=0.215
+5
View File
@@ -0,0 +1,5 @@
$ python3 -m black --check app infra migrations scripts student-tests tests
All done! ✨ 🍰 ✨
131 files would be left unchanged.
exit_code=0; duration_seconds=1.921
+4
View File
@@ -0,0 +1,4 @@
$ python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
exit_code=0; duration_seconds=0.165
+4
View File
@@ -0,0 +1,4 @@
$ python3 scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
exit_code=0; duration_seconds=1.425
File diff suppressed because it is too large Load Diff
+47
View File
@@ -0,0 +1,47 @@
$ docker compose --env-file .env up -d --wait --wait-timeout 240 nginx
Container messenger-test-redis-1 Running
Container messenger-test-postgres-1 Running
Container messenger-test-minio-1 Running
Container messenger-test-api-1 Running
Container messenger-test-api-2 Running
Container messenger-test-nginx-1 Running
Container messenger-test-postgres-1 Waiting
Container messenger-test-minio-1 Waiting
Container messenger-test-postgres-1 Healthy
Container messenger-test-minio-1 Healthy
Container messenger-test-minio-bootstrap-1 Starting
Container messenger-test-migrate-1 Starting
Container messenger-test-migrate-1 Started
Container messenger-test-minio-bootstrap-1 Started
Container messenger-test-postgres-1 Waiting
Container messenger-test-redis-1 Waiting
Container messenger-test-migrate-1 Waiting
Container messenger-test-minio-1 Waiting
Container messenger-test-minio-bootstrap-1 Waiting
Container messenger-test-minio-1 Healthy
Container messenger-test-postgres-1 Healthy
Container messenger-test-redis-1 Healthy
Container messenger-test-migrate-1 Exited
Container messenger-test-minio-bootstrap-1 Exited
Container messenger-test-api-2 Waiting
Container messenger-test-api-1 Waiting
Container messenger-test-api-2 Healthy
Container messenger-test-api-1 Healthy
Container messenger-test-minio-1 Waiting
Container messenger-test-postgres-1 Waiting
Container messenger-test-redis-1 Waiting
Container messenger-test-api-2 Waiting
Container messenger-test-api-1 Waiting
Container messenger-test-migrate-1 Waiting
Container messenger-test-nginx-1 Waiting
Container messenger-test-minio-bootstrap-1 Waiting
Container messenger-test-migrate-1 Exited
Container messenger-test-postgres-1 Healthy
Container messenger-test-minio-1 Healthy
Container messenger-test-minio-bootstrap-1 Exited
Container messenger-test-nginx-1 Healthy
Container messenger-test-redis-1 Healthy
Container messenger-test-api-2 Healthy
Container messenger-test-api-1 Healthy
exit_code=0; duration_seconds=7.972
@@ -0,0 +1,3 @@
$ docker compose --env-file .env exec -T postgres createdb -U messenger messenger_test
exit_code=0; duration_seconds=0.266
+7
View File
@@ -0,0 +1,7 @@
$ python3 -m pytest -q --junitxml=evidence/pytest.xml
........................................................................ [ 35%]
........................................................................ [ 71%]
......................................................... [100%]
201 passed in 111.29s (0:01:51)
exit_code=0; duration_seconds=115.857
+4
View File
@@ -0,0 +1,4 @@
$ python3 tests/smoke.py --base-url http://127.0.0.1:28080
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
exit_code=0; duration_seconds=4.085

Some files were not shown because too many files have changed in this diff Show More