prepared skeleton for lab5

This commit is contained in:
2026-11-07 16:10:32 +03:00
parent 276dbdad8a
commit 0a70358829
135 changed files with 9647 additions and 34 deletions
+1
View File
@@ -18,3 +18,4 @@ __pycache__
node_modules
evidence
.DS_Store
docs
+67 -11
View File
@@ -1,25 +1,81 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Скопируйте в .env для локального окружения; .env не коммитится.
# Настройки приложения и Compose; параметры будущей очереди пока не используются.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
LOG_LEVEL=INFO
MAX_MESSAGES=100000
# Лимит одновременных запросов НА ОДНУ реплику
MAX_IN_FLIGHT=16
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
SESSION_STORE=redis
REDIS_HOST=redis
REDIS_PORT=6379
REDIS_DB=0
REDIS_MAX_CONNECTIONS=20
REDIS_POOL_TIMEOUT_SECONDS=1
REDIS_CONNECT_TIMEOUT_SECONDS=1
REDIS_SOCKET_TIMEOUT_SECONDS=0.5
DB_HOST=postgres
DB_PORT=5432
DB_NAME=messenger
DB_USER=messenger
# Пул на одну реплику: суммарно API_REPLICAS * (POOL_SIZE + MAX_OVERFLOW)
# соединений должно помещаться в max_connections PostgreSQL.
DB_POOL_SIZE=5
DB_MAX_OVERFLOW=10
DB_POOL_TIMEOUT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
# --- Балансировка и идемпотентность ---
API_REPLICAS=2
API_IMAGE=mtusi-messenger:lab5
API_BUILD_REVISION=development
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
SHUTDOWN_DRAIN_SECONDS=1
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5
READINESS_TIMEOUT_SECONDS=2
# Порт единственной публичной точки входа (nginx)
LB_HOST_PORT=18081
# Используются только тестовым стендом (compose.test.yaml)
DB_HOST_PORT=25432
REDIS_HOST_PORT=26379
# --- Lab4: MinIO, ограниченный upload и очистка журнала ---
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
S3_CONNECT_TIMEOUT_SECONDS=2
S3_READ_TIMEOUT_SECONDS=10
S3_UPLOAD_TIMEOUT_SECONDS=30
S3_DOWNLOAD_TIMEOUT_SECONDS=60
S3_IMAGE_TIMEOUT_SECONDS=10
S3_MAX_UPLOAD_BYTES=10485760
S3_MAX_IMAGE_PIXELS=40000000
S3_CHUNK_BYTES=65536
S3_MAX_CONCURRENT_UPLOADS=2
S3_CLEANUP_GRACE_SECONDS=86400
S3_CLEANUP_INTERVAL_SECONDS=300
# Пути секретов внутри контейнеров; должны совпадать с secrets mounts.
POSTGRES_PASSWORD_FILE=/run/secrets/db_password
MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
MINIO_BROWSER=off
# Lab5: будущие параметры очереди/worker, пока без реализации.
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Брокер пока не выбран; файл ещё не создаётся и не монтируется.
BROKER_URL_FILE=/run/secrets/broker_url
+6
View File
@@ -28,5 +28,11 @@ coverage/
.DS_Store
.idea/
.vscode/
venv/
.pytest_cache/
.ruff_cache/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
# Private working documentation
/docs/
+31
View File
@@ -0,0 +1,31 @@
FROM python:3.12.15-slim-bookworm
LABEL io.mtusi.messenger.secrets="file-v1"
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
RUN useradd --system --uid 10001 --no-create-home appuser
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY alembic.ini ./
COPY migrations ./migrations
COPY app ./app
ARG API_BUILD_REVISION
LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
CMD ["python", "-m", "app"]
+202 -4
View File
@@ -1,13 +1,86 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
-include .env
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard venv/bin/python),venv/bin/python,python3))
PYTEST_ARGS ?=
BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080)
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
COMPOSE = docker compose --env-file .env
# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis
# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh.
TEST_PROJECT ?= mtusi-messenger-lab5-test
TEST_LB_PORT ?= 28080
TEST_DB_PORT ?= $(DB_HOST_PORT)
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
TEST_BASE_URL = http://localhost:$(TEST_LB_PORT)
TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT)
TEST_COMPOSE = $(TEST_ENV) $(COMPOSE)
TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file <secrets/db_password),$(DB_PASSWORD))
TEST_DATABASE_URL ?= postgresql+asyncpg://$(DB_USER):$(TEST_DB_PASSWORD)@localhost:$(TEST_DB_PORT)/$(DB_NAME)_test
TEST_REDIS_HOST ?= localhost
TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/redis_password),$(REDIS_PASSWORD))
TEST_REDIS_DB ?= 15
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
define wait_ready
@for i in $$(seq 1 90); do \
curl -sf "$(1)/health/ready" >/dev/null && exit 0; sleep 1; \
done; echo "API $(1) не готов за 90 секунд" >&2; exit 1
endef
.PHONY: help check validate test load test-persistence test-failover \
init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \
up down restart restore ps logs migrate migration instances \
test-stand-up test-stand-down test-stand-destroy test-stand-ps \
test-stand-smoke test-stand-persistence test-stand-failover \
test-own test-docker test-limits test-concurrency test-migrations test-all \
explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \
rolling-update failover-all test-lab3-grade4
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply"
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
@echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps"
@echo " test-stand-smoke | test-stand-persistence | test-stand-failover"
@echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all"
@echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)"
@echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)"
@echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)"
@echo "База lab3: rolling-update | failover-all"
@echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)"
@echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)"
# --- Проверки API и сценариев отказа (tests/) ---
init:
$(PYTHON) scripts/init_local.py
test-grade4:
$(MAKE) check validate
$(MAKE) test-stand-up
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
$(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults
$(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
test-s3:
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
load-uploads:
$(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10
cleanup-dry-run:
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once
cleanup-apply:
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once --apply
check:
$(PYTHON) scripts/check_contract.py
@@ -15,6 +88,22 @@ check:
validate:
$(PYTHON) scripts/validate_openapi.py
test-nginx:
$(PYTHON) scripts/check_nginx.py
test-lab3-grade3:
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
$(MAKE) check validate; \
$(MAKE) test-stand-up; \
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
test-lab3-grade4:
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
$(MAKE) check validate; \
$(MAKE) test-stand-up; \
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +117,112 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- Основной стенд ---
rolling-update:
$(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS)
failover-all:
$(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS)
up:
$(COMPOSE) up -d --build
$(call wait_ready,$(BASE_URL))
down:
$(COMPOSE) down
restart:
scripts/restart.sh
$(call wait_ready,$(BASE_URL))
# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры).
restore:
$(COMPOSE) up -d --wait api
ps:
$(COMPOSE) ps
logs:
$(COMPOSE) logs -f api nginx
migrate:
$(COMPOSE) run --rm --build migrate
# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id).
instances:
@for i in $$(seq 1 10); do \
curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \
done
# Генерация миграции против БД тестового стенда (основной стенд БД не публикует).
migration:
DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)"
# --- Тестовый стенд ---
test-stand-up:
$(TEST_COMPOSE) up -d --build
$(call wait_ready,$(TEST_BASE_URL))
$(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
test-stand-down:
$(TEST_COMPOSE) down
test-stand-destroy:
$(TEST_COMPOSE) down -v
test-stand-ps:
$(TEST_COMPOSE) ps
test-stand-smoke:
$(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)"
test-stand-idempotency:
$(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)"
test-stand-persistence:
$(TEST_COMPOSE) ps
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)"
$(TEST_COMPOSE) ps
test-stand-failover:
$(TEST_COMPOSE) ps
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)"
$(TEST_COMPOSE) ps -a api
$(TEST_COMPOSE) up -d --wait api
# --- Собственные тесты (student-tests/) ---
test-own:
$(PYTHON) -m pytest -q $(PYTEST_ARGS)
test-docker:
$(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS)
test-limits:
$(PYTHON) -m pytest student-tests/test_limits.py -q
test-concurrency:
$(PYTHON) -m pytest student-tests/test_concurrency.py -q
test-migrations:
$(PYTHON) -m pytest student-tests/test_migrations.py -q
test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence
# --- Демо (на тестовом стенде) ---
explain:
$(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py
expiry-demo: test-stand-up
$(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api
$(call wait_ready,$(TEST_BASE_URL))
BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh
$(TEST_COMPOSE) up -d api
$(call wait_ready,$(TEST_BASE_URL))
failure-demo: test-stand-up
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
+57 -19
View File
@@ -1,37 +1,75 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4).
- Целевая оценка: 5; исследовательский трек пока не выбран.
- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5.
- Дата переноса: 07.11.2026.
- Commit/PR lab5: заполнить после публикации.
- Использованные источники: README-файлы и contracts/ репозитория lab5,
исходники lab4. Использована помощь Codex. Новых зависимостей нет.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
make up
```
HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных.
Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`.
Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда,
`docker compose --env-file .env down --volumes` для основного.
Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4.
Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии,
идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные
тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO
bootstrap из исходного рабочего дерева. Исходный lab4 не изменён.
Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки —
.env.example; там же указаны будущие параметры очереди, пока не используемые.
Compose получает настройки environment из .env, включая пути S3, PostgreSQL
и MinIO к файлам секретов; секретные значения остаются в secrets/.
Пути *_FILE в Compose также задают точки монтирования соответствующих secrets;
MinIO bootstrap читает эти пути из окружения.
Публичные tests/, contracts/, lab.json и README.md lab5 сохранены.
Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher →
broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL.
Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready.
Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет |
| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 |
| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет |
| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет |
| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет |
| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет |
| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет |
| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет |
| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты,
версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность,
прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready,
RSS и другие метрики выбранного трека.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и
асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся.
Сначала просмотр пользователем и выбор трека/брокера, затем реализация;
тестовый прогон и новые evidence — по отдельному указанию после реализации.
+38
View File
@@ -0,0 +1,38 @@
[alembic]
script_location = migrations
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
View File
+4
View File
@@ -0,0 +1,4 @@
from app.main import main
if __name__ == "__main__":
main()
View File
+60
View File
@@ -0,0 +1,60 @@
"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис."""
import argparse
import asyncio
from dataclasses import asdict
import json
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.s3 import S3Settings
from app.domain.attachments import reconcile_attachments
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
from app.storage.s3 import create_s3_storage
async def cleanup_once(factory, storage, apply: bool) -> bool:
completed = True
try:
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
print(json.dumps(asdict(result)), flush=True)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
return completed
async def run(once: bool, apply: bool) -> None:
settings = Settings() # type: ignore
s3_settings = S3Settings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(s3_settings)
try:
while True:
completed = await cleanup_once(factory, storage, apply)
if once:
if not completed:
raise SystemExit(1)
break
await asyncio.sleep(s3_settings.cleanup_interval_seconds)
finally:
await storage.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
parser.add_argument("--once", action="store_true")
parser.add_argument(
"--apply",
action="store_true",
help="удалять безопасные кандидаты; по умолчанию dry-run",
)
args = parser.parse_args()
asyncio.run(run(args.once, args.apply))
if __name__ == "__main__":
main()
+129
View File
@@ -0,0 +1,129 @@
from enum import StrEnum
from pathlib import Path
from typing import Literal, Self
from pydantic import Field, SecretStr, model_validator
from pydantic_settings import (
BaseSettings,
PydanticBaseSettingsSource,
SettingsConfigDict,
)
from sqlalchemy import URL
from app.core.s3 import S3Settings
ASYNC_DRIVER = "postgresql+asyncpg"
SECRETS_DIR = next(
(
path
for path in (
Path("/run/secrets"),
Path(__file__).resolve().parents[2] / "secrets",
)
if (path / "db_password").is_file()
),
None,
)
class LogLevel(StrEnum):
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
CRITICAL = "CRITICAL"
class Settings(BaseSettings):
s3_enabled: bool = True
s3: S3Settings = Field(default_factory=S3Settings)
app_host: str
app_port: int = Field(ge=1, le=65535)
app_env: str
log_level: LogLevel = LogLevel.INFO
max_messages: int = Field(ge=1)
max_in_flight: int = Field(ge=1)
session_ttl_seconds: int = Field(default=3600, ge=1)
session_store: Literal["redis"] = "redis"
redis_host: str = "localhost"
redis_port: int = Field(default=6379, ge=1, le=65535)
redis_db: int = Field(default=0, ge=0)
redis_password: SecretStr | None = None
redis_max_connections: int = Field(default=20, ge=1)
redis_pool_timeout_seconds: float = Field(default=1.0, gt=0)
redis_connect_timeout_seconds: float = Field(default=1.0, gt=0)
redis_socket_timeout_seconds: float = Field(default=0.5, gt=0)
database_url: SecretStr | None = None
db_host: str | None = None
db_port: int = Field(default=5432, ge=1, le=65535)
db_name: str | None = None
db_user: str | None = None
db_password: SecretStr | None = None
db_pool_size: int = Field(default=5, ge=1)
db_max_overflow: int = Field(default=5, ge=0)
db_pool_timeout_seconds: float = Field(default=3.0, gt=0)
db_connect_timeout_seconds: int = Field(default=3, ge=1)
db_statement_timeout_ms: int = Field(default=5000, ge=1)
instance_id: str | None = None
idempotency_ttl_seconds: int = Field(default=86400, ge=1)
shutdown_drain_seconds: float = Field(default=1, ge=0)
graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1)
readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5)
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore",
secrets_dir=SECRETS_DIR,
)
@classmethod
def settings_customise_sources(
cls,
settings_cls: type[BaseSettings],
init_settings: PydanticBaseSettingsSource,
env_settings: PydanticBaseSettingsSource,
dotenv_settings: PydanticBaseSettingsSource,
file_secret_settings: PydanticBaseSettingsSource,
) -> tuple[PydanticBaseSettingsSource, ...]:
"""Явные параметры и файлы имеют приоритет над окружением."""
return init_settings, file_secret_settings, env_settings, dotenv_settings
@model_validator(mode="after")
def _require_database_source(self) -> Self:
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
raise ValueError(
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
)
if self.database_url is not None:
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
return self
if not (self.db_host and self.db_name and self.db_user and self.db_password):
raise ValueError(
"Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password"
)
return self
def get_database_url(self) -> str:
if self.database_url is not None:
return self.database_url.get_secret_value()
assert self.db_password is not None
url = URL.create(
ASYNC_DRIVER,
username=self.db_user,
password=self.db_password.get_secret_value(),
host=self.db_host,
port=self.db_port,
database=self.db_name,
)
return url.render_as_string(hide_password=False)
+28
View File
@@ -0,0 +1,28 @@
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from app.core.config import Settings
def create_db_engine(settings: Settings) -> AsyncEngine:
return create_async_engine(
settings.get_database_url(),
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout_seconds,
pool_pre_ping=True,
connect_args={
"timeout": settings.db_connect_timeout_seconds,
"server_settings": {
"statement_timeout": str(settings.db_statement_timeout_ms)
},
},
)
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
+24
View File
@@ -0,0 +1,24 @@
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
from pathlib import Path
import resource
import sys
from app.domain.image_validation import decode_image
from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code
def main() -> int:
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
path, content_type, maximum = sys.argv[1:]
status = 0
try:
decode_image(Path(path), content_type, int(maximum))
except IMAGE_WORKER_ERRORS as error:
status = image_worker_error_code(error)
return status
if __name__ == "__main__":
raise SystemExit(main())
+40
View File
@@ -0,0 +1,40 @@
"""Идентификатор API-реплики и заголовок X-Instance-Id."""
import os
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
INSTANCE_ID_HEADER = "X-Instance-Id"
def resolve_instance_id(configured: str | None) -> str:
"""Выбирает INSTANCE_ID, HOSTNAME или UUID при старте приложения."""
if configured and configured.strip():
return configured.strip()
hostname = os.getenv("HOSTNAME", "").strip()
if hostname:
return hostname
return str(uuid4())
class InstanceIdMiddleware(BaseHTTPMiddleware):
"""Добавляет сохранённый идентификатор реплики в ответ."""
def __init__(self, app, instance_id: str) -> None:
super().__init__(app)
self.instance_id = instance_id
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
response = await call_next(request)
response.headers[INSTANCE_ID_HEADER] = self.instance_id
return response
+17
View File
@@ -0,0 +1,17 @@
"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker.
Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная
concurrency/prefetch, восстановление pending и остановка с сохранением работы.
Каждая задача получает отдельную AsyncSession. Обработка изображений идёт
в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется.
В Compose и API модуль пока не подключён.
"""
async def run_worker() -> None:
"""Запустить consumer с lease/retry, ограничением ресурсов и корректным ack.
SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка
worker не заставляет API выполнять resize. Инициализация ресурсов — здесь.
"""
raise NotImplementedError("Lab5: отдельный worker ещё не реализован")
+50
View File
@@ -0,0 +1,50 @@
"""Готовность реплики и мягкое завершение сервера."""
import asyncio
from types import FrameType
import uvicorn
class Lifecycle:
"""Хранит состояние завершения реплики."""
def __init__(self) -> None:
self._draining = False
@property
def draining(self) -> bool:
return self._draining
def start_draining(self) -> None:
"""Отключает приём новых бизнес-запросов."""
self._draining = True
class GracefulServer(uvicorn.Server):
"""Оставляет время для исключения завершающейся реплики."""
def __init__(
self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float
) -> None:
super().__init__(config)
self.lifecycle = lifecycle
self.drain_seconds = drain_seconds
self._drain_task: asyncio.Task[None] | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
"""Первый сигнал запускает drain, повторный ускоряет остановку."""
if self._drain_task is not None:
if not self._drain_task.done():
self._drain_task.cancel()
super().handle_exit(sig, frame)
return
self.lifecycle.start_draining()
self._drain_task = asyncio.get_running_loop().create_task(
self._finish_draining(sig, frame)
)
async def _finish_draining(self, sig: int, frame: FrameType | None) -> None:
"""Передаёт остановку Uvicorn после drain-периода."""
await asyncio.sleep(self.drain_seconds)
super().handle_exit(sig, frame)
+15
View File
@@ -0,0 +1,15 @@
"""Заготовка процесса доставки PostgreSQL outbox в durable broker.
Не держать транзакцию/блокировки во время бесконечного сетевого ожидания.
Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish.
Процесс пока не подключён к Compose или жизненному циклу API.
"""
async def run_publisher() -> None:
"""Публиковать сохранённые намерения и фиксировать подтверждение брокера.
После crash между publish и отметкой допустим дубль того же job_id.
Недоступность брокера не должна терять уже принятые задания.
"""
raise NotImplementedError("Lab5: outbox publisher ещё не реализован")
+25
View File
@@ -0,0 +1,25 @@
from redis.asyncio import BlockingConnectionPool, Redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
)
pool = BlockingConnectionPool(
host=settings.redis_host,
port=settings.redis_port,
db=settings.redis_db,
password=password,
max_connections=settings.redis_max_connections,
timeout=settings.redis_pool_timeout_seconds,
socket_connect_timeout=settings.redis_connect_timeout_seconds,
socket_timeout=settings.redis_socket_timeout_seconds,
retry=Retry(NoBackoff(), retries=1),
decode_responses=True,
)
return Redis.from_pool(pool)
+42
View File
@@ -0,0 +1,42 @@
from collections.abc import AsyncIterator
from fastapi import Request
from app.errors.exceptions import rate_limited, unavailable
class RequestCapacity:
def __init__(self, max_in_flight: int) -> None:
self._max_in_flight = max_in_flight
self._in_flight = 0
@property
def in_flight(self) -> int:
return self._in_flight
def try_acquire(self) -> bool:
if self._in_flight >= self._max_in_flight:
return False
self._in_flight += 1
return True
def release(self) -> None:
self._in_flight -= 1
def create_request_capacity(max_in_flight: int) -> RequestCapacity:
return RequestCapacity(max_in_flight)
async def acquire_request_slot(request: Request) -> AsyncIterator[None]:
if request.app.state.lifecycle.draining:
raise unavailable("Реплика завершает работу")
capacity: RequestCapacity = request.app.state.request_capacity
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
+22
View File
@@ -0,0 +1,22 @@
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
REQUEST_ID_HEADER = "X-Request-Id"
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
request_id = str(uuid4())
request.state.request_id = request_id
response = await call_next(request)
response.headers[REQUEST_ID_HEADER] = request_id
return response
+35
View File
@@ -0,0 +1,35 @@
"""Конфигурация MinIO; значения credentials читаются только из файлов."""
from pathlib import Path
from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
bucket: str = "messenger-attachments"
access_key_id_file: Path = Path("/run/secrets/s3_access_key_id")
secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key")
connect_timeout_seconds: float = Field(default=2, gt=0, le=5)
read_timeout_seconds: float = Field(default=10, gt=0, le=15)
upload_timeout_seconds: float = Field(default=30, gt=0, le=60)
download_timeout_seconds: float = Field(default=60, gt=0, le=120)
image_timeout_seconds: float = Field(default=10, gt=0, le=15)
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144)
max_concurrent_uploads: int = Field(default=2, ge=1, le=2)
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
return self
+63
View File
@@ -0,0 +1,63 @@
"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий."""
import asyncio
import hashlib
import os
import secrets
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
TOKEN_BYTES = 32
_hasher = PasswordHasher()
_executor = ThreadPoolExecutor(
max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)),
thread_name_prefix="argon2",
)
def _hash_password_sync(password: str) -> str:
"""Блокирующий расчёт Argon2id-хеша со случайной солью в формате PHC."""
return _hasher.hash(password)
def _verify_password_sync(password: str, password_hash: str) -> bool:
"""Блокирующая проверка пароля по PHC-строке; False при несовпадении или повреждённом хеше."""
try:
return _hasher.verify(password_hash, password)
except (VerificationError, InvalidHashError):
return False
_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES))
async def hash_password(password: str) -> str:
"""Хеширует пароль в пуле потоков, не блокируя event loop."""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(_executor, _hash_password_sync, password)
async def verify_password(password: str, password_hash: str) -> bool:
"""Проверяет пароль в пуле потоков, не блокируя event loop."""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
_executor, _verify_password_sync, password, password_hash
)
async def dummy_verify_password(password: str) -> None:
"""Проверка против фиктивного хеша: время ответа не выдаёт отсутствие пользователя."""
await verify_password(password, _DUMMY_HASH)
def generate_token() -> str:
"""Непрозрачный токен сессии: 32 случайных байта в base64url."""
return secrets.token_urlsafe(TOKEN_BYTES)
def hash_token(token: str) -> str:
"""SHA-256 токена в hex: детерминированный ключ для хранилища сессий."""
return hashlib.sha256(token.encode()).hexdigest()
+1
View File
@@ -0,0 +1 @@
"""Бизнес-логика: проверки доступа, транзакции и ошибки API."""
+45
View File
@@ -0,0 +1,45 @@
"""Заготовки сценариев lab5; не подключены к upload или worker.
Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются
в storage/. Одна AsyncSession принадлежит одной последовательной задаче.
"""
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID:
"""Создать job и outbox в транзакции queued metadata после durable staging.
Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload;
202 допустим только после устойчивого оригинала и успешного commit.
"""
raise NotImplementedError("Lab5: постановка задания ещё не реализована")
async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None:
"""Захватить job с lease и опубликовать original и обязательные варианты.
Ready-дубль не меняет результат. Lease и версия защищают от устаревшей
попытки. S3/metadata фиксируются до ack; отмена не подавляется.
"""
raise NotImplementedError("Lab5: обработка задания ещё не реализована")
async def recover_expired_jobs(session: AsyncSession) -> int:
"""Вернуть зависший processing в очередь после истечения lease.
Учитывать лимит попыток и защиту от конкурентного завершения старым worker.
Точное правило heartbeat/deadline определяется до реализации.
"""
raise NotImplementedError("Lab5: восстановление lease ещё не реализовано")
async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None:
"""Явно повторить только failed job, сохранив аудит и ключ логического эффекта.
Не переисполнять ready. Проверка допустимости, лимиты повторов и способ
операторского вызова будут определены вместе с реестром failed/DLQ.
"""
raise NotImplementedError("Lab5: replay ещё не реализован")
+315
View File
@@ -0,0 +1,315 @@
"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
from collections.abc import AsyncIterator, Iterator
from contextlib import contextmanager, suppress
from dataclasses import dataclass
import hashlib
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from anyio import CancelScope
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.request_capacity import RequestCapacity
from app.domain.chats import ensure_chat_member
from app.domain.images import validate_filename, validate_image
from app.errors.exceptions import (
conflict,
invalid_request,
not_found,
payload_too_large,
rate_limited,
unavailable,
unsupported_media_type,
)
from app.errors.handlers import (
RECOVERABLE_STORAGE_ERRORS,
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class CleanupResult:
examined: int
candidates: int
deleted: int
@dataclass(frozen=True)
class OriginalFile:
path: Path
size_bytes: int
sha256: str
def require_storage(storage: MinioStorage | None) -> MinioStorage:
if storage is None:
raise unavailable("Объектное хранилище не настроено")
return storage
def validate_declared_size(value: str | None, maximum: int) -> None:
if value is not None:
if not value.isascii() or not value.isdecimal() or len(value) > 20:
raise invalid_request("Некорректный Content-Length")
if int(value) > maximum:
raise payload_too_large()
@contextmanager
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
async def receive_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
chat_id: UUID,
*,
capacity: RequestCapacity,
filename: str,
content_type: str,
declared_length: str | None,
chunks: AsyncIterator[bytes],
) -> Attachment:
storage = require_storage(storage)
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
timeout_error = invalid_request("Истекло время загрузки")
with upload_slot(capacity):
async with operation_timeout(
storage.settings.upload_timeout_seconds, timeout_error
):
result = await upload_attachment(
session,
storage,
user,
chat_id,
filename=filename,
content_type=content_type,
chunks=chunks,
)
return result
async def read_original(
chunks: AsyncIterator[bytes],
path: Path,
maximum: int,
chunk_bytes: int,
) -> OriginalFile:
checksum = hashlib.sha256()
size = 0
with path.open("wb") as target:
async for chunk in chunks:
size += len(chunk)
if size > maximum:
raise payload_too_large()
checksum.update(chunk)
view = memoryview(chunk)
for offset in range(0, len(view), chunk_bytes):
target.write(view[offset : offset + chunk_bytes])
if size == 0:
raise unsupported_media_type("Пустой файл")
return OriginalFile(path, size, checksum.hexdigest())
async def compensate_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
await session.rollback()
intent = await repository.lock_upload(session, id)
if intent is not None:
if await repository.get_by_key(session, intent.object_key) is None:
await storage.delete_object(intent.object_key)
await repository.finish_upload(session, id)
await session.commit()
async def recover_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
try:
await compensate_upload(session, storage, id)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Upload compensation", error)
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
with suppress(*RECOVERABLE_STORAGE_ERRORS):
await session.rollback()
async def publish_original(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
original: OriginalFile,
filename: str,
content_type: str,
) -> Attachment:
id = uuid4()
key = f"originals/{uuid4().hex}"
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
await repository.reserve_upload(session, id, key)
await session.commit()
try:
if await repository.lock_upload(session, id) is None:
raise unavailable("Загрузка отменена до записи оригинала")
await storage.put_file(
key,
original.path,
size_bytes=original.size_bytes,
content_type=content_type,
sha256=original.sha256,
)
record = await repository.create_attachment(
session,
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=key,
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await repository.finish_upload(session, id)
result = Attachment.model_validate(record)
await session.commit()
except BaseException:
# Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT.
with CancelScope(shield=True):
await recover_upload(session, storage, id)
raise
return result
async def upload_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
filename: str,
content_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
await ensure_chat_member(session, user, chat_id)
validate_filename(filename)
if content_type not in {"image/png", "image/jpeg"}:
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
# Не держим соединение PostgreSQL во время чтения медленного клиента.
await session.rollback()
settings = storage.settings
with TemporaryDirectory(prefix="messenger-upload-") as directory:
original = await read_original(
chunks,
Path(directory) / "original",
settings.max_upload_bytes,
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await publish_original(
session,
storage,
user,
chat_id,
original=original,
filename=filename,
content_type=content_type,
)
return result
async def get_attachment(
session: AsyncSession, user: User, attachment_id: UUID
) -> Attachment:
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
await ensure_chat_member(session, user, record.chat_id)
return Attachment.model_validate(record)
async def download_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[Attachment, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if variant != "original":
raise not_found()
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
key = record.object_key
await session.rollback()
info = await storage.stat_object(key)
if (
info is None
or info.size_bytes != result.size_bytes
or info.content_type != result.content_type
):
raise unavailable("Оригинал временно недоступен")
stream = await storage.open_object(key)
return result, stream
async def validate_message_attachments(
session: AsyncSession,
user: User,
chat_id: UUID,
attachment_ids: list[UUID],
) -> None:
await ensure_chat_member(session, user, chat_id)
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status != "ready" for record in records):
raise conflict("Вложение ещё не готово")
async def reconcile_attachments(
session: AsyncSession,
storage: MinioStorage,
*,
batch_size: int = 20,
dry_run: bool = True,
) -> CleanupResult:
if not 1 <= batch_size <= 100:
raise invalid_request("batch_size должен быть от 1 до 100")
intents = await repository.stale_uploads(
session, storage.settings.cleanup_grace_seconds, batch_size
)
candidates = deleted = 0
for intent in intents:
if await repository.get_by_key(session, intent.object_key) is None:
candidates += 1
if not dry_run:
await storage.delete_object(intent.object_key)
deleted += 1
if not dry_run:
await repository.finish_upload(session, intent.id)
if dry_run:
await session.rollback()
else:
await session.commit()
return CleanupResult(len(intents), candidates, deleted)
+84
View File
@@ -0,0 +1,84 @@
import re
from datetime import UTC, datetime, timedelta
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.domain.users import to_user_schema
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.session import AuthSession
from app.schemas.user import (
PASSWORD_MAX_LENGTH,
PASSWORD_MIN_LENGTH,
USERNAME_PATTERN,
User,
)
from app.storage import sessions as sessions_storage
from app.storage import users as users_storage
USERNAME_RE = re.compile(USERNAME_PATTERN)
async def create_session(
session: AsyncSession,
redis: Redis,
username: str,
password: str,
ttl_seconds: int,
) -> AuthSession:
"""Проверяет учётные данные и выдаёт новый Bearer-токен."""
if (
USERNAME_RE.fullmatch(username) is None
or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH
or not password.strip()
):
raise unauthorized_basic()
user = await users_storage.get_user_by_username(session, username)
await session.commit()
if user is None:
await security.dummy_verify_password(password)
raise unauthorized_basic()
if not await security.verify_password(password, user.password_hash):
raise unauthorized_basic()
token = security.generate_token()
expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
await sessions_storage.create_session(
redis,
token_hash=security.hash_token(token),
user_id=user.id,
expires_at=expires_at,
)
auth_session = AuthSession(
token=token, expires_at=expires_at, user=to_user_schema(user)
)
return auth_session
async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User:
"""Возвращает владельца действующей сессии или бросает 401."""
user_id = await sessions_storage.get_session_user_id(
redis, security.hash_token(token)
)
if user_id is None:
raise unauthorized_bearer()
user = await users_storage.get_user_by_id(session, user_id)
if user is None:
raise unauthorized_bearer()
current_user = to_user_schema(user)
return current_user
async def revoke_session(redis: Redis, token: str) -> None:
"""Отзывает текущую сессию; повторный отзыв даёт 401."""
is_deleted = await sessions_storage.delete_session(
redis, security.hash_token(token)
)
if not is_deleted:
raise unauthorized_bearer()
+104
View File
@@ -0,0 +1,104 @@
from uuid import UUID
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.pagination import (
KIND_CHATS,
build_next_cursor,
decode_cursor,
split_page,
)
from app.errors.exceptions import invalid_request, not_found
from app.models.chat import Chat as ChatModel
from app.schemas.chat import Chat, ChatPage, CreateChat
from app.schemas.user import User
from app.storage import chats as chats_storage
from app.storage import users as users_storage
from app.storage.types import Position
UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя"
def to_chat_schema(chat: ChatModel) -> Chat:
"""Преобразует ORM-модель чата с загруженными участниками в схему."""
chat_schema = Chat(
id=chat.id,
title=chat.title,
member_ids=[member.user_id for member in chat.members],
created_at=chat.created_at,
)
return chat_schema
def validate_new_chat(creator: User, new_chat: CreateChat) -> None:
"""Проверяет состав участников без обращения к базе."""
member_ids = new_chat.member_ids
if creator.id not in member_ids:
raise invalid_request("Создатель чата должен входить в member_ids")
if len(set(member_ids)) != len(member_ids):
raise invalid_request("member_ids содержит дубликаты")
async def create_chat(
session: AsyncSession, creator: User, new_chat: CreateChat
) -> Chat:
"""Создаёт чат с участниками атомарно; неверный состав даёт 400."""
validate_new_chat(creator, new_chat)
existing_ids = await users_storage.get_existing_user_ids(
session, new_chat.member_ids
)
if existing_ids != set(new_chat.member_ids):
raise invalid_request(UNKNOWN_MEMBER_MESSAGE)
try:
chat = await chats_storage.create_chat(
session, title=new_chat.title, member_ids=new_chat.member_ids
)
await session.commit()
except IntegrityError:
await session.rollback()
raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None
created_chat = to_chat_schema(chat)
return created_chat
async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat:
"""Возвращает чат участнику; чужой и несуществующий чат дают 404."""
chat = await chats_storage.get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise not_found()
found_chat = to_chat_schema(chat)
return found_chat
async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None:
"""Бросает 404, если пользователь не состоит в чате или чата нет."""
if not await chats_storage.is_chat_member(session, chat_id, user.id):
raise not_found()
async def list_chats(
session: AsyncSession,
user: User,
cursor: str | None,
limit: int,
) -> ChatPage:
"""Возвращает страницу чатов пользователя по (created_at, id)."""
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_CHATS, user.id)
rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
items = [to_chat_schema(chat) for chat in page_rows]
page = ChatPage(
items=items,
next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next),
)
return page
+33
View File
@@ -0,0 +1,33 @@
"""Условия готовности реплики и ограниченная проверка зависимостей."""
import asyncio
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncEngine
from app.core.lifecycle import Lifecycle
from app.errors.exceptions import unavailable
from app.errors.handlers import operation_timeout
from app.schemas.base import Health
from app.storage.health import check_database, check_redis
from app.storage.s3 import MinioStorage
def liveness() -> Health:
return Health(status='ok')
async def readiness(
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
lifecycle: Lifecycle, timeout_seconds: float,
) -> Health:
if lifecycle.draining:
raise unavailable()
checks = [check_database(engine), check_redis(redis)]
if storage is not None:
checks.append(storage.check_available())
async with operation_timeout(timeout_seconds, unavailable()):
dependencies = await asyncio.gather(*checks, return_exceptions=True)
if not all(result is True for result in dependencies):
raise unavailable()
return Health(status='ok')
+72
View File
@@ -0,0 +1,72 @@
"""Атомарная отправка сообщения с защитой от повторов."""
import hashlib
import json
import re
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.attachments import validate_message_attachments
from app.errors.exceptions import conflict, invalid_request, unavailable
from app.schemas.message import CreateMessage, Message
from app.schemas.user import User
from app.storage import idempotency as key_storage
from app.storage import messages as message_storage
from app.storage import attachments as attachment_storage
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def validate_idempotency_key(key: str | None) -> str:
"""Проверяет обязательный ASCII-ключ запроса."""
if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None:
raise invalid_request("Требуется корректный Idempotency-Key")
return key
def request_fingerprint(new_message: CreateMessage) -> str:
"""Хеширует нормализованное содержимое запроса."""
body = new_message.model_dump(mode="json")
# Сохраняет отпечатки текстовых запросов lab3 и равенство absent / [].
if not body["attachment_ids"]:
body.pop("attachment_ids")
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
return hashlib.sha256(canonical.encode()).hexdigest()
async def send_message_idempotent(
session: AsyncSession,
sender: User,
chat_id: UUID,
new_message: CreateMessage,
idempotency_key: str,
max_messages: int,
ttl_seconds: int,
) -> Message:
"""Создаёт сообщение или возвращает результат прежнего запроса."""
await ensure_chat_member(session, sender, chat_id)
scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key)
fingerprint = request_fingerprint(new_message)
await key_storage.lock_key(session, scope)
previous = await key_storage.get_active_key(session, scope, ttl_seconds)
if previous is not None:
if previous.request_hash != fingerprint:
raise conflict("Idempotency-Key уже использован для другого сообщения")
message = await message_storage.get_message(session, previous.message_id)
else:
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
if not await message_storage.reserve_message_slot(session, max_messages):
raise unavailable("Достигнут лимит количества сообщений")
message = await message_storage.create_message(
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
)
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
await key_storage.save_result(session, scope, fingerprint, message.id)
links = await attachment_storage.message_attachment_ids(session, [message.id])
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
await session.commit()
return result
+34
View File
@@ -0,0 +1,34 @@
"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP.
Pipeline запускается внутри ограниченного процесса worker. Исходные bytes
не перекодируются; варианты удаляют метаданные по документированной политике.
Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md.
"""
from pathlib import Path
def create_thumbnail(source: Path, destination: Path) -> None:
"""Создать JPEG, вписанный в 256×256 без увеличения.
Применить EXIF orientation, alpha положить на белый; округлять размеры
вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160.
"""
raise NotImplementedError("Lab5: thumbnail ещё не реализован")
def create_crop(source: Path, destination: Path) -> None:
"""Создать центрированный квадрат JPEG 128×128; увеличение допустимо.
Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail.
"""
raise NotImplementedError("Lab5: crop ещё не реализован")
def create_optimized(source: Path, destination: Path) -> None:
"""Создать WebP, вписанный в 1280×1280 без увеличения.
Удалить метаданные и применить описанную политику качества. Уменьшение
размера любого маленького исходника не гарантируется контрактом.
"""
raise NotImplementedError("Lab5: optimized ещё не реализован")
+23
View File
@@ -0,0 +1,23 @@
"""Синхронная проверка изображения; вызывается только из ограниченного процесса."""
from pathlib import Path
import warnings
from PIL import Image
from app.errors.exceptions import payload_too_large, unsupported_media_type
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
Image.MAX_IMAGE_PIXELS = max_pixels
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(path, formats=["PNG", "JPEG"]) as image:
if image.width * image.height > max_pixels:
raise payload_too_large("Превышен лимит пикселей изображения")
image_format = image.format
if image_format is None or Image.MIME.get(image_format) != content_type:
raise unsupported_media_type("Содержимое не соответствует Content-Type")
image.verify()
# Для JPEG одной verify() недостаточно: декодировать весь оригинал.
with Image.open(path, formats=["PNG", "JPEG"]) as image:
image.load()
+48
View File
@@ -0,0 +1,48 @@
"""Проверка имени и запуск ограниченного декодера."""
import asyncio
from pathlib import Path
import sys
import unicodedata
from anyio import CancelScope
from app.core.s3 import S3Settings
from app.errors.exceptions import (
image_validation_error,
invalid_request,
payload_too_large,
)
from app.errors.handlers import operation_timeout
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or any(
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
for char in filename
):
raise invalid_request("Недопустимое имя файла")
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.core.image_worker",
str(path),
content_type,
str(settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
error = payload_too_large("Превышен бюджет проверки изображения")
try:
async with operation_timeout(settings.image_timeout_seconds, error):
code = await process.wait()
finally:
if process.returncode is None:
process.kill()
with CancelScope(shield=True):
await process.wait()
if code != 0:
raise image_validation_error(code)
+43
View File
@@ -0,0 +1,43 @@
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.pagination import (
KIND_MESSAGES,
build_next_cursor,
decode_cursor,
split_page,
)
from app.schemas.message import Message, MessagePage
from app.schemas.user import User
from app.storage import messages as messages_storage
from app.storage import attachments as attachments_storage
from app.storage.types import Position
async def list_messages(
session: AsyncSession,
user: User,
chat_id: UUID,
cursor: str | None,
limit: int,
) -> MessagePage:
"""Возвращает страницу истории чата по (created_at, id)."""
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_MESSAGES, chat_id)
await ensure_chat_member(session, user, chat_id)
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
items = [Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
) for message in page_rows]
page = MessagePage(
items=items,
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
)
return page
+83
View File
@@ -0,0 +1,83 @@
import base64
import json
from collections.abc import Sequence
from datetime import datetime
from typing import Protocol, TypeVar
from uuid import UUID
from app.errors.exceptions import invalid_request
from app.storage.types import Position
KIND_CHATS = "chats"
KIND_MESSAGES = "messages"
T = TypeVar("T")
class Positioned(Protocol):
"""Объект с позицией в порядке (created_at, id)."""
@property
def id(self) -> UUID: ...
@property
def created_at(self) -> datetime: ...
def encode_cursor(
kind: str,
resource_id: UUID,
created_at: datetime,
item_id: UUID,
) -> str:
"""Кодирует позицию и ресурс, к которому она привязана, в непрозрачный cursor."""
payload = {
"kind": kind,
"resource_id": str(resource_id),
"created_at": created_at.isoformat(),
"item_id": str(item_id),
}
raw = json.dumps(payload).encode()
return base64.urlsafe_b64encode(raw).decode()
def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position:
"""Возвращает позицию из cursor; битый или чужой cursor даёт 400."""
try:
payload = json.loads(base64.urlsafe_b64decode(cursor))
cursor_kind = payload["kind"]
cursor_resource_id = UUID(payload["resource_id"])
created_at = datetime.fromisoformat(payload["created_at"])
item_id = UUID(payload["item_id"])
except (ValueError, KeyError, TypeError, AttributeError):
raise invalid_request("Некорректный cursor") from None
if (
cursor_kind != kind
or cursor_resource_id != resource_id
or created_at.tzinfo is None
):
raise invalid_request("Некорректный cursor")
return created_at, item_id
def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]:
"""Отделяет страницу от лишней строки limit + 1, по которой видно продолжение."""
has_next = len(rows) > limit
items = rows[:limit]
return items, has_next
def build_next_cursor(
kind: str,
resource_id: UUID,
items: Sequence[Positioned],
has_next: bool,
) -> str | None:
"""Возвращает cursor на последний элемент страницы или None, если она последняя."""
if not has_next:
return None
last_item = items[-1]
return encode_cursor(kind, resource_id, last_item.created_at, last_item.id)
+41
View File
@@ -0,0 +1,41 @@
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.errors.exceptions import conflict
from app.models.user import User as UserModel
from app.schemas.user import CreateUser, User
from app.storage import users as users_storage
USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято"
def to_user_schema(user: UserModel) -> User:
"""Преобразует ORM-модель в публичную схему без password_hash."""
user_schema = User.model_validate(user)
return user_schema
async def register_user(session: AsyncSession, new_user: CreateUser) -> User:
"""Регистрирует пользователя; занятый username даёт 409."""
existing_user = await users_storage.get_user_by_username(session, new_user.username)
await session.commit()
if existing_user is not None:
raise conflict(USERNAME_TAKEN_MESSAGE)
password_hash = await security.hash_password(new_user.password)
try:
user = await users_storage.create_user(
session,
username=new_user.username,
display_name=new_user.display_name,
password_hash=password_hash,
)
await session.commit()
except IntegrityError:
await session.rollback()
raise conflict(USERNAME_TAKEN_MESSAGE) from None
created_user = to_user_schema(user)
return created_user
View File
+52
View File
@@ -0,0 +1,52 @@
"""HTTP-параметры и упаковка ответов; сценарии находятся в domain."""
from typing import Annotated
from urllib.parse import quote
from uuid import UUID
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from starlette.background import BackgroundTask
from app.domain import attachments
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
from app.schemas.attachment import Attachment
router = APIRouter(prefix='/api/v1', tags=['attachments'])
@router.post('/chats/{chat_id}/attachments', status_code=201)
async def upload(
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
storage: S3Storage, capacity: UploadCapacity,
filename: Annotated[str, Query(min_length=1, max_length=255)],
) -> Attachment:
return await attachments.receive_attachment(
session, storage, user, chat_id, capacity=capacity, filename=filename,
content_type=request.headers.get('content-type', '').lower(),
declared_length=request.headers.get('content-length'), chunks=request.stream(),
)
@router.get('/attachments/{attachment_id}')
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
return await attachments.get_attachment(session, user, attachment_id)
@router.get('/attachments/{attachment_id}/content')
async def content(
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
variant: Annotated[str, Query()] = 'original',
) -> StreamingResponse:
record, stream = await attachments.download_attachment(
session, storage, user, attachment_id, variant=variant,
)
return StreamingResponse(
stream, media_type=record.content_type,
headers={
'Content-Length': str(record.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
},
background=BackgroundTask(stream.aclose),
)
+32
View File
@@ -0,0 +1,32 @@
from fastapi import APIRouter, Response
from app.domain.auth import create_session, revoke_session
from app.endpoints.dependencies import (
AppSettings,
BasicCredentials,
BearerToken,
DbSession,
RedisClient,
)
from app.schemas.session import AuthSession
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
@router.post("/sessions", status_code=201)
async def login(
credentials: BasicCredentials,
session: DbSession,
redis: RedisClient,
settings: AppSettings,
) -> AuthSession:
username, password = credentials
return await create_session(
session, redis, username, password, settings.session_ttl_seconds
)
@router.delete("/sessions/current", status_code=204)
async def logout(token: BearerToken, redis: RedisClient) -> Response:
await revoke_session(redis, token)
return Response(status_code=204)
+32
View File
@@ -0,0 +1,32 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Query
from app.domain.chats import create_chat, get_chat, list_chats
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.chat import Chat, ChatPage, CreateChat
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
@router.post("", status_code=201)
async def create_new_chat(
new_chat: CreateChat, creator: CurrentUser, session: DbSession
) -> Chat:
return await create_chat(session, creator, new_chat)
@router.get("")
async def list_user_chats(
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> ChatPage:
return await list_chats(session, user, cursor, limit)
@router.get("/{chat_id}")
async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat:
return await get_chat(session, user, chat_id)
+112
View File
@@ -0,0 +1,112 @@
import base64
import binascii
from collections.abc import AsyncIterator
from typing import Annotated
from fastapi import Depends, Request, Security
from fastapi.openapi.models import HTTPBase, HTTPBearer
from fastapi.security.base import SecurityBase
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import Settings
from app.core.request_capacity import RequestCapacity
from app.domain.auth import authenticate_token
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.user import User
from app.storage.s3 import MinioStorage
async def get_settings(request: Request) -> Settings:
settings: Settings = request.app.state.settings
return settings
async def get_db(request: Request) -> AsyncIterator[AsyncSession]:
"""Одна AsyncSession на HTTP-запрос; незакоммиченная транзакция откатывается."""
session_factory = request.app.state.session_factory
async with session_factory() as session:
yield session
async def get_redis(request: Request) -> Redis:
redis: Redis = request.app.state.redis
return redis
def get_s3(request: Request) -> MinioStorage | None:
return request.app.state.s3
def get_upload_capacity(request: Request) -> RequestCapacity:
return request.app.state.upload_capacity
class AuthorizationHeader(SecurityBase):
"""Отдаёт сырой заголовок Authorization и описывает схему для OpenAPI."""
def __init__(self, model: HTTPBase, scheme_name: str) -> None:
self.model = model
self.scheme_name = scheme_name
async def __call__(self, request: Request) -> str | None:
return request.headers.get("Authorization")
basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth")
bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth")
def _split_authorization(authorization: str | None) -> tuple[str, str] | None:
if authorization is None:
return None
scheme, _, value = authorization.partition(" ")
if not value.strip():
return None
return scheme.lower(), value.strip()
async def get_bearer_token(
authorization: Annotated[str | None, Security(bearer_auth)],
) -> str:
parts = _split_authorization(authorization)
if parts is None or parts[0] != "bearer":
raise unauthorized_bearer()
return parts[1]
async def get_basic_credentials(
authorization: Annotated[str | None, Security(basic_auth)],
) -> tuple[str, str]:
"""(username, password) из `Authorization: Basic`; пароль может содержать ':'."""
parts = _split_authorization(authorization)
if parts is None or parts[0] != "basic":
raise unauthorized_basic()
try:
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError, ValueError):
raise unauthorized_basic() from None
username, separator, password = decoded.partition(":")
if not separator or not username:
raise unauthorized_basic()
return username, password
DbSession = Annotated[AsyncSession, Depends(get_db)]
RedisClient = Annotated[Redis, Depends(get_redis)]
AppSettings = Annotated[Settings, Depends(get_settings)]
S3Storage = Annotated[MinioStorage | None, Depends(get_s3)]
UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)]
BearerToken = Annotated[str, Depends(get_bearer_token)]
BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)]
async def get_current_user(
token: BearerToken, session: DbSession, redis: RedisClient
) -> User:
return await authenticate_token(session, redis, token)
CurrentUser = Annotated[User, Depends(get_current_user)]
+20
View File
@@ -0,0 +1,20 @@
from fastapi import APIRouter, Request
from app.domain import health
from app.schemas.base import Health
router = APIRouter(prefix='/health', tags=['health'])
@router.get('/live')
async def liveness() -> Health:
return health.liveness()
@router.get('/ready')
async def readiness(request: Request) -> Health:
state = request.app.state
return await health.readiness(
state.engine, state.redis, state.s3, state.lifecycle,
state.settings.readiness_timeout_seconds,
)
+38
View File
@@ -0,0 +1,38 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Header, Query
from app.domain.idempotency import send_message_idempotent, validate_idempotency_key
from app.domain.messages import list_messages
from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession
from app.schemas.message import CreateMessage, Message, MessagePage
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
@router.post("/{chat_id}/messages", status_code=201)
async def send_new_message(
chat_id: UUID,
new_message: CreateMessage,
sender: CurrentUser,
session: DbSession,
settings: AppSettings,
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
) -> Message:
return await send_message_idempotent(
session, sender, chat_id, new_message,
validate_idempotency_key(idempotency_key),
settings.max_messages, settings.idempotency_ttl_seconds,
)
@router.get("/{chat_id}/messages")
async def get_history_messages(
chat_id: UUID,
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> MessagePage:
return await list_messages(session, user, chat_id, cursor, limit)
+17
View File
@@ -0,0 +1,17 @@
from fastapi import APIRouter
from app.domain.users import register_user
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.user import CreateUser, User
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", status_code=201)
async def create_user(new_user: CreateUser, session: DbSession) -> User:
return await register_user(session, new_user)
@router.get("/me")
async def get_me(user: CurrentUser) -> User:
return user
View File
+109
View File
@@ -0,0 +1,109 @@
from collections.abc import Mapping
from PIL import Image, UnidentifiedImageError
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(
self,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers
def invalid_request(message: str = "Неверный запрос") -> ApiError:
"""Создаёт ошибку некорректного или недопустимого запроса."""
return ApiError(400, "invalid_request", message)
def unauthorized(
message: str = "Нет действительной аутентификации",
www_authenticate: str | None = None,
) -> ApiError:
"""Создаёт ошибку отсутствующей или недействительной аутентификации."""
headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None
return ApiError(401, "unauthorized", message, headers)
def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError:
"""401 для POST /auth/sessions (одинаковый для неверного username и пароля)."""
return unauthorized(message, BASIC_CHALLENGE)
def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError:
"""401 для бизнес-endpoint: нет, истёк, отозван или неизвестен токен."""
return unauthorized(message, BEARER_CHALLENGE)
def not_found(message: str = "Объект не найден") -> ApiError:
"""Создаёт ошибку, указывающую на отсутствие запрошенного ресурса."""
return ApiError(404, "not_found", message)
def conflict(message: str = "Конфликт") -> ApiError:
"""Создаёт ошибку конфликта с текущим состоянием ресурса."""
return ApiError(409, "conflict", message)
def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError:
"""Создаёт ошибку, указывающую на превышение допустимого размера запроса."""
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(
message: str = "Требуется Content-Type: application/json",
) -> ApiError:
"""Создаёт ошибку неподдерживаемого формата или типа содержимого запроса."""
return ApiError(415, "unsupported_media_type", message)
def rate_limited(retry_after: int = 1) -> ApiError:
"""Создаёт ошибку превышения допустимой частоты запросов."""
return ApiError(
429,
"rate_limited",
"Слишком много запросов",
{"Retry-After": str(max(1, retry_after))},
)
def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
"""Создаёт ошибку временной недоступности сервиса."""
return ApiError(503, "unavailable", message)
def internal_error() -> ApiError:
"""Создаёт ошибку внутренней ошибки сервера."""
return ApiError(500, "internal_error", "Внутренняя ошибка сервера")
IMAGE_WORKER_ERRORS = (
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
)
def image_worker_error_code(error: Exception) -> int:
code = 2
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
declared_limit = isinstance(error, ApiError) and error.status_code == 413
if resource_error or declared_limit:
code = 3
return code
def image_validation_error(code: int) -> ApiError:
error = unsupported_media_type("Повреждённое изображение или несовпадение MIME")
if code == 3 or code < 0:
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
return error
+225
View File
@@ -0,0 +1,225 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from collections.abc import Mapping
from uuid import uuid4
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from minio.error import MinioException
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import TimeoutError as RedisTimeoutError
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.requests import ClientDisconnect
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.instance import INSTANCE_ID_HEADER
from app.core.request_id import REQUEST_ID_HEADER
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
logger = logging.getLogger(__name__)
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
TRANSIENT_SQLSTATES = frozenset({"55P03"})
RECOVERABLE_STORAGE_ERRORS = (
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
)
@asynccontextmanager
async def operation_timeout(seconds: float, error: ApiError):
"""Единое преобразование таймаутов; владелец операции задаёт ожидаемую ошибку."""
try:
async with asyncio.timeout(seconds):
yield
except TimeoutError:
raise error from None
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
def log_recoverable_error(operation: str, error: Exception) -> None:
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
logger.warning('%s deferred: %s', operation, type(error).__name__)
def _get_request_id(request: Request) -> str:
request_id = getattr(request.state, "request_id", None)
if request_id is None:
request_id = str(uuid4())
request.state.request_id = request_id
return request_id
def _build_response(
request: Request,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> JSONResponse:
request_id = _get_request_id(request)
response_headers = {
**(headers or {}),
REQUEST_ID_HEADER: request_id,
}
instance_id = getattr(request.app.state, "instance_id", None)
if instance_id is not None:
response_headers[INSTANCE_ID_HEADER] = instance_id
body = {
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
}
return JSONResponse(
status_code=status_code,
content=body,
headers=response_headers,
)
async def api_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, ApiError)
return _build_response(
request,
exc.status_code,
exc.code,
exc.message,
exc.headers,
)
async def routing_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, StarletteHTTPException)
error = invalid_request()
if exc.status_code == 404:
error = not_found()
return _build_response(
request,
exc.status_code,
error.code,
error.message,
exc.headers,
)
async def validation_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, RequestValidationError)
fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()})
message = "Неверный запрос"
if fields:
message = f"{message}: {', '.join(fields)}"
return _build_response(
request,
400,
"invalid_request",
message,
)
async def dependency_unavailable_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
"""PostgreSQL/Redis/S3 недоступны либо исчерпан пул: 503 без внутренних деталей."""
request_id = _get_request_id(request)
logger.error(
"Dependency unavailable request_id=%s: %s",
request_id,
type(exc).__name__,
)
error = unavailable()
return _build_response(
request,
error.status_code,
error.code,
error.message,
{"Retry-After": "1"},
)
def _is_transient_database_error(exc: DBAPIError) -> bool:
sqlstate = getattr(exc.orig, "sqlstate", None) or ""
return (
sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES)
or sqlstate in TRANSIENT_SQLSTATES
)
async def database_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
"""Временные ошибки PostgreSQL (таймаут запроса, блокировки, нехватка ресурсов) — 503."""
assert isinstance(exc, DBAPIError)
handler = unhandled_error_handler
if _is_transient_database_error(exc):
handler = dependency_unavailable_handler
return await handler(request, exc)
async def unhandled_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
request_id = _get_request_id(request)
logger.error(
"Unhandled error request_id=%s",
request_id,
exc_info=exc,
)
error = internal_error()
return _build_response(
request,
error.status_code,
error.code,
error.message,
)
def register_exception_handlers(app: FastAPI) -> None:
app.add_exception_handler(ApiError, api_error_handler)
app.add_exception_handler(ClientDisconnect, client_disconnect_handler)
app.add_exception_handler(MinioException, dependency_unavailable_handler)
app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler)
app.add_exception_handler(StarletteHTTPException, routing_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(OperationalError, dependency_unavailable_handler)
app.add_exception_handler(InterfaceError, dependency_unavailable_handler)
app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(DBAPIError, database_error_handler)
app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler)
app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(OSError, dependency_unavailable_handler)
app.add_exception_handler(Exception, unhandled_error_handler)
+92
View File
@@ -0,0 +1,92 @@
import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
import uvicorn
from fastapi import Depends, FastAPI
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.instance import InstanceIdMiddleware, resolve_instance_id
from app.core.lifecycle import GracefulServer, Lifecycle
from app.core.redis import create_redis_client
from app.core.request_capacity import acquire_request_slot, create_request_capacity
from app.core.request_id import RequestIdMiddleware
from app.endpoints.auth import router as auth_router
from app.endpoints.attachments import router as attachments_router
from app.endpoints.chats import router as chats_router
from app.endpoints.health import router as health_router
from app.endpoints.messages import router as messages_router
from app.endpoints.users import router as users_router
from app.errors.handlers import register_exception_handlers
from app.storage.s3 import create_s3_storage
def configure_logging(settings: Settings) -> None:
logging.basicConfig(
level=settings.log_level,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings() # type: ignore
configure_logging(settings)
engine = create_db_engine(settings)
redis = create_redis_client(settings)
s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None
@asynccontextmanager
async def lifespan(_: FastAPI) -> AsyncGenerator[None]:
yield
if s3 is not None:
await s3.aclose()
await redis.aclose()
await engine.dispose()
application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan)
application.state.settings = settings
application.state.engine = engine
application.state.session_factory = create_session_factory(engine)
application.state.redis = redis
application.state.s3 = s3
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
application.state.lifecycle = Lifecycle()
application.add_middleware(RequestIdMiddleware)
application.add_middleware(
InstanceIdMiddleware, instance_id=application.state.instance_id
)
register_exception_handlers(application)
limited = [Depends(acquire_request_slot)]
application.include_router(health_router)
application.include_router(users_router, dependencies=limited)
application.include_router(auth_router, dependencies=limited)
application.include_router(chats_router, dependencies=limited)
application.include_router(messages_router, dependencies=limited)
application.include_router(attachments_router, dependencies=limited)
return application
def main() -> None:
settings = Settings() # type: ignore
application = create_app(settings)
config = uvicorn.Config(
application,
host=settings.app_host,
port=settings.app_port,
log_level=settings.log_level.lower(),
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
)
GracefulServer(
config, application.state.lifecycle, settings.shutdown_drain_seconds
).run()
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
from app.models.base import Base
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.models.counter import Counter
from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
+57
View File
@@ -0,0 +1,57 @@
"""Метаданные оригиналов и журнал операций до записи в S3."""
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Attachment(Base):
__tablename__ = "attachments"
__table_args__ = (
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status = 'ready'", name="status"),
Index("ix_attachments_chat_id", "chat_id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
object_key: Mapped[str] = mapped_column(String(128), unique=True)
filename: Mapped[str] = mapped_column(String(255))
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int]
sha256: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16), server_default="ready")
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int]
class UploadIntent(Base):
"""Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных."""
__tablename__ = "upload_intents"
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
object_key: Mapped[str] = mapped_column(String(128), unique=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+27
View File
@@ -0,0 +1,27 @@
import uuid
from datetime import datetime
from typing import Any, ClassVar
from sqlalchemy import DateTime, MetaData
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.types import TypeEngine
NAMING_CONVENTION = {
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
class Base(DeclarativeBase):
"""Базовый класс ORM-моделей."""
metadata = MetaData(naming_convention=NAMING_CONVENTION)
type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = {
uuid.UUID: PG_UUID(as_uuid=True),
datetime: DateTime(timezone=True),
}
+32
View File
@@ -0,0 +1,32 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class Chat(Base):
"""Чат с фиксированным составом участников."""
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
title: Mapped[str] = mapped_column(String(100))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
members: Mapped[list["ChatMember"]] = relationship(
back_populates="chat",
cascade="all, delete-orphan",
passive_deletes=True,
lazy="selectin",
)
messages: Mapped[list["Message"]] = relationship(
back_populates="chat", lazy="raise"
)
+26
View File
@@ -0,0 +1,26 @@
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class ChatMember(Base):
"""Членство пользователя в чате."""
__tablename__ = "chat_members"
__table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),)
chat_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise")
user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise")
+13
View File
@@ -0,0 +1,13 @@
from sqlalchemy import BigInteger, String
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Counter(Base):
"""Именованный счётчик с атомарным обновлением."""
__tablename__ = "counters"
name: Mapped[str] = mapped_column(String(64), primary_key=True)
value: Mapped[int] = mapped_column(BigInteger)
+24
View File
@@ -0,0 +1,24 @@
"""Сохранённый результат отправки сообщения."""
import uuid
from datetime import datetime
from sqlalchemy import ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class IdempotencyKey(Base):
"""Ключ запроса и связанное сообщение."""
__tablename__ = "idempotency_keys"
__table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
method: Mapped[str] = mapped_column(String(8), primary_key=True)
path: Mapped[str] = mapped_column(String(255), primary_key=True)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
request_hash: Mapped[str] = mapped_column(String(64))
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id"))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+16
View File
@@ -0,0 +1,16 @@
"""Место для будущих ORM-моделей job, outbox и вариантов lab5.
Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4
до реализации миграции. Планируемые данные:
* job: job_id, attachment_id, pipeline_version, attempt/max_attempts,
status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps;
* outbox: event_id, job_id, payload/version, состояние публикации и её lease;
* variant: attachment_id, pipeline_version, variant, приватный object_key,
content_type, size_bytes, sha256, размеры;
* replay audit: операторское действие и связь с прежней логической job.
Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker
не публикует metadata поверх новой версии; точная схема зависит от трека 5.
Изменение ready-only constraints attachments — отдельная новая миграция.
"""
+30
View File
@@ -0,0 +1,30 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index, Text, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class Message(Base):
"""Сообщение в чате."""
__tablename__ = "messages"
__table_args__ = (
Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
text: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise")
sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise")
+31
View File
@@ -0,0 +1,31 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class User(Base):
"""Пользователь и его учётные данные."""
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
username: Mapped[str] = mapped_column(String(32), unique=True)
display_name: Mapped[str] = mapped_column(String(100))
password_hash: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
memberships: Mapped[list["ChatMember"]] = relationship(
back_populates="user", lazy="raise"
)
messages: Mapped[list["Message"]] = relationship(
back_populates="sender", lazy="raise"
)
+6
View File
@@ -0,0 +1,6 @@
from app.schemas.chat import Chat
from app.schemas.message import Message
from app.schemas.session import AuthSession
from app.schemas.user import User
__all__ = ["AuthSession", "Chat", "Message", "User"]
+23
View File
@@ -0,0 +1,23 @@
"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту."""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import Field
from app.schemas.base import Base
class Attachment(Base):
chat_id: UUID
owner_id: UUID
filename: str = Field(min_length=1, max_length=255)
content_type: Literal["image/png", "image/jpeg"]
size_bytes: int = Field(ge=1, le=10_485_760)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
status: Literal["ready"] = "ready"
variants: list[object] = Field(default_factory=list, max_length=0)
error_code: None = None
created_at: datetime
updated_at: datetime
+35
View File
@@ -0,0 +1,35 @@
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, field_validator
class Base(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
id: UUID
class Health(BaseModel):
model_config = ConfigDict(extra="forbid")
status: Literal["ok"]
class BaseCreate(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
"""Отклоняет строки из одних пробелов и строки с NUL, который не хранит PostgreSQL."""
if isinstance(value, str):
if not value.strip():
raise ValueError("Строка не может состоять только из пробелов")
if "\x00" in value:
raise ValueError("Строка не может содержать символ NUL")
return value
class BasePage(BaseModel):
model_config = ConfigDict(extra="forbid")
+22
View File
@@ -0,0 +1,22 @@
from datetime import datetime
from uuid import UUID
from pydantic import Field
from app.schemas.base import Base, BaseCreate, BasePage
class Chat(Base):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
created_at: datetime
class CreateChat(BaseCreate):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
class ChatPage(BasePage):
items: list[Chat]
next_cursor: str | None
+10
View File
@@ -0,0 +1,10 @@
"""Место для будущей схемы сообщения contracts/jobs.schema.json.
Пока классы не реализованы и сообщения не принимаются/не публикуются.
Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id,
pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339).
Допустимы traceparent и tracestate; дополнительные поля запрещены.
job_id сохраняется при redelivery; очередная управляемая попытка отражается
в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются.
Pydantic-схема должна соответствовать публичному JSON Schema, не менять его.
"""
+41
View File
@@ -0,0 +1,41 @@
from datetime import datetime
from typing import Self
from uuid import UUID
from pydantic import Field, field_validator, model_validator
from app.schemas.base import Base, BaseCreate, BasePage
class Message(Base):
chat_id: UUID
sender_id: UUID
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
created_at: datetime
class CreateMessage(BaseCreate):
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
if isinstance(value, str):
if "\x00" in value or (value != "" and not value.strip()):
raise ValueError("Некорректная строка сообщения")
return value
@model_validator(mode="after")
def require_content(self) -> Self:
if not self.text and not self.attachment_ids:
raise ValueError("Нужен текст или вложение")
if len(self.attachment_ids) != len(set(self.attachment_ids)):
raise ValueError("attachment_ids содержит дубликаты")
return self
class MessagePage(BasePage):
items: list[Message]
next_cursor: str | None
+17
View File
@@ -0,0 +1,17 @@
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from app.schemas.user import User
class AuthSession(BaseModel):
"""Ответ POST /auth/sessions (в OpenAPI — Session)."""
model_config = ConfigDict(extra="forbid")
token: str = Field(min_length=32)
token_type: Literal["Bearer"] = "Bearer"
expires_at: datetime
user: User
+25
View File
@@ -0,0 +1,25 @@
from datetime import datetime
from pydantic import Field
from app.schemas.base import Base, BaseCreate
USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$"
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
class User(Base):
"""Публичное представление пользователя. Пароль/хеш сюда не попадают никогда."""
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
created_at: datetime
class CreateUser(BaseCreate):
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(
min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False
)
+1
View File
@@ -0,0 +1 @@
"""Доступ к данным PostgreSQL и Redis."""
+99
View File
@@ -0,0 +1,99 @@
"""SQL-запросы метаданных и журнала; commit выполняется в domain."""
from datetime import timedelta
from uuid import UUID
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
async def create_attachment(session: AsyncSession, **values) -> Attachment:
record = Attachment(**values)
session.add(record)
await session.flush()
return record
async def get_attachment(
session: AsyncSession, attachment_id: UUID
) -> Attachment | None:
return await session.get(Attachment, attachment_id)
async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
async def get_for_chat(
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
) -> list[Attachment]:
records = []
if attachment_ids:
records = list(
await session.scalars(
select(Attachment).where(
Attachment.id.in_(attachment_ids),
Attachment.chat_id == chat_id,
)
)
)
return records
async def bind_attachments(
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
) -> None:
session.add_all(
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
for position, id in enumerate(attachment_ids)
)
await session.flush()
async def message_attachment_ids(
session: AsyncSession, message_ids: list[UUID]
) -> dict[UUID, list[UUID]]:
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
if not message_ids:
return result
rows = await session.execute(
select(MessageAttachment)
.where(MessageAttachment.message_id.in_(message_ids))
.order_by(MessageAttachment.position)
)
for link in rows.scalars():
result[link.message_id].append(link.attachment_id)
return result
async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
session.add(UploadIntent(id=id, object_key=key))
await session.flush()
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
return await session.scalar(
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
)
async def finish_upload(session: AsyncSession, id: UUID) -> None:
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
async def stale_uploads(
session: AsyncSession, grace_seconds: int, limit: int
) -> list[UploadIntent]:
result = await session.scalars(
select(UploadIntent)
.where(
UploadIntent.created_at
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
)
.order_by(UploadIntent.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
return list(result)
+27
View File
@@ -0,0 +1,27 @@
"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран.
Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные
поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity,
формат delivery handle и интерфейсы уточнить после выбора брокера/версии.
Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/.
"""
from collections.abc import Mapping
async def publish_job(payload: Mapping[str, object]) -> str:
"""Опубликовать устойчивое сообщение и вернуть идентификатор доставки.
Дождаться подтверждения сохранения брокером; неопределённый исход разрешает
повтор с тем же job_id. Не считать публикацию обработкой изображения.
"""
raise NotImplementedError("Lab5: публикация в брокер ещё не реализована")
async def acknowledge_job(delivery_id: str) -> None:
"""Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля.
Retry сначала устойчиво планируется, затем закрывается старая доставка.
Конкретный delivery handle зависит от брокера; сигнатура предварительная.
"""
raise NotImplementedError("Lab5: ack ещё не реализован")
+71
View File
@@ -0,0 +1,71 @@
from collections.abc import Sequence
from uuid import UUID
from sqlalchemy import exists, select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.storage.types import Position
async def create_chat(
session: AsyncSession,
*,
title: str,
member_ids: Sequence[UUID],
) -> Chat:
"""Добавляет чат вместе с участниками."""
chat = Chat(
title=title,
members=[ChatMember(user_id=user_id) for user_id in member_ids],
)
session.add(chat)
await session.flush()
return chat
async def get_chat_for_member(
session: AsyncSession,
chat_id: UUID,
user_id: UUID,
) -> Chat | None:
"""Возвращает чат, если пользователь в нём состоит, иначе None."""
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
chat = result.scalar_one_or_none()
return chat
async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool:
"""Возвращает True, если пользователь состоит в чате; одна проверка по PK."""
stmt = select(
exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
is_member = bool(result.scalar_one())
return is_member
async def list_chats_for_user(
session: AsyncSession,
user_id: UUID,
after: Position | None,
limit: int,
) -> list[Chat]:
"""Возвращает чаты пользователя по (created_at, id) строго после позиции after."""
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(ChatMember.user_id == user_id)
)
if after is not None:
stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit)
result = await session.execute(stmt)
chats = list(result.scalars().all())
return chats
+15
View File
@@ -0,0 +1,15 @@
"""Простые проверки соединения с общими хранилищами."""
from redis.asyncio import Redis
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
return True
async def check_redis(client: Redis) -> bool:
return bool(await client.ping())
+63
View File
@@ -0,0 +1,63 @@
"""Хранение и блокировка ключей в PostgreSQL."""
from datetime import timedelta
import hashlib
import json
from uuid import UUID
from sqlalchemy import BigInteger, func, literal, select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.idempotency_key import IdempotencyKey
Scope = tuple[UUID, str, str, str]
async def lock_key(session: AsyncSession, scope: Scope) -> None:
"""Сериализует один ключ до конца транзакции."""
raw = json.dumps([str(scope[0]), *scope[1:]]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
await session.execute(
select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger)))
)
async def get_active_key(
session: AsyncSession, scope: Scope, ttl_seconds: int
) -> IdempotencyKey | None:
"""Читает результат, пока срок ключа не истёк."""
stmt = select(IdempotencyKey).where(
tuple_(
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
) == tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
return await session.scalar(stmt)
async def save_result(
session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID
) -> None:
"""Сохраняет новый результат или заменяет истёкший."""
stmt = insert(IdempotencyKey).values(
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
request_hash=request_hash, message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id, IdempotencyKey.method,
IdempotencyKey.path, IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
"message_id": message_id,
"created_at": func.clock_timestamp(),
},
)
)
+37
View File
@@ -0,0 +1,37 @@
"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет.
Будущие ORM-операции захвата, lease, retry и результата должны защищаться
от конкурирующих попыток. Границы транзакций находятся в domain/.
Тип результата захвата уточняется после определения модели job.
"""
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
async def claim_job(session: AsyncSession, job_id: UUID) -> None:
"""Условно захватить допустимое задание и сохранить lease/номер попытки.
Ready и актуальная чужая аренда не переходят обратно в processing.
Будущий результат должен различать захват, завершённый дубль и занятость.
"""
raise NotImplementedError("Lab5: захват job ещё не реализован")
async def complete_job(session: AsyncSession, job_id: UUID) -> None:
"""Атомарно записать metadata вариантов и ready после сохранения объектов.
Предусловия lease/attempt/pipeline_version и структура результата будут
добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage.
"""
raise NotImplementedError("Lab5: фиксация результата ещё не реализована")
async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None:
"""Записать безопасный код сбоя без stack trace и secrets.
Domain выбирает queued с next_attempt_at либо конечный failed согласно
классификации ошибки и лимиту попыток; storage лишь сохраняет решение.
"""
raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано")
+62
View File
@@ -0,0 +1,62 @@
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.counter import Counter
from app.models.message import Message
from app.storage.types import Position
MESSAGES_COUNTER = "messages"
async def get_message(session: AsyncSession, message_id: UUID) -> Message | None:
return await session.get(Message, message_id)
async def create_message(
session: AsyncSession,
*,
chat_id: UUID,
sender_id: UUID,
text: str,
) -> Message:
"""Добавляет сообщение; id и created_at назначает сервер."""
message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
session.add(message)
await session.flush()
return message
async def list_messages(
session: AsyncSession,
chat_id: UUID,
after: Position | None,
limit: int,
) -> list[Message]:
"""Возвращает сообщения чата по (created_at, id) строго после позиции after."""
stmt = select(Message).where(Message.chat_id == chat_id)
if after is not None:
stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
stmt = stmt.order_by(Message.created_at, Message.id).limit(limit)
result = await session.execute(stmt)
messages = list(result.scalars().all())
return messages
async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool:
"""Атомарно увеличивает счётчик сообщений, если лимит не исчерпан."""
stmt = (
insert(Counter)
.values(name=MESSAGES_COUNTER, value=1)
.on_conflict_do_update(
index_elements=[Counter.name],
set_={"value": Counter.value + 1},
where=Counter.value < max_messages,
)
.returning(Counter.value)
)
result = await session.execute(stmt)
is_reserved = result.scalar_one_or_none() is not None
return is_reserved
+23
View File
@@ -0,0 +1,23 @@
"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker.
Outbox — таблица намерений публикации в той же транзакции, что и job.
Повторная публикация после неопределённого confirm допустима; job_id стабилен.
Форма записей, захват пачки и lease publisher уточняются вместе с моделью.
"""
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
async def add_job_event(session: AsyncSession, job_id: UUID) -> None:
"""Сохранить намерение публикации job без commit и обращения к брокеру."""
raise NotImplementedError("Lab5: запись outbox ещё не реализована")
async def mark_event_published(session: AsyncSession, event_id: UUID) -> None:
"""Отметить событие опубликованным после устойчивого подтверждения брокера.
Не помечать job завершённой: publisher confirm отличается от consumer ack.
"""
raise NotImplementedError("Lab5: отметка outbox ещё не реализована")
+180
View File
@@ -0,0 +1,180 @@
"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
import asyncio
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import TypeVar
import time
from urllib.parse import urlsplit
from anyio import CancelScope
from minio import Minio
import urllib3
from app.core.s3 import S3Settings
T = TypeVar("T")
async def run_sync(
operation: Callable[..., T], *args, discard_result=None, **kwargs
) -> T:
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
try:
result = await asyncio.shield(task)
except asyncio.CancelledError:
with CancelScope(shield=True):
await settle_cancelled_call(task, discard_result)
raise
return result
async def settle_cancelled_call(
task: asyncio.Task, discard_result: Callable | None
) -> None:
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
while not task.done():
with suppress(asyncio.CancelledError):
await asyncio.wait([task])
completed = not task.cancelled() and task.exception() is None
if completed and discard_result is not None:
discard_result(task.result())
@dataclass(frozen=True)
class ObjectInfo:
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
key: str
size_bytes: int | None
content_type: str | None
last_modified: datetime | None
def close_response(response) -> None:
response.close()
response.release_conn()
class ObjectStream:
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
def __init__(self, storage, response):
self.storage = storage
self.response = response
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
self.closed = False
def __aiter__(self):
return self
async def __anext__(self):
if self.closed:
raise StopAsyncIteration
try:
remaining = self.deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError
async with asyncio.timeout(remaining):
chunk = await self.storage._call(
lambda: self.response.read(self.storage.settings.chunk_bytes)
)
if not chunk:
await self.aclose()
raise StopAsyncIteration
except BaseException:
await self.aclose()
raise
return chunk
async def aclose(self):
if not self.closed:
self.closed = True
close_response(self.response)
class MinioStorage:
def __init__(self, settings: S3Settings) -> None:
self.settings = settings
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
url = urlsplit(settings.endpoint)
self._http = urllib3.PoolManager(
timeout=urllib3.Timeout(
connect=settings.connect_timeout_seconds,
read=settings.read_timeout_seconds,
),
retries=urllib3.Retry(total=0),
maxsize=32,
)
self._health_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=0.25, read=0.5),
retries=urllib3.Retry(total=0),
)
self._client = Minio(
url.netloc,
secure=url.scheme == "https",
region=settings.region,
access_key=settings.access_key_id_file.read_text().strip(),
secret_key=settings.secret_access_key_file.read_text().strip(),
http_client=self._http,
)
async def _call(self, operation: Callable, *, discard_result=None):
return await run_sync(operation, discard_result=discard_result)
async def put_file(
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
) -> None:
def put():
with path.open("rb") as source:
# Один PUT: размер части равен максимальному разрешённому файлу.
self._client.put_object(
self.settings.bucket,
key,
source,
size_bytes,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10_485_760,
num_parallel_uploads=1,
)
await self._call(put)
async def open_object(self, key: str) -> ObjectStream:
response = await self._call(
lambda: self._client.get_object(self.settings.bucket, key),
discard_result=close_response,
)
return ObjectStream(self, response)
async def stat_object(self, key: str) -> ObjectInfo:
def stat():
info = self._client.stat_object(self.settings.bucket, key)
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
return await self._call(stat)
async def delete_object(self, key: str) -> None:
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
async def check_available(self) -> bool:
status = await self._call(
lambda: self._health_http.request(
"GET",
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status
)
return status == 200
async def aclose(self) -> None:
self._http.clear()
self._health_http.clear()
def create_s3_storage(settings: S3Settings) -> MinioStorage:
return MinioStorage(settings)
+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime
from uuid import UUID
from redis.asyncio import Redis
SESSION_KEY_PREFIX = "session:"
def session_key(token_hash: str) -> str:
"""Возвращает ключ Redis для хеша токена."""
return f"{SESSION_KEY_PREFIX}{token_hash}"
async def create_session(
redis: Redis,
*,
token_hash: str,
user_id: UUID,
expires_at: datetime,
) -> None:
"""Добавляет сессию с абсолютным временем истечения."""
await redis.set(session_key(token_hash), str(user_id), pxat=expires_at)
async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None:
"""Возвращает id владельца действующей сессии или None."""
raw_user_id = await redis.get(session_key(token_hash))
if isinstance(raw_user_id, bytes):
raw_user_id = raw_user_id.decode()
user_id = UUID(raw_user_id) if raw_user_id else None
return user_id
async def delete_session(redis: Redis, token_hash: str) -> bool:
"""Удаляет сессию; возвращает True, если она существовала."""
deleted_count = await redis.delete(session_key(token_hash))
is_deleted = deleted_count > 0
return is_deleted
+6
View File
@@ -0,0 +1,6 @@
"""Общие типы слоя хранения."""
from datetime import datetime
from uuid import UUID
Position = tuple[datetime, UUID]
+47
View File
@@ -0,0 +1,47 @@
from collections.abc import Collection
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.user import User
async def create_user(
session: AsyncSession,
*,
username: str,
display_name: str,
password_hash: str,
) -> User:
"""Добавляет пользователя; занятый username даёт IntegrityError при flush."""
user = User(
username=username, display_name=display_name, password_hash=password_hash
)
session.add(user)
await session.flush()
return user
async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
"""Возвращает пользователя по id или None."""
user = await session.get(User, user_id)
return user
async def get_user_by_username(session: AsyncSession, username: str) -> User | None:
"""Возвращает пользователя по username или None."""
stmt = select(User).where(User.username == username)
result = await session.execute(stmt)
user = result.scalar_one_or_none()
return user
async def get_existing_user_ids(
session: AsyncSession, user_ids: Collection[UUID]
) -> set[UUID]:
"""Возвращает те из user_ids, что существуют, одним запросом."""
stmt = select(User.id).where(User.id.in_(user_ids))
result = await session.execute(stmt)
existing_ids = set(result.scalars().all())
return existing_ids
+10
View File
@@ -0,0 +1,10 @@
# Override для отдельного тестового стенда (make test-stand-up):
# публикует PostgreSQL и Redis на 127.0.0.1, чтобы pytest из student-tests/
# мог подключаться к ним напрямую. В основном стенде эти порты закрыты.
services:
postgres:
ports:
- "127.0.0.1:${DB_HOST_PORT:?Set DB_HOST_PORT in .env}:5432"
redis:
ports:
- "127.0.0.1:${REDIS_HOST_PORT:?Set REDIS_HOST_PORT in .env}:6379"
+295
View File
@@ -0,0 +1,295 @@
name: mtusi-messenger-lab5
x-api-image: &api-image
build:
context: .
args:
API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env}
image: ${API_IMAGE:?Set API_IMAGE in .env}
x-base-env: &base-env
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
DB_HOST: ${DB_HOST:?Set DB_HOST in .env}
DB_PORT: ${DB_PORT:?Set DB_PORT in .env}
DB_NAME: ${DB_NAME:?Set DB_NAME in .env}
DB_USER: ${DB_USER:?Set DB_USER in .env}
DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env}
DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env}
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env}
SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env}
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env}
READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env}
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_REGION: ${S3_REGION:?Set S3_REGION in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env}
S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env}
S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env}
S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env}
S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env}
S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env}
S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env}
S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env}
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
services:
minio:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: server
image: mtusi-messenger-minio:lab5
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
restart: unless-stopped
environment:
MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
MINIO_BROWSER: ${MINIO_BROWSER:?Set MINIO_BROWSER in .env}
secrets:
- source: minio_root_user
target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
- source: minio_root_password
target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
volumes:
- miniodata:/data
networks: [objects]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
minio-bootstrap:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: bootstrap
image: mtusi-messenger-minio-bootstrap:lab5
environment:
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
secrets:
- source: minio_root_user
target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
- source: minio_root_password
target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [objects]
restart: "no"
depends_on:
minio:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
postgres:
image: postgres:16.10-bookworm
restart: unless-stopped
environment:
POSTGRES_USER: ${DB_USER:?Set DB_USER in .env}
POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env}
POSTGRES_PASSWORD_FILE: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
secrets:
- source: db_password
target: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
redis:
image: redis:7.4.2-bookworm
restart: unless-stopped
entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"]
secrets:
- redis_password
volumes:
- ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
- ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro
- redisdata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
migrate:
<<: *api-image
command: ["alembic", "upgrade", "head"]
environment: *base-env
secrets:
- db_password
depends_on:
postgres:
condition: service_healthy
restart: "no"
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
api:
<<: *api-image
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env}
REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env}
REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env}
REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env}
REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env}
REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env}
REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env}
SESSION_STORE: ${SESSION_STORE:?Set SESSION_STORE in .env}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env}
secrets:
- db_password
- redis_password
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
deploy:
replicas: ${API_REPLICAS:?Set API_REPLICAS in .env}
resources:
limits:
cpus: "1.0"
memory: 1536M
healthcheck:
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
interval: 3s
timeout: 3s
start_period: 10s
retries: 2
stop_signal: SIGTERM
stop_grace_period: 8s
depends_on:
minio:
condition: service_healthy
minio-bootstrap:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
cleanup:
<<: *api-image
command: ["python", "-m", "app.core.cleanup", "--apply"]
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
secrets:
- db_password
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
minio-bootstrap:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
nginx:
image: nginx:1.28.0-alpine
restart: unless-stopped
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"]
interval: 5s
timeout: 3s
retries: 5
depends_on:
api:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
volumes:
pgdata:
redisdata:
miniodata:
networks:
default:
objects:
internal: true
secrets:
db_password:
file: ./secrets/db_password
redis_password:
file: ./secrets/redis_password
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+10
View File
@@ -5,3 +5,13 @@
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
## Перенесённая база lab4
В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap,
migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления.
Роли broker, outbox publisher и worker пока не реализованы.
core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди.
Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md.
Настройки environment в Compose задаются через .env; образец — .env.example.
Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам.
+22
View File
@@ -0,0 +1,22 @@
FROM golang:1.24.8-bookworm AS compiler
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM python:3.12.15-slim-bookworm AS runtime
RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
USER minio
FROM runtime AS server
COPY --from=compiler /out/minio /usr/local/bin/minio
LABEL org.opencontainers.image.source="https://github.com/minio/minio" \
org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z"
VOLUME ["/data"]
ENTRYPOINT ["/usr/local/bin/minio"]
FROM runtime AS bootstrap
COPY --from=compiler /out/mc /usr/local/bin/mc
COPY infra/minio/bootstrap.py /bootstrap.py
ENTRYPOINT ["python", "/bootstrap.py"]
+117
View File
@@ -0,0 +1,117 @@
"""Идемпотентный private bucket и отдельная ограниченная учётная запись API."""
import json
import os
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory
import time
from urllib.request import urlopen
def secret(name):
value = Path(os.environ[name.upper() + "_FILE"]).read_text().strip()
if not value:
raise ValueError("Пустой файл секрета")
return value
def main():
endpoint = os.environ["S3_ENDPOINT"]
bucket = os.environ["S3_BUCKET"]
root_user, root_password = secret("minio_root_user"), secret("minio_root_password")
access, password = secret("s3_access_key_id"), secret("s3_secret_access_key")
if access == root_user:
raise ValueError("API не должен использовать root credentials")
deadline = time.monotonic() + 60
while True:
try:
with urlopen(endpoint + "/minio/health/ready", timeout=2) as response:
if response.status == 200:
break
except OSError:
pass
if time.monotonic() >= deadline:
raise RuntimeError("MinIO не готов за 60 секунд")
time.sleep(1)
with TemporaryDirectory(prefix="minio-bootstrap-") as directory:
config = Path(directory)
# Credentials не передаются mc alias set через аргументы или окружение.
config_file = config / "config.json"
config_file.write_text(
json.dumps(
{
"version": "10",
"aliases": {
"storage": {
"url": endpoint,
"accessKey": root_user,
"secretKey": root_password,
"api": "S3v4",
"path": "auto",
}
},
}
)
)
config_file.chmod(0o600)
def mc(*args):
result = subprocess.run(
["mc", "--config-dir", directory, *args],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=30,
)
if result.returncode:
raise RuntimeError(
"Ошибка bootstrap MinIO; значения credentials скрыты"
)
mc("mb", "--ignore-existing", "storage/" + bucket)
mc("anonymous", "set", "none", "storage/" + bucket)
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": [f"arn:aws:s3:::{bucket}"],
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": [f"arn:aws:s3:::{bucket}/originals/*"],
},
],
}
policy_file = config / "policy.json"
policy_file.write_text(json.dumps(policy))
mc(
"admin",
"policy",
"create",
"storage",
"messenger-attachments",
str(policy_file),
)
# mc не поддерживает stdin для user add: вывод и ошибки команды скрыты.
mc("admin", "user", "add", "storage", access, password)
mc(
"admin",
"policy",
"attach",
"storage",
"messenger-attachments",
"--user",
access,
)
print("MinIO private bucket and API policy configured", flush=True)
if __name__ == "__main__":
try:
main()
except Exception as error:
print("MinIO bootstrap failed: " + type(error).__name__, flush=True)
raise SystemExit(1) from None
+140
View File
@@ -0,0 +1,140 @@
# Единая точка входа: round-robin между API-репликами, без sticky sessions.
# Директивы рассчитаны на закреплённый nginx:1.28.0-alpine.
worker_processes 2;
events {
worker_connections 1024;
}
http {
# В логе используем request_id ответа API, для собственных ошибок — id nginx.
map $upstream_http_x_request_id $response_request_id {
"" $request_id;
default $upstream_http_x_request_id;
}
log_format requests escape=json
'{"time":"$time_iso8601","request_id":"$response_request_id",'
'"method":"$request_method","uri":"$uri","status":$status,'
'"bytes":$body_bytes_sent,"request_time":$request_time,'
'"upstream":"$upstream_addr","upstream_status":"$upstream_status",'
'"upstream_time":"$upstream_response_time",'
'"instance_id":"$upstream_http_x_instance_id",'
'"limit_status":"$limit_req_status"}';
access_log /dev/stdout requests;
error_log /dev/stderr warn;
server_tokens off;
# DNS Docker возвращает адреса всех реплик сервиса api.
resolver 127.0.0.11 valid=5s ipv6=off;
resolver_timeout 1s;
# Общий для worker-процессов лимит на IP, который видит nginx.
# Клиентский X-Forwarded-For не используется в качестве ключа лимита.
limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s;
gzip on;
gzip_vary on;
gzip_min_length 1024;
gzip_comp_level 3;
gzip_types application/json;
upstream api_backend {
zone api_backend 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
# Кэш свободных соединений на worker, а не лимит запросов к API.
keepalive 16;
keepalive_timeout 4s;
}
server {
listen 8080;
client_header_timeout 10s;
client_body_timeout 10s;
# Размер ограничивает API при чтении, включая chunked-передачу.
client_max_body_size 0;
client_body_buffer_size 64k;
send_timeout 10s;
keepalive_timeout 30s;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 500ms;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
# Пассивное обнаружение отказов; Docker healthcheck сам не управляет LB.
# non_idempotent не включён: уже отправленные POST не повторяем.
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
# Окно перехода на другой backend, не общий deadline ответа.
proxy_next_upstream_timeout 2s;
# JSON и заголовки ошибок API передаются без подмены.
proxy_intercept_errors off;
error_page 429 = @rate_limited;
error_page 502 503 504 = @unavailable;
# Проверка самого LB для healthcheck в compose; не проксируется в API.
location = /lb-health {
access_log off;
default_type text/plain;
return 200 "ok\n";
}
# Проверки готовности/жизни не расходуют бюджет бизнес-запросов.
location /health/ {
proxy_pass http://api_backend;
}
location / {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
limit_req_log_level warn;
proxy_pass http://api_backend;
}
location ~ ^/api/v1/chats/[^/]+/attachments$ {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
proxy_request_buffering off;
proxy_buffering off;
proxy_read_timeout 65s;
proxy_send_timeout 15s;
# Ошибка одного upload не должна скрыть работающую реплику для чатов.
proxy_next_upstream off;
proxy_pass http://api_backend;
}
location ~ ^/api/v1/attachments/[^/]+/content$ {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
proxy_buffering off;
proxy_read_timeout 65s;
proxy_pass http://api_backend;
}
# Это ответы балансировщика: X-Instance-Id реплики у них отсутствует.
location @rate_limited {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header Retry-After 1 always;
return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}';
}
location @unavailable {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header Retry-After 1 always;
return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}';
}
}
}
+5
View File
@@ -0,0 +1,5 @@
dir /data
appendonly yes
appendfsync always
maxmemory 256mb
maxmemory-policy noeviction
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
password_hash=$(tr -d '\r\n' </run/secrets/redis_password | sha256sum | cut -d ' ' -f 1)
mkdir -p /run/redis
umask 077
printf 'user default reset on #%s ~* &* +@all\n' "$password_hash" >/run/redis/users.acl
chown redis:redis /run/redis/users.acl
chmod 600 /run/redis/users.acl
exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl
+68
View File
@@ -0,0 +1,68 @@
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from app.core.config import Settings
from app.models import Base
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name, disable_existing_loggers=False)
target_metadata = Base.metadata
def _database_url() -> str:
explicit = config.get_main_option("sqlalchemy.url")
if explicit:
return explicit
return Settings().get_database_url() # type: ignore
def run_migrations_offline() -> None:
context.configure(
url=_database_url(),
target_metadata=target_metadata,
literal_binds=True,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
section = config.get_section(config.config_ini_section, {})
section["sqlalchemy.url"] = _database_url().replace("%", "%%")
connectable = async_engine_from_config(
section, prefix="sqlalchemy.", poolclass=pool.NullPool
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+25
View File
@@ -0,0 +1,25 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
View File
@@ -0,0 +1,43 @@
"""Добавляет результаты идемпотентных запросов."""
from alembic import op
import sqlalchemy as sa
revision = "49d2c947a3f1"
down_revision = "abd273076068"
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Создаёт таблицу ключей без изменения существующих данных."""
op.create_table(
"idempotency_keys",
sa.Column("user_id", sa.UUID(), nullable=False),
sa.Column("method", sa.String(8), nullable=False),
sa.Column("path", sa.String(255), nullable=False),
sa.Column("key", sa.String(128), nullable=False),
sa.Column("request_hash", sa.String(64), nullable=False),
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"), nullable=False,
),
sa.ForeignKeyConstraint(
["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
),
sa.ForeignKeyConstraint(
["message_id"], ["messages.id"],
name=op.f("fk_idempotency_keys_message_id_messages"),
),
sa.PrimaryKeyConstraint(
"user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
),
)
op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"])
def downgrade() -> None:
"""Удаляет таблицу результатов запросов."""
op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys")
op.drop_table("idempotency_keys")
@@ -0,0 +1,61 @@
"""Attachment, связи сообщений и устойчивый журнал загрузок."""
from alembic import op
import sqlalchemy as sa
revision = "8f31b94a2d10"
down_revision = "49d2c947a3f1"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"attachments",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("owner_id", sa.UUID(), nullable=False),
sa.Column("object_key", sa.String(128), nullable=False),
sa.Column("filename", sa.String(255), nullable=False),
sa.Column("content_type", sa.String(32), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), server_default="ready", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")),
sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")),
sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")),
sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")),
sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
)
op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"])
op.create_table(
"message_attachments",
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column("attachment_id", sa.UUID(), nullable=False),
sa.Column("position", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")),
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")),
sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")),
)
op.create_table(
"upload_intents",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("object_key", sa.String(128), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
)
op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"])
def downgrade() -> None:
op.drop_index("ix_upload_intents_created_at", table_name="upload_intents")
op.drop_table("upload_intents")
op.drop_table("message_attachments")
op.drop_index("ix_attachments_chat_id", table_name="attachments")
op.drop_table("attachments")
@@ -0,0 +1,33 @@
"""add counters
Revision ID: abd273076068
Revises: db06fd3126ff
Create Date: 2026-10-26 22:01:01.910396
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = 'abd273076068'
down_revision: Union[str, None] = 'db06fd3126ff'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('counters',
sa.Column('name', sa.String(length=64), nullable=False),
sa.Column('value', sa.BigInteger(), nullable=False),
sa.PrimaryKeyConstraint('name', name=op.f('pk_counters'))
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('counters')
# ### end Alembic commands ###
@@ -0,0 +1,107 @@
"""create initial schema
Revision ID: db06fd3126ff
Revises:
Create Date: 2026-10-26 17:09:16.675339
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "db06fd3126ff"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"chats",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("title", sa.String(length=100), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")),
)
op.create_table(
"users",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("username", sa.String(length=32), nullable=False),
sa.Column("display_name", sa.String(length=100), nullable=False),
sa.Column("password_hash", sa.String(length=255), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
sa.UniqueConstraint("username", name=op.f("uq_users_username")),
)
op.create_table(
"chat_members",
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("user_id", sa.UUID(), nullable=False),
sa.ForeignKeyConstraint(
["chat_id"],
["chats.id"],
name=op.f("fk_chat_members_chat_id_chats"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users")
),
sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")),
)
op.create_index(
"ix_chat_members_user_id_chat_id",
"chat_members",
["user_id", "chat_id"],
unique=False,
)
op.create_table(
"messages",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("sender_id", sa.UUID(), nullable=False),
sa.Column("text", sa.Text(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.ForeignKeyConstraint(
["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats")
),
sa.ForeignKeyConstraint(
["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users")
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")),
)
op.create_index(
"ix_messages_chat_id_created_at_id",
"messages",
["chat_id", "created_at", "id"],
unique=False,
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages")
op.drop_table("messages")
op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members")
op.drop_table("chat_members")
op.drop_table("users")
op.drop_table("chats")
# ### end Alembic commands ###
+6
View File
@@ -0,0 +1,6 @@
[pytest]
testpaths = student-tests
python_files = test_*.py
asyncio_mode = auto
asyncio_default_fixture_loop_scope = session
asyncio_default_test_loop_scope = session
+52
View File
@@ -0,0 +1,52 @@
alembic==1.20.0
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
attrs==26.1.0
certifi==2026.7.22
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.143.3
h11==0.16.0
httpcore2==2.13.1
httpx2==2.13.1
idna==3.20
iniconfig==2.3.1
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.12.0
Mako==1.4.3
MarkupSafe==3.0.4
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
opentelemetry-api==1.45.1
packaging==26.3
pathable==0.4.4
pluggy==1.7.0
pydantic==2.14.1
pydantic_core==2.50.1
Pygments==2.21.0
pytest==9.1.1
PyYAML==6.0.3
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
six==1.17.0
SQLAlchemy==2.1.4
starlette==1.8.0
truststore==0.10.5
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.54.0
asyncpg==0.32.0
argon2-cffi==25.1.0
redis==8.1.0
greenlet==3.5.6
pydantic-settings==2.16.0
pytest-asyncio==1.4.1
httpx==0.28.1
minio==7.2.20
Pillow==13.0.0
+1
View File
@@ -0,0 +1 @@
"""Команды управления учебным стендом."""
+355
View File
@@ -0,0 +1,355 @@
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
import asyncio
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
from threading import Event
import time
from uuid import uuid4
import asyncpg
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from scripts.failover_all import exercise, failure_kind, history
from scripts.rolling_update import update
from tests.client import Client, Failure, arguments, check, run
PROBE = """import json, sys, urllib.request, urllib.error
payload = json.load(sys.stdin)
results = []
for item in payload:
data = json.dumps(item['body']).encode() if 'body' in item else None
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
try:
response = urllib.request.urlopen(request, timeout=7)
except urllib.error.HTTPError as error:
response = error
with response:
results.append({'status': response.status, 'body': json.loads(response.read())})
print(json.dumps(results))
"""
def probe(project, runner, requests):
"""Обращается к внутреннему API, передавая токен только через stdin."""
result = subprocess.run(
["docker", "exec", "-i", runner, "python", "-c", PROBE],
input=json.dumps(requests),
text=True,
capture_output=True,
timeout=10,
)
check(result.returncode == 0, "internal HTTP probe failed")
return json.loads(result.stdout)
def audit(project):
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
allowed = {
"postgres": {"db_password"},
"redis": {"redis_password"},
"migrate": {"db_password"},
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
"minio": {"minio_root_user", "minio_root_password"},
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
"nginx": set(),
}
ids = project.run("ps", "--all", "--quiet").splitlines()
containers = json.loads(project.docker("inspect", *ids))
for item in containers:
service = item["Config"]["Labels"]["com.docker.compose.service"]
check(service in allowed, "unexpected service")
host = item["HostConfig"]
check(
host["Memory"] > 0 and host["NanoCpus"] > 0,
service + ": missing CPU/RAM limit",
)
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
check(
not env_names
& {
"DB_PASSWORD",
"REDIS_PASSWORD",
"POSTGRES_PASSWORD",
"REDISCLI_AUTH",
"DATABASE_URL",
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
},
service + ": password in environment",
)
secrets = {
Path(mount["Destination"]).name
for mount in item["Mounts"]
if mount["Destination"].startswith("/run/secrets/")
}
check(secrets == allowed[service], service + ": incorrect secret scope")
if service in {"api", "minio"}:
check(not host["PortBindings"], service + " publishes ports")
for mount in item["Mounts"]:
if mount["Destination"].startswith("/run/secrets/"):
check(not mount["RW"], "writable secret mount")
check(
"container_name" not in project.config["services"][service],
"fixed container name",
)
check(
len(containers) == len(allowed) - 1 + len(project.replicas()),
"unexpected container count",
)
print(
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
flush=True,
)
async def graceful(project, base_url):
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
replicas = project.replicas()
target, runner = replicas[:2]
client = Client(base_url)
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
raw = json.dumps([actor["id"], "POST", path, key]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
database = project.config["services"]["postgres"]
env = database["environment"]
conn = await asyncpg.connect(
host="127.0.0.1",
port=int(database["ports"][0]["published"]),
database=env["POSTGRES_DB"],
user=env["POSTGRES_USER"],
password=(ROOT / "secrets/db_password").read_text().strip(),
timeout=3,
)
network = project.config['networks']['default']['name']
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
address = project.docker(
"inspect",
"--format",
address_template,
target.id,
)
check(bool(address), 'API has no address in the application network')
port = project.config["services"]["api"]["environment"]["APP_PORT"]
direct = f"http://{address}:{port}"
started = time.monotonic()
task = None
try:
async with conn.transaction():
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
task = asyncio.create_task(
asyncio.to_thread(
probe,
project,
runner.id,
[
{
"url": direct + path,
"method": "POST",
"body": {"text": "graceful-" + key},
"headers": {
"Authorization": "Bearer " + actor["token"],
"Idempotency-Key": key,
"Content-Type": "application/json",
},
}
],
)
)
deadline = time.monotonic() + 3
while not await conn.fetchval(
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
):
check(time.monotonic() < deadline, "POST did not reach database lock")
await asyncio.sleep(0.02)
started = time.monotonic()
await asyncio.to_thread(
project.docker, "kill", "--signal", "SIGTERM", target.id
)
results = await asyncio.to_thread(
probe,
project,
runner.id,
[
{"url": direct + "/health/ready"},
{"url": direct + "/health/live"},
{"url": direct + "/api/v1/users/me"},
],
)
check(
[item["status"] for item in results] == [503, 200, 503],
"incorrect drain readiness/liveness/new work",
)
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
result = (await task)[0]
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
state = json.loads(
project.docker("inspect", "--format", "{{json .State}}", target.id)
)
check(
not state["OOMKilled"] and state["ExitCode"] != 137,
"forced shutdown instead of graceful completion",
)
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
items = await asyncio.to_thread(history, client, actor, chat)
check(
result["body"]["id"] in {item["id"] for item in items},
"acknowledged in-flight message lost",
)
print(
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
flush=True,
)
finally:
await conn.close()
if task:
await asyncio.gather(task, return_exceptions=True)
if any(item.id == target.id and item.running for item in project.replicas()):
await asyncio.to_thread(project.docker, "stop", target.id)
await asyncio.to_thread(project.docker, "start", target.id)
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
class ObservedProject(ComposeProject):
"""Подсчитывает миграции и проверяет готовность перед заменой."""
def __init__(self):
self.migrations = 0
self.replacements = 0
self.replacement_started = []
super().__init__()
def run(self, *args, **kwargs):
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
self.migrations += 1
return super().run(*args, **kwargs)
def docker(self, *args, **kwargs):
if args[0] == "stop":
healthy = [
item
for item in self.replicas()
if item.id != args[1] and item.running and item.health == "healthy"
]
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
self.replacements += 1
self.replacement_started.append(time.monotonic())
return super().docker(*args, **kwargs)
def rolling(base_url):
"""Заменяет образ под постоянной записью с проверкой сохранности."""
project = ObservedProject()
before = project.replicas()
client = Client(base_url)
actor, _, _, chat = client.bootstrap()
stop = Event()
results, errors = [], []
def worker():
local = Client(base_url, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "rolling-" + key, key=key)
results.append(
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
)
key = uuid4().hex
except Failure as error:
errors.append(failure_kind(error))
stop.wait(0.08)
with ThreadPoolExecutor(max_workers=2) as pool:
jobs = [pool.submit(worker) for _ in range(2)]
try:
time.sleep(1)
started = time.monotonic()
update(project, base_url, revision="verification-" + uuid4().hex[:12])
finished = time.monotonic()
time.sleep(1)
finally:
stop.set()
for job in jobs:
job.result(timeout=5)
after = project.replicas()
check(
project.migrations == 1 and project.replacements == len(before),
"migration/replacement count incorrect",
)
check(
{item.image for item in before}.isdisjoint({item.image for item in after}),
"new image was not deployed",
)
check(
{item.id for item in before}.isdisjoint({item.id for item in after}),
"old containers remain",
)
writes = [row for row in results if started <= row[0] <= finished]
check(bool(writes), "no writes during update")
check(
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
"no writes during container replacement",
)
items = history(client, actor, chat)
check(
{row[2] for row in results} <= {item["id"] for item in items},
"rolling update lost confirmed messages",
)
check(
len(items) == len({item["text"] for item in items}),
"rolling update created duplicate messages",
)
latencies = sorted(row[3] for row in writes)
print(
json.dumps(
{
"scenario": "rolling update",
"replacements": project.replacements,
"migration_runs": project.migrations,
"successful_writes": len(writes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(finished - started, 3),
"successful_rps": round(len(writes) / (finished - started), 3),
"p95_seconds": round(
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
),
}
),
flush=True,
)
def main():
"""Выполняет проверки только на отдельном тестовом проекте."""
def configure(parser):
parser.add_argument(
"--phase", choices=("all", "update-and-failover"), default="all"
)
args = arguments("Grade 4 integration checks", configure)
project = ComposeProject()
check(project.name.endswith("-test"), "use isolated test Compose project")
with deployment_lock(project):
audit(project)
if args.phase == "all":
asyncio.run(graceful(project, args.base_url))
rolling(args.base_url)
args.duration, args.concurrency = 10, 2
exercise(project, args)
audit(project)
if __name__ == "__main__":
run(main)
+39
View File
@@ -0,0 +1,39 @@
"""Проверяет конкурентные повторы через балансировщик."""
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from tests.client import Client, arguments, check, run
def main() -> None:
"""Отправляет один ключ параллельно через разные реплики."""
args = arguments("Concurrent idempotency through load balancer")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = uuid4().hex
body = "parallel-" + key
def send(_):
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send, range(24)))
ids = {result["json"]["id"] for result in results}
replicas = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel retries created duplicate messages")
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
history = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]["items"]
check(
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
)
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
if __name__ == "__main__":
run(main)
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
from concurrent.futures import ThreadPoolExecutor
import gzip
import json
from pathlib import Path
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
IMAGE = "nginx:1.28.0-alpine"
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
def check(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
class Stand:
def __init__(self, directory: str) -> None:
self.directory = Path(directory)
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
self.base_url = ""
backend = self.directory / "backend.conf"
backend.write_text(
"worker_processes 1;\n"
"events { worker_connections 1024; }\n"
"http {\n"
" log_format backend 'BACKEND $request_method $uri';\n"
" access_log /dev/stdout backend;\n"
" server {\n"
" listen 8080;\n"
" default_type application/json;\n"
" add_header X-Instance-Id $hostname always;\n"
" add_header X-Request-Id $request_id always;\n"
" add_header Retry-After 7 always;\n"
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
" location = /api/limited {\n"
' return 429 \'{"error":{"code":"rate_limited",'
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
" }\n"
" location = /api/post-failure {\n"
' return 503 \'{"error":{"code":"unavailable",'
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
" }\n"
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
" }\n"
"}\n"
)
self.compose_file = self.directory / "compose.json"
self.compose_file.write_text(
json.dumps(
{
"services": {
"api": {
"image": IMAGE,
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
},
"nginx": {
"image": IMAGE,
"volumes": [
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
],
"ports": ["127.0.0.1::8080"],
},
}
}
)
)
def compose(self, *args: str) -> str:
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
"/dev/null",
"-f",
str(self.compose_file),
"-p",
self.project,
*args,
],
cwd=self.directory,
capture_output=True,
text=True,
timeout=45,
)
if result.returncode:
raise RuntimeError(result.stdout + result.stderr)
return result.stdout + result.stderr
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
request = Request(
self.base_url + path,
method=method,
headers={"Accept-Encoding": encoding},
data=b"{}" if method == "POST" else None,
)
try:
response = urlopen(request, timeout=3)
except HTTPError as error:
response = error
except (URLError, TimeoutError):
return 0, {}, b""
with response:
return (
response.status,
{key.lower(): value for key, value in response.headers.items()},
response.read(),
)
def wait_instances(self, count: int) -> set[str]:
seen: set[str] = set()
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
status, headers, _ = self.call("/health/live")
if status == 200:
seen.add(headers["x-instance-id"])
if len(seen) == count:
return seen
time.sleep(0.05)
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
def check_error(response, status: int, code: str) -> None:
actual_status, headers, body = response
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
check(
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
)
error = json.loads(body)["error"]
check(error["code"] == code, "Неверный код ошибки")
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
def run_checks(stand: Stand) -> None:
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
address = stand.compose("port", "nginx", "8080").strip()
stand.base_url = "http://" + address
stand.wait_instances(2)
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
stand.compose("exec", "-T", "nginx", "nginx", "-t")
print("PASS: nginx -t и балансировка двух реплик")
status, headers, body = stand.call("/api/large", encoding="gzip")
check(
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
)
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
check(
"accept-encoding" in headers.get("vary", "").lower(),
"Нет Vary: Accept-Encoding",
)
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
small = stand.call("/health/live", encoding="gzip")
check(
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
)
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
limited = stand.call("/api/limited")
check_error(limited, 429, "rate_limited")
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
print("PASS: JSON и заголовки ошибки API передаются без подмены")
failure = stand.call("/api/post-failure", method="POST")
check_error(failure, 503, "unavailable")
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
print("PASS: отправленный POST не повторяется на другой реплике")
with ThreadPoolExecutor(max_workers=32) as pool:
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
rejected = [response for response in responses if response[0] == 429]
check(bool(rejected), "Не удалось вызвать ограничение частоты")
check(
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
)
for response in rejected:
check_error(response, 429, "rate_limited")
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
with ThreadPoolExecutor(max_workers=16) as pool:
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
check(all(response[0] == 200 for response in health), "Лимит затронул health")
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
stand.compose(
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
)
instances = stand.wait_instances(3)
print("PASS: третья реплика обнаружена через DNS без reload nginx")
target = sorted(instances)[0]
containers = stand.compose("ps", "-q", "api").splitlines()
matching = [container for container in containers if container.startswith(target)]
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
# Все id взяты только из уникального временного Compose-проекта.
started = time.monotonic()
subprocess.run(
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
)
recovery = None
for _ in range(20):
status, headers, _ = stand.call("/health/live")
if status == 200:
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
if recovery is None:
recovery = time.monotonic() - started
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
stand.compose("stop", "--timeout", "1", "api")
unavailable = stand.call("/health/ready")
check_error(unavailable, 503, "unavailable")
check(
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
)
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
print("PASS: недоступность всех backend возвращает JSON 503")
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
check(bool(entries), "Нет JSON access log")
check(
all("request_id" in entry and "upstream_time" in entry for entry in entries),
"Неполный лог",
)
check(
any(entry["limit_status"] == "REJECTED" for entry in entries),
"Лимит не отражён в логе",
)
print("PASS: JSON access log содержит время backend и результат лимита")
def main() -> None:
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
stand = Stand(directory)
try:
run_checks(stand)
except Exception:
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
raise
finally:
stand.compose("down", "--remove-orphans", "--timeout", "1")
if __name__ == "__main__":
main()

Some files were not shown because too many files have changed in this diff Show More