From 0a703588290e26d0f148b45d0e6bee2e8eefce7d Mon Sep 17 00:00:00 2001 From: gytall Date: Sat, 7 Nov 2026 16:10:32 +0300 Subject: [PATCH 1/6] prepared skeleton for lab5 --- .dockerignore | 1 + .env.example | 78 +++- .gitignore | 6 + Dockerfile | 31 ++ Makefile | 206 +++++++++- REPORT.md | 76 +++- alembic.ini | 38 ++ app/__init__.py | 0 app/__main__.py | 4 + app/core/__init__.py | 0 app/core/cleanup.py | 60 +++ app/core/config.py | 129 +++++++ app/core/database.py | 28 ++ app/core/image_worker.py | 24 ++ app/core/instance.py | 40 ++ app/core/job_worker.py | 17 + app/core/lifecycle.py | 50 +++ app/core/outbox_publisher.py | 15 + app/core/redis.py | 25 ++ app/core/request_capacity.py | 42 ++ app/core/request_id.py | 22 ++ app/core/s3.py | 35 ++ app/core/security.py | 63 +++ app/domain/__init__.py | 1 + app/domain/attachment_jobs.py | 45 +++ app/domain/attachments.py | 315 +++++++++++++++ app/domain/auth.py | 84 ++++ app/domain/chats.py | 104 +++++ app/domain/health.py | 33 ++ app/domain/idempotency.py | 72 ++++ app/domain/image_pipeline.py | 34 ++ app/domain/image_validation.py | 23 ++ app/domain/images.py | 48 +++ app/domain/messages.py | 43 +++ app/domain/pagination.py | 83 ++++ app/domain/users.py | 41 ++ app/endpoints/__init__.py | 0 app/endpoints/attachments.py | 52 +++ app/endpoints/auth.py | 32 ++ app/endpoints/chats.py | 32 ++ app/endpoints/dependencies.py | 112 ++++++ app/endpoints/health.py | 20 + app/endpoints/messages.py | 38 ++ app/endpoints/users.py | 17 + app/errors/__init__.py | 0 app/errors/exceptions.py | 109 ++++++ app/errors/handlers.py | 225 +++++++++++ app/main.py | 92 +++++ app/models/__init__.py | 10 + app/models/attachment.py | 57 +++ app/models/base.py | 27 ++ app/models/chat.py | 32 ++ app/models/chat_member.py | 26 ++ app/models/counter.py | 13 + app/models/idempotency_key.py | 24 ++ app/models/job.py | 16 + app/models/message.py | 30 ++ app/models/user.py | 31 ++ app/schemas/__init__.py | 6 + app/schemas/attachment.py | 23 ++ app/schemas/base.py | 35 ++ app/schemas/chat.py | 22 ++ app/schemas/job.py | 10 + app/schemas/message.py | 41 ++ app/schemas/session.py | 17 + app/schemas/user.py | 25 ++ app/storage/__init__.py | 1 + app/storage/attachments.py | 99 +++++ app/storage/broker.py | 27 ++ app/storage/chats.py | 71 ++++ app/storage/health.py | 15 + app/storage/idempotency.py | 63 +++ app/storage/jobs.py | 37 ++ app/storage/messages.py | 62 +++ app/storage/outbox.py | 23 ++ app/storage/s3.py | 180 +++++++++ app/storage/sessions.py | 38 ++ app/storage/types.py | 6 + app/storage/users.py | 47 +++ compose.test.yaml | 10 + compose.yaml | 295 ++++++++++++++ infra/README.md | 10 + infra/minio/Dockerfile | 22 ++ infra/minio/bootstrap.py | 117 ++++++ infra/nginx/nginx.conf | 140 +++++++ infra/redis/redis.conf | 5 + infra/redis/start-redis.sh | 10 + migrations/env.py | 68 ++++ migrations/script.py.mako | 25 ++ migrations/versions/.gitkeep | 0 .../49d2c947a3f1_add_idempotency_keys.py | 43 +++ .../versions/8f31b94a2d10_add_attachments.py | 61 +++ .../versions/abd273076068_add_counters.py | 33 ++ .../db06fd3126ff_create_initial_schema.py | 107 +++++ pytest.ini | 6 + requirements.txt | 52 +++ scripts/__init__.py | 1 + scripts/check_grade4.py | 355 +++++++++++++++++ scripts/check_idempotency.py | 39 ++ scripts/check_nginx.py | 261 +++++++++++++ scripts/check_s3.py | 137 +++++++ scripts/collect_evidence.py | 204 ++++++++++ scripts/deployment.py | 107 +++++ scripts/expiry_demo.sh | 28 ++ scripts/explain_history.py | 132 +++++++ scripts/failover_all.py | 148 +++++++ scripts/failure_demo.sh | 64 +++ scripts/init_local.py | 30 ++ scripts/restart.sh | 32 ++ scripts/rolling_update.py | 92 +++++ scripts/s3_faults.py | 108 ++++++ scripts/stop-one.sh | 30 ++ scripts/upload_load.py | 113 ++++++ student-tests/conftest.py | 202 ++++++++++ student-tests/helpers.py | 88 +++++ student-tests/test_attachments.py | 365 ++++++++++++++++++ student-tests/test_auth.py | 298 ++++++++++++++ student-tests/test_chats.py | 209 ++++++++++ student-tests/test_concurrency.py | 266 +++++++++++++ student-tests/test_constraints.py | 46 +++ student-tests/test_errors.py | 76 ++++ student-tests/test_failures.py | 138 +++++++ student-tests/test_health.py | 62 +++ student-tests/test_history_plan.py | 78 ++++ student-tests/test_idempotency.py | 156 ++++++++ student-tests/test_instance.py | 106 +++++ student-tests/test_lifecycle.py | 50 +++ student-tests/test_limits.py | 98 +++++ student-tests/test_messages.py | 238 ++++++++++++ student-tests/test_migrations.py | 67 ++++ student-tests/test_pagination.py | 137 +++++++ student-tests/test_s3_stream.py | 56 +++ student-tests/test_sessions.py | 173 +++++++++ student-tests/test_users.py | 128 ++++++ tests/__init__.py | 0 135 files changed, 9647 insertions(+), 34 deletions(-) create mode 100644 Dockerfile create mode 100644 alembic.ini create mode 100644 app/__init__.py create mode 100644 app/__main__.py create mode 100644 app/core/__init__.py create mode 100644 app/core/cleanup.py create mode 100644 app/core/config.py create mode 100644 app/core/database.py create mode 100644 app/core/image_worker.py create mode 100644 app/core/instance.py create mode 100644 app/core/job_worker.py create mode 100644 app/core/lifecycle.py create mode 100644 app/core/outbox_publisher.py create mode 100644 app/core/redis.py create mode 100644 app/core/request_capacity.py create mode 100644 app/core/request_id.py create mode 100644 app/core/s3.py create mode 100644 app/core/security.py create mode 100644 app/domain/__init__.py create mode 100644 app/domain/attachment_jobs.py create mode 100644 app/domain/attachments.py create mode 100644 app/domain/auth.py create mode 100644 app/domain/chats.py create mode 100644 app/domain/health.py create mode 100644 app/domain/idempotency.py create mode 100644 app/domain/image_pipeline.py create mode 100644 app/domain/image_validation.py create mode 100644 app/domain/images.py create mode 100644 app/domain/messages.py create mode 100644 app/domain/pagination.py create mode 100644 app/domain/users.py create mode 100644 app/endpoints/__init__.py create mode 100644 app/endpoints/attachments.py create mode 100644 app/endpoints/auth.py create mode 100644 app/endpoints/chats.py create mode 100644 app/endpoints/dependencies.py create mode 100644 app/endpoints/health.py create mode 100644 app/endpoints/messages.py create mode 100644 app/endpoints/users.py create mode 100644 app/errors/__init__.py create mode 100644 app/errors/exceptions.py create mode 100644 app/errors/handlers.py create mode 100644 app/main.py create mode 100644 app/models/__init__.py create mode 100644 app/models/attachment.py create mode 100644 app/models/base.py create mode 100644 app/models/chat.py create mode 100644 app/models/chat_member.py create mode 100644 app/models/counter.py create mode 100644 app/models/idempotency_key.py create mode 100644 app/models/job.py create mode 100644 app/models/message.py create mode 100644 app/models/user.py create mode 100644 app/schemas/__init__.py create mode 100644 app/schemas/attachment.py create mode 100644 app/schemas/base.py create mode 100644 app/schemas/chat.py create mode 100644 app/schemas/job.py create mode 100644 app/schemas/message.py create mode 100644 app/schemas/session.py create mode 100644 app/schemas/user.py create mode 100644 app/storage/__init__.py create mode 100644 app/storage/attachments.py create mode 100644 app/storage/broker.py create mode 100644 app/storage/chats.py create mode 100644 app/storage/health.py create mode 100644 app/storage/idempotency.py create mode 100644 app/storage/jobs.py create mode 100644 app/storage/messages.py create mode 100644 app/storage/outbox.py create mode 100644 app/storage/s3.py create mode 100644 app/storage/sessions.py create mode 100644 app/storage/types.py create mode 100644 app/storage/users.py create mode 100644 compose.test.yaml create mode 100644 compose.yaml create mode 100644 infra/minio/Dockerfile create mode 100644 infra/minio/bootstrap.py create mode 100644 infra/nginx/nginx.conf create mode 100644 infra/redis/redis.conf create mode 100644 infra/redis/start-redis.sh create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/.gitkeep create mode 100644 migrations/versions/49d2c947a3f1_add_idempotency_keys.py create mode 100644 migrations/versions/8f31b94a2d10_add_attachments.py create mode 100644 migrations/versions/abd273076068_add_counters.py create mode 100644 migrations/versions/db06fd3126ff_create_initial_schema.py create mode 100644 pytest.ini create mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/check_grade4.py create mode 100644 scripts/check_idempotency.py create mode 100644 scripts/check_nginx.py create mode 100644 scripts/check_s3.py create mode 100644 scripts/collect_evidence.py create mode 100644 scripts/deployment.py create mode 100755 scripts/expiry_demo.sh create mode 100644 scripts/explain_history.py create mode 100644 scripts/failover_all.py create mode 100755 scripts/failure_demo.sh create mode 100644 scripts/init_local.py create mode 100755 scripts/restart.sh create mode 100644 scripts/rolling_update.py create mode 100644 scripts/s3_faults.py create mode 100755 scripts/stop-one.sh create mode 100644 scripts/upload_load.py create mode 100644 student-tests/conftest.py create mode 100644 student-tests/helpers.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_chats.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_constraints.py create mode 100644 student-tests/test_errors.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_health.py create mode 100644 student-tests/test_history_plan.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_instance.py create mode 100644 student-tests/test_lifecycle.py create mode 100644 student-tests/test_limits.py create mode 100644 student-tests/test_messages.py create mode 100644 student-tests/test_migrations.py create mode 100644 student-tests/test_pagination.py create mode 100644 student-tests/test_s3_stream.py create mode 100644 student-tests/test_sessions.py create mode 100644 student-tests/test_users.py create mode 100644 tests/__init__.py diff --git a/.dockerignore b/.dockerignore index bc16e556..47a11469 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +docs diff --git a/.env.example b/.env.example index 119384b5..ac8273f4 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,81 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Скопируйте в .env для локального окружения; .env не коммитится. +# Настройки приложения и Compose; параметры будущей очереди пока не используются. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=INFO + +MAX_MESSAGES=100000 +# Лимит одновременных запросов НА ОДНУ реплику +MAX_IN_FLIGHT=16 + SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_STORE=redis + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DB=0 +REDIS_MAX_CONNECTIONS=20 +REDIS_POOL_TIMEOUT_SECONDS=1 +REDIS_CONNECT_TIMEOUT_SECONDS=1 +REDIS_SOCKET_TIMEOUT_SECONDS=0.5 + +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=messenger +DB_USER=messenger + +# Пул на одну реплику: суммарно API_REPLICAS * (POOL_SIZE + MAX_OVERFLOW) +# соединений должно помещаться в max_connections PostgreSQL. +DB_POOL_SIZE=5 +DB_MAX_OVERFLOW=10 +DB_POOL_TIMEOUT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 + +# --- Балансировка и идемпотентность --- API_REPLICAS=2 +API_IMAGE=mtusi-messenger:lab5 +API_BUILD_REVISION=development IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +SHUTDOWN_DRAIN_SECONDS=1 +GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5 +READINESS_TIMEOUT_SECONDS=2 + +# Порт единственной публичной точки входа (nginx) +LB_HOST_PORT=18081 +# Используются только тестовым стендом (compose.test.yaml) +DB_HOST_PORT=25432 +REDIS_HOST_PORT=26379 + +# --- Lab4: MinIO, ограниченный upload и очистка журнала --- +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +S3_CONNECT_TIMEOUT_SECONDS=2 +S3_READ_TIMEOUT_SECONDS=10 +S3_UPLOAD_TIMEOUT_SECONDS=30 +S3_DOWNLOAD_TIMEOUT_SECONDS=60 +S3_IMAGE_TIMEOUT_SECONDS=10 +S3_MAX_UPLOAD_BYTES=10485760 +S3_MAX_IMAGE_PIXELS=40000000 +S3_CHUNK_BYTES=65536 +S3_MAX_CONCURRENT_UPLOADS=2 +S3_CLEANUP_GRACE_SECONDS=86400 +S3_CLEANUP_INTERVAL_SECONDS=300 + +# Пути секретов внутри контейнеров; должны совпадать с secrets mounts. +POSTGRES_PASSWORD_FILE=/run/secrets/db_password +MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user +MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password +MINIO_BROWSER=off + +# Lab5: будущие параметры очереди/worker, пока без реализации. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +# Брокер пока не выбран; файл ещё не создаётся и не монтируется. +BROKER_URL_FILE=/run/secrets/broker_url diff --git a/.gitignore b/.gitignore index b5a71896..f1736d4c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,11 @@ coverage/ .DS_Store .idea/ .vscode/ +venv/ +.pytest_cache/ +.ruff_cache/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +# Private working documentation +/docs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..17f476a1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12.15-slim-bookworm + +LABEL io.mtusi.messenger.secrets="file-v1" + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +RUN useradd --system --uid 10001 --no-create-home appuser + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY alembic.ini ./ +COPY migrations ./migrations +COPY app ./app + +ARG API_BUILD_REVISION +LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"] + +CMD ["python", "-m", "app"] diff --git a/Makefile b/Makefile index 5d86d171..112af63c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,86 @@ -PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +-include .env + +PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard venv/bin/python),venv/bin/python,python3)) +PYTEST_ARGS ?= +BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080) CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +COMPOSE = docker compose --env-file .env + +# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis +# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh. +TEST_PROJECT ?= mtusi-messenger-lab5-test +TEST_LB_PORT ?= 28080 +TEST_DB_PORT ?= $(DB_HOST_PORT) +TEST_REDIS_PORT ?= $(REDIS_HOST_PORT) +TEST_BASE_URL = http://localhost:$(TEST_LB_PORT) +TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \ + LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT) +TEST_COMPOSE = $(TEST_ENV) $(COMPOSE) + +TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file /dev/null && exit 0; sleep 1; \ + done; echo "API $(1) не готов за 90 секунд" >&2; exit 1 +endef + +.PHONY: help check validate test load test-persistence test-failover \ + init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \ + up down restart restore ps logs migrate migration instances \ + test-stand-up test-stand-down test-stand-destroy test-stand-ps \ + test-stand-smoke test-stand-persistence test-stand-failover \ + test-own test-docker test-limits test-concurrency test-migrations test-all \ + explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \ + rolling-update failover-all test-lab3-grade4 help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply" + @echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh" + @echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh" + @echo "Основной: up | down | restart | restore | ps | logs | migrate | instances" + @echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps" + @echo " test-stand-smoke | test-stand-persistence | test-stand-failover" + @echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all" + @echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)" + @echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)" + @echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)" + @echo "База lab3: rolling-update | failover-all" + @echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)" + @echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)" + +# --- Проверки API и сценариев отказа (tests/) --- + +init: + $(PYTHON) scripts/init_local.py + +test-grade4: + $(MAKE) check validate + $(MAKE) test-stand-up + $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover + $(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults + $(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10 + $(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)" + +test-s3: + $(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults + +load-uploads: + $(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10 + +cleanup-dry-run: + $(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once + +cleanup-apply: + $(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once --apply check: $(PYTHON) scripts/check_contract.py @@ -15,6 +88,22 @@ check: validate: $(PYTHON) scripts/validate_openapi.py +test-nginx: + $(PYTHON) scripts/check_nginx.py + +test-lab3-grade3: + @set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \ + $(MAKE) check validate; \ + $(MAKE) test-stand-up; \ + $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover + +test-lab3-grade4: + @set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \ + $(MAKE) check validate; \ + $(MAKE) test-stand-up; \ + $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \ + $(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)" + test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +117,112 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --- Основной стенд --- + +rolling-update: + $(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS) + +failover-all: + $(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS) + +up: + $(COMPOSE) up -d --build + $(call wait_ready,$(BASE_URL)) + +down: + $(COMPOSE) down + +restart: + scripts/restart.sh + $(call wait_ready,$(BASE_URL)) + +# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры). +restore: + $(COMPOSE) up -d --wait api + +ps: + $(COMPOSE) ps + +logs: + $(COMPOSE) logs -f api nginx + +migrate: + $(COMPOSE) run --rm --build migrate + +# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id). +instances: + @for i in $$(seq 1 10); do \ + curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \ + done + +# Генерация миграции против БД тестового стенда (основной стенд БД не публикует). +migration: + DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)" + +# --- Тестовый стенд --- + +test-stand-up: + $(TEST_COMPOSE) up -d --build + $(call wait_ready,$(TEST_BASE_URL)) + $(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true' + +test-stand-down: + $(TEST_COMPOSE) down + +test-stand-destroy: + $(TEST_COMPOSE) down -v + +test-stand-ps: + $(TEST_COMPOSE) ps + +test-stand-smoke: + $(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)" + +test-stand-idempotency: + $(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)" + +test-stand-persistence: + $(TEST_COMPOSE) ps + $(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)" + $(TEST_COMPOSE) ps + +test-stand-failover: + $(TEST_COMPOSE) ps + $(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)" + $(TEST_COMPOSE) ps -a api + $(TEST_COMPOSE) up -d --wait api + +# --- Собственные тесты (student-tests/) --- + +test-own: + $(PYTHON) -m pytest -q $(PYTEST_ARGS) + +test-docker: + $(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS) + +test-limits: + $(PYTHON) -m pytest student-tests/test_limits.py -q + +test-concurrency: + $(PYTHON) -m pytest student-tests/test_concurrency.py -q + +test-migrations: + $(PYTHON) -m pytest student-tests/test_migrations.py -q + +test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence + +# --- Демо (на тестовом стенде) --- + +explain: + $(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py + +expiry-demo: test-stand-up + $(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api + $(call wait_ready,$(TEST_BASE_URL)) + BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh + $(TEST_COMPOSE) up -d api + $(call wait_ready,$(TEST_BASE_URL)) + +failure-demo: test-stand-up + $(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh diff --git a/REPORT.md b/REPORT.md index e98f75e9..037d0fea 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,75 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4). +- Целевая оценка: 5; исследовательский трек пока не выбран. +- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5. +- Дата переноса: 07.11.2026. +- Commit/PR lab5: заполнить после публикации. +- Использованные источники: README-файлы и contracts/ репозитория lab5, + исходники lab4. Использована помощь Codex. Новых зависимостей нет. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make init +make up +``` + +HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных. +Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`. +Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда, +`docker compose --env-file .env down --volumes` для основного. +Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4. +Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии, +идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные +тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO +bootstrap из исходного рабочего дерева. Исходный lab4 не изменён. + +Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки — +.env.example; там же указаны будущие параметры очереди, пока не используемые. +Compose получает настройки environment из .env, включая пути S3, PostgreSQL +и MinIO к файлам секретов; секретные значения остаются в secrets/. +Пути *_FILE в Compose также задают точки монтирования соответствующих secrets; +MinIO bootstrap читает эти пути из окружения. +Публичные tests/, contracts/, lab.json и README.md lab5 сохранены. + +Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher → +broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL. +Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready. +Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет | +| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 | +| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет | +| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет | +| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет | +| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет | +| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет | +| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет | +| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты, +версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность, +прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready, +RSS и другие метрики выбранного трека. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и +асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся. +Сначала просмотр пользователем и выбор трека/брокера, затем реализация; +тестовый прогон и новые evidence — по отдельному указанию после реализации. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..f95b20ac --- /dev/null +++ b/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..ee209955 --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +from app.main import main + +if __name__ == "__main__": + main() diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/core/cleanup.py b/app/core/cleanup.py new file mode 100644 index 00000000..7e755075 --- /dev/null +++ b/app/core/cleanup.py @@ -0,0 +1,60 @@ +"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис.""" + +import argparse +import asyncio +from dataclasses import asdict +import json + +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.s3 import S3Settings +from app.domain.attachments import reconcile_attachments +from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error +from app.storage.s3 import create_s3_storage + + +async def cleanup_once(factory, storage, apply: bool) -> bool: + completed = True + try: + async with factory() as session: + result = await reconcile_attachments(session, storage, dry_run=not apply) + print(json.dumps(asdict(result)), flush=True) + except RECOVERABLE_STORAGE_ERRORS as error: + log_recoverable_error("Reconciliation", error) + completed = False + return completed + + +async def run(once: bool, apply: bool) -> None: + settings = Settings() # type: ignore + s3_settings = S3Settings() + engine = create_db_engine(settings) + factory = create_session_factory(engine) + storage = create_s3_storage(s3_settings) + try: + while True: + completed = await cleanup_once(factory, storage, apply) + if once: + if not completed: + raise SystemExit(1) + break + await asyncio.sleep(s3_settings.cleanup_interval_seconds) + finally: + await storage.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД") + parser.add_argument("--once", action="store_true") + parser.add_argument( + "--apply", + action="store_true", + help="удалять безопасные кандидаты; по умолчанию dry-run", + ) + args = parser.parse_args() + asyncio.run(run(args.once, args.apply)) + + +if __name__ == "__main__": + main() diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 00000000..6b743709 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,129 @@ +from enum import StrEnum +from pathlib import Path +from typing import Literal, Self + +from pydantic import Field, SecretStr, model_validator +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) +from sqlalchemy import URL +from app.core.s3 import S3Settings + +ASYNC_DRIVER = "postgresql+asyncpg" +SECRETS_DIR = next( + ( + path + for path in ( + Path("/run/secrets"), + Path(__file__).resolve().parents[2] / "secrets", + ) + if (path / "db_password").is_file() + ), + None, +) + + +class LogLevel(StrEnum): + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +class Settings(BaseSettings): + s3_enabled: bool = True + s3: S3Settings = Field(default_factory=S3Settings) + app_host: str + app_port: int = Field(ge=1, le=65535) + app_env: str + log_level: LogLevel = LogLevel.INFO + + max_messages: int = Field(ge=1) + max_in_flight: int = Field(ge=1) + + session_ttl_seconds: int = Field(default=3600, ge=1) + session_store: Literal["redis"] = "redis" + + redis_host: str = "localhost" + redis_port: int = Field(default=6379, ge=1, le=65535) + redis_db: int = Field(default=0, ge=0) + redis_password: SecretStr | None = None + + redis_max_connections: int = Field(default=20, ge=1) + redis_pool_timeout_seconds: float = Field(default=1.0, gt=0) + redis_connect_timeout_seconds: float = Field(default=1.0, gt=0) + redis_socket_timeout_seconds: float = Field(default=0.5, gt=0) + + database_url: SecretStr | None = None + db_host: str | None = None + db_port: int = Field(default=5432, ge=1, le=65535) + db_name: str | None = None + db_user: str | None = None + db_password: SecretStr | None = None + + db_pool_size: int = Field(default=5, ge=1) + db_max_overflow: int = Field(default=5, ge=0) + db_pool_timeout_seconds: float = Field(default=3.0, gt=0) + db_connect_timeout_seconds: int = Field(default=3, ge=1) + db_statement_timeout_ms: int = Field(default=5000, ge=1) + + instance_id: str | None = None + idempotency_ttl_seconds: int = Field(default=86400, ge=1) + shutdown_drain_seconds: float = Field(default=1, ge=0) + graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1) + readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5) + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + secrets_dir=SECRETS_DIR, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Явные параметры и файлы имеют приоритет над окружением.""" + return init_settings, file_secret_settings, env_settings, dotenv_settings + + @model_validator(mode="after") + def _require_database_source(self) -> Self: + if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8: + raise ValueError( + "Drain и graceful shutdown должны суммарно занимать менее 8 секунд" + ) + if self.database_url is not None: + if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER): + raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://") + return self + + if not (self.db_host and self.db_name and self.db_user and self.db_password): + raise ValueError( + "Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password" + ) + return self + + def get_database_url(self) -> str: + if self.database_url is not None: + return self.database_url.get_secret_value() + + assert self.db_password is not None + url = URL.create( + ASYNC_DRIVER, + username=self.db_user, + password=self.db_password.get_secret_value(), + host=self.db_host, + port=self.db_port, + database=self.db_name, + ) + return url.render_as_string(hide_password=False) diff --git a/app/core/database.py b/app/core/database.py new file mode 100644 index 00000000..c7a65381 --- /dev/null +++ b/app/core/database.py @@ -0,0 +1,28 @@ +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.core.config import Settings + + +def create_db_engine(settings: Settings) -> AsyncEngine: + return create_async_engine( + settings.get_database_url(), + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout_seconds, + pool_pre_ping=True, + connect_args={ + "timeout": settings.db_connect_timeout_seconds, + "server_settings": { + "statement_timeout": str(settings.db_statement_timeout_ms) + }, + }, + ) + + +def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False) diff --git a/app/core/image_worker.py b/app/core/image_worker.py new file mode 100644 index 00000000..a74795c3 --- /dev/null +++ b/app/core/image_worker.py @@ -0,0 +1,24 @@ +"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU""" + +from pathlib import Path +import resource +import sys + +from app.domain.image_validation import decode_image +from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code + + +def main() -> int: + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) + path, content_type, maximum = sys.argv[1:] + status = 0 + try: + decode_image(Path(path), content_type, int(maximum)) + except IMAGE_WORKER_ERRORS as error: + status = image_worker_error_code(error) + return status + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/core/instance.py b/app/core/instance.py new file mode 100644 index 00000000..75bf8449 --- /dev/null +++ b/app/core/instance.py @@ -0,0 +1,40 @@ +"""Идентификатор API-реплики и заголовок X-Instance-Id.""" + +import os +from uuid import uuid4 + +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response + +INSTANCE_ID_HEADER = "X-Instance-Id" + + +def resolve_instance_id(configured: str | None) -> str: + """Выбирает INSTANCE_ID, HOSTNAME или UUID при старте приложения.""" + if configured and configured.strip(): + return configured.strip() + + hostname = os.getenv("HOSTNAME", "").strip() + if hostname: + return hostname + + return str(uuid4()) + + +class InstanceIdMiddleware(BaseHTTPMiddleware): + """Добавляет сохранённый идентификатор реплики в ответ.""" + + def __init__(self, app, instance_id: str) -> None: + super().__init__(app) + self.instance_id = instance_id + + async def dispatch( + self, + request: Request, + call_next: RequestResponseEndpoint, + ) -> Response: + response = await call_next(request) + response.headers[INSTANCE_ID_HEADER] = self.instance_id + + return response diff --git a/app/core/job_worker.py b/app/core/job_worker.py new file mode 100644 index 00000000..14f36298 --- /dev/null +++ b/app/core/job_worker.py @@ -0,0 +1,17 @@ +"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker. + +Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная +concurrency/prefetch, восстановление pending и остановка с сохранением работы. +Каждая задача получает отдельную AsyncSession. Обработка изображений идёт +в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется. +В Compose и API модуль пока не подключён. +""" + + +async def run_worker() -> None: + """Запустить consumer с lease/retry, ограничением ресурсов и корректным ack. + + SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка + worker не заставляет API выполнять resize. Инициализация ресурсов — здесь. + """ + raise NotImplementedError("Lab5: отдельный worker ещё не реализован") diff --git a/app/core/lifecycle.py b/app/core/lifecycle.py new file mode 100644 index 00000000..c9e06a8c --- /dev/null +++ b/app/core/lifecycle.py @@ -0,0 +1,50 @@ +"""Готовность реплики и мягкое завершение сервера.""" + +import asyncio +from types import FrameType + +import uvicorn + + +class Lifecycle: + """Хранит состояние завершения реплики.""" + + def __init__(self) -> None: + self._draining = False + + @property + def draining(self) -> bool: + return self._draining + + def start_draining(self) -> None: + """Отключает приём новых бизнес-запросов.""" + self._draining = True + + +class GracefulServer(uvicorn.Server): + """Оставляет время для исключения завершающейся реплики.""" + + def __init__( + self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float + ) -> None: + super().__init__(config) + self.lifecycle = lifecycle + self.drain_seconds = drain_seconds + self._drain_task: asyncio.Task[None] | None = None + + def handle_exit(self, sig: int, frame: FrameType | None) -> None: + """Первый сигнал запускает drain, повторный ускоряет остановку.""" + if self._drain_task is not None: + if not self._drain_task.done(): + self._drain_task.cancel() + super().handle_exit(sig, frame) + return + self.lifecycle.start_draining() + self._drain_task = asyncio.get_running_loop().create_task( + self._finish_draining(sig, frame) + ) + + async def _finish_draining(self, sig: int, frame: FrameType | None) -> None: + """Передаёт остановку Uvicorn после drain-периода.""" + await asyncio.sleep(self.drain_seconds) + super().handle_exit(sig, frame) diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py new file mode 100644 index 00000000..04f2cf58 --- /dev/null +++ b/app/core/outbox_publisher.py @@ -0,0 +1,15 @@ +"""Заготовка процесса доставки PostgreSQL outbox в durable broker. + +Не держать транзакцию/блокировки во время бесконечного сетевого ожидания. +Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish. +Процесс пока не подключён к Compose или жизненному циклу API. +""" + + +async def run_publisher() -> None: + """Публиковать сохранённые намерения и фиксировать подтверждение брокера. + + После crash между publish и отметкой допустим дубль того же job_id. + Недоступность брокера не должна терять уже принятые задания. + """ + raise NotImplementedError("Lab5: outbox publisher ещё не реализован") diff --git a/app/core/redis.py b/app/core/redis.py new file mode 100644 index 00000000..ea525f8b --- /dev/null +++ b/app/core/redis.py @@ -0,0 +1,25 @@ +from redis.asyncio import BlockingConnectionPool, Redis +from redis.asyncio.retry import Retry +from redis.backoff import NoBackoff + +from app.core.config import Settings + +def create_redis_client(settings: Settings) -> Redis: + password = ( + settings.redis_password.get_secret_value() if settings.redis_password else None + ) + pool = BlockingConnectionPool( + host=settings.redis_host, + port=settings.redis_port, + db=settings.redis_db, + password=password, + max_connections=settings.redis_max_connections, + timeout=settings.redis_pool_timeout_seconds, + socket_connect_timeout=settings.redis_connect_timeout_seconds, + socket_timeout=settings.redis_socket_timeout_seconds, + retry=Retry(NoBackoff(), retries=1), + decode_responses=True, + ) + return Redis.from_pool(pool) + + diff --git a/app/core/request_capacity.py b/app/core/request_capacity.py new file mode 100644 index 00000000..31968e70 --- /dev/null +++ b/app/core/request_capacity.py @@ -0,0 +1,42 @@ +from collections.abc import AsyncIterator + +from fastapi import Request + +from app.errors.exceptions import rate_limited, unavailable + + +class RequestCapacity: + def __init__(self, max_in_flight: int) -> None: + self._max_in_flight = max_in_flight + self._in_flight = 0 + + @property + def in_flight(self) -> int: + return self._in_flight + + def try_acquire(self) -> bool: + if self._in_flight >= self._max_in_flight: + return False + self._in_flight += 1 + return True + + def release(self) -> None: + self._in_flight -= 1 + + +def create_request_capacity(max_in_flight: int) -> RequestCapacity: + return RequestCapacity(max_in_flight) + + +async def acquire_request_slot(request: Request) -> AsyncIterator[None]: + if request.app.state.lifecycle.draining: + raise unavailable("Реплика завершает работу") + capacity: RequestCapacity = request.app.state.request_capacity + + if not capacity.try_acquire(): + raise rate_limited() + + try: + yield + finally: + capacity.release() diff --git a/app/core/request_id.py b/app/core/request_id.py new file mode 100644 index 00000000..a78a8607 --- /dev/null +++ b/app/core/request_id.py @@ -0,0 +1,22 @@ +from uuid import uuid4 + +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response + +REQUEST_ID_HEADER = "X-Request-Id" + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch( + self, + request: Request, + call_next: RequestResponseEndpoint, + ) -> Response: + request_id = str(uuid4()) + request.state.request_id = request_id + + response = await call_next(request) + response.headers[REQUEST_ID_HEADER] = request_id + + return response diff --git a/app/core/s3.py b/app/core/s3.py new file mode 100644 index 00000000..c90137f8 --- /dev/null +++ b/app/core/s3.py @@ -0,0 +1,35 @@ +"""Конфигурация MinIO; значения credentials читаются только из файлов.""" + +from pathlib import Path +from urllib.parse import urlsplit + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +class S3Settings(BaseSettings): + endpoint: str = "http://minio:9000" + region: str = "us-east-1" + bucket: str = "messenger-attachments" + access_key_id_file: Path = Path("/run/secrets/s3_access_key_id") + secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key") + connect_timeout_seconds: float = Field(default=2, gt=0, le=5) + read_timeout_seconds: float = Field(default=10, gt=0, le=15) + upload_timeout_seconds: float = Field(default=30, gt=0, le=60) + download_timeout_seconds: float = Field(default=60, gt=0, le=120) + image_timeout_seconds: float = Field(default=10, gt=0, le=15) + max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760) + max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000) + chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144) + max_concurrent_uploads: int = Field(default=2, ge=1, le=2) + cleanup_grace_seconds: int = Field(default=86_400, ge=300) + cleanup_interval_seconds: int = Field(default=300, ge=1) + + model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True) + + @model_validator(mode="after") + def validate_endpoint(self): + url = urlsplit(self.endpoint) + if (url.scheme not in {"http", "https"} or not url.hostname or url.username + or url.password or url.path not in {"", "/"} or url.query or url.fragment): + raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути") + return self diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 00000000..bfddf197 --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,63 @@ +"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий.""" + +import asyncio +import hashlib +import os +import secrets +from concurrent.futures import ThreadPoolExecutor + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +TOKEN_BYTES = 32 + +_hasher = PasswordHasher() +_executor = ThreadPoolExecutor( + max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)), + thread_name_prefix="argon2", +) + + +def _hash_password_sync(password: str) -> str: + """Блокирующий расчёт Argon2id-хеша со случайной солью в формате PHC.""" + return _hasher.hash(password) + + +def _verify_password_sync(password: str, password_hash: str) -> bool: + """Блокирующая проверка пароля по PHC-строке; False при несовпадении или повреждённом хеше.""" + try: + return _hasher.verify(password_hash, password) + except (VerificationError, InvalidHashError): + return False + + +_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES)) + + +async def hash_password(password: str) -> str: + """Хеширует пароль в пуле потоков, не блокируя event loop.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_executor, _hash_password_sync, password) + + +async def verify_password(password: str, password_hash: str) -> bool: + """Проверяет пароль в пуле потоков, не блокируя event loop.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + _executor, _verify_password_sync, password, password_hash + ) + + +async def dummy_verify_password(password: str) -> None: + """Проверка против фиктивного хеша: время ответа не выдаёт отсутствие пользователя.""" + await verify_password(password, _DUMMY_HASH) + + +def generate_token() -> str: + """Непрозрачный токен сессии: 32 случайных байта в base64url.""" + return secrets.token_urlsafe(TOKEN_BYTES) + + +def hash_token(token: str) -> str: + """SHA-256 токена в hex: детерминированный ключ для хранилища сессий.""" + return hashlib.sha256(token.encode()).hexdigest() diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 00000000..93c96d54 --- /dev/null +++ b/app/domain/__init__.py @@ -0,0 +1 @@ +"""Бизнес-логика: проверки доступа, транзакции и ошибки API.""" diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py new file mode 100644 index 00000000..01deb9bb --- /dev/null +++ b/app/domain/attachment_jobs.py @@ -0,0 +1,45 @@ +"""Заготовки сценариев lab5; не подключены к upload или worker. + +Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются +в storage/. Одна AsyncSession принадлежит одной последовательной задаче. +""" + +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + + +async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID: + """Создать job и outbox в транзакции queued metadata после durable staging. + + Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload; + 202 допустим только после устойчивого оригинала и успешного commit. + """ + raise NotImplementedError("Lab5: постановка задания ещё не реализована") + + +async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None: + """Захватить job с lease и опубликовать original и обязательные варианты. + + Ready-дубль не меняет результат. Lease и версия защищают от устаревшей + попытки. S3/metadata фиксируются до ack; отмена не подавляется. + """ + raise NotImplementedError("Lab5: обработка задания ещё не реализована") + + +async def recover_expired_jobs(session: AsyncSession) -> int: + """Вернуть зависший processing в очередь после истечения lease. + + Учитывать лимит попыток и защиту от конкурентного завершения старым worker. + Точное правило heartbeat/deadline определяется до реализации. + """ + raise NotImplementedError("Lab5: восстановление lease ещё не реализовано") + + +async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None: + """Явно повторить только failed job, сохранив аудит и ключ логического эффекта. + + Не переисполнять ready. Проверка допустимости, лимиты повторов и способ + операторского вызова будут определены вместе с реестром failed/DLQ. + """ + raise NotImplementedError("Lab5: replay ещё не реализован") diff --git a/app/domain/attachments.py b/app/domain/attachments.py new file mode 100644 index 00000000..3216c636 --- /dev/null +++ b/app/domain/attachments.py @@ -0,0 +1,315 @@ +"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO.""" + +from collections.abc import AsyncIterator, Iterator +from contextlib import contextmanager, suppress +from dataclasses import dataclass +import hashlib +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +from anyio import CancelScope +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.request_capacity import RequestCapacity +from app.domain.chats import ensure_chat_member +from app.domain.images import validate_filename, validate_image +from app.errors.exceptions import ( + conflict, + invalid_request, + not_found, + payload_too_large, + rate_limited, + unavailable, + unsupported_media_type, +) +from app.errors.handlers import ( + RECOVERABLE_STORAGE_ERRORS, + log_recoverable_error, + operation_timeout, +) +from app.schemas.attachment import Attachment +from app.schemas.user import User +from app.storage import attachments as repository +from app.storage.s3 import MinioStorage + + +@dataclass(frozen=True) +class CleanupResult: + examined: int + candidates: int + deleted: int + + +@dataclass(frozen=True) +class OriginalFile: + path: Path + size_bytes: int + sha256: str + + +def require_storage(storage: MinioStorage | None) -> MinioStorage: + if storage is None: + raise unavailable("Объектное хранилище не настроено") + return storage + + +def validate_declared_size(value: str | None, maximum: int) -> None: + if value is not None: + if not value.isascii() or not value.isdecimal() or len(value) > 20: + raise invalid_request("Некорректный Content-Length") + if int(value) > maximum: + raise payload_too_large() + + +@contextmanager +def upload_slot(capacity: RequestCapacity) -> Iterator[None]: + if not capacity.try_acquire(): + raise rate_limited() + try: + yield + finally: + capacity.release() + + +async def receive_attachment( + session: AsyncSession, + storage: MinioStorage | None, + user: User, + chat_id: UUID, + *, + capacity: RequestCapacity, + filename: str, + content_type: str, + declared_length: str | None, + chunks: AsyncIterator[bytes], +) -> Attachment: + storage = require_storage(storage) + validate_declared_size(declared_length, storage.settings.max_upload_bytes) + timeout_error = invalid_request("Истекло время загрузки") + with upload_slot(capacity): + async with operation_timeout( + storage.settings.upload_timeout_seconds, timeout_error + ): + result = await upload_attachment( + session, + storage, + user, + chat_id, + filename=filename, + content_type=content_type, + chunks=chunks, + ) + return result + + +async def read_original( + chunks: AsyncIterator[bytes], + path: Path, + maximum: int, + chunk_bytes: int, +) -> OriginalFile: + checksum = hashlib.sha256() + size = 0 + with path.open("wb") as target: + async for chunk in chunks: + size += len(chunk) + if size > maximum: + raise payload_too_large() + checksum.update(chunk) + view = memoryview(chunk) + for offset in range(0, len(view), chunk_bytes): + target.write(view[offset : offset + chunk_bytes]) + if size == 0: + raise unsupported_media_type("Пустой файл") + return OriginalFile(path, size, checksum.hexdigest()) + + +async def compensate_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: + """Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется.""" + await session.rollback() + intent = await repository.lock_upload(session, id) + if intent is not None: + if await repository.get_by_key(session, intent.object_key) is None: + await storage.delete_object(intent.object_key) + await repository.finish_upload(session, id) + await session.commit() + + +async def recover_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: + """Локальное восстановление после ошибки; журнал остаётся при отказе компенсации.""" + try: + await compensate_upload(session, storage, id) + except RECOVERABLE_STORAGE_ERRORS as error: + log_recoverable_error("Upload compensation", error) + # Ошибка повторного rollback не должна скрыть исходную ошибку или отмену. + with suppress(*RECOVERABLE_STORAGE_ERRORS): + await session.rollback() + + +async def publish_original( + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + original: OriginalFile, + filename: str, + content_type: str, +) -> Attachment: + id = uuid4() + key = f"originals/{uuid4().hex}" + # Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT. + await repository.reserve_upload(session, id, key) + await session.commit() + try: + if await repository.lock_upload(session, id) is None: + raise unavailable("Загрузка отменена до записи оригинала") + await storage.put_file( + key, + original.path, + size_bytes=original.size_bytes, + content_type=content_type, + sha256=original.sha256, + ) + record = await repository.create_attachment( + session, + id=id, + chat_id=chat_id, + owner_id=user.id, + object_key=key, + filename=filename, + content_type=content_type, + size_bytes=original.size_bytes, + sha256=original.sha256, + ) + await repository.finish_upload(session, id) + result = Attachment.model_validate(record) + await session.commit() + except BaseException: + # Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT. + with CancelScope(shield=True): + await recover_upload(session, storage, id) + raise + return result + + +async def upload_attachment( + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + filename: str, + content_type: str, + chunks: AsyncIterator[bytes], +) -> Attachment: + await ensure_chat_member(session, user, chat_id) + validate_filename(filename) + if content_type not in {"image/png", "image/jpeg"}: + raise unsupported_media_type("Принимаются только image/png и image/jpeg") + # Не держим соединение PostgreSQL во время чтения медленного клиента. + await session.rollback() + settings = storage.settings + with TemporaryDirectory(prefix="messenger-upload-") as directory: + original = await read_original( + chunks, + Path(directory) / "original", + settings.max_upload_bytes, + settings.chunk_bytes, + ) + await validate_image(original.path, content_type, settings) + result = await publish_original( + session, + storage, + user, + chat_id, + original=original, + filename=filename, + content_type=content_type, + ) + return result + + +async def get_attachment( + session: AsyncSession, user: User, attachment_id: UUID +) -> Attachment: + record = await repository.get_attachment(session, attachment_id) + if record is None: + raise not_found() + await ensure_chat_member(session, user, record.chat_id) + return Attachment.model_validate(record) + + +async def download_attachment( + session: AsyncSession, + storage: MinioStorage | None, + user: User, + attachment_id: UUID, + *, + variant: str = "original", +) -> tuple[Attachment, AsyncIterator[bytes]]: + result = await get_attachment(session, user, attachment_id) + if variant != "original": + raise not_found() + storage = require_storage(storage) + record = await repository.get_attachment(session, attachment_id) + if record is None: + raise not_found() + key = record.object_key + await session.rollback() + info = await storage.stat_object(key) + if ( + info is None + or info.size_bytes != result.size_bytes + or info.content_type != result.content_type + ): + raise unavailable("Оригинал временно недоступен") + stream = await storage.open_object(key) + return result, stream + + +async def validate_message_attachments( + session: AsyncSession, + user: User, + chat_id: UUID, + attachment_ids: list[UUID], +) -> None: + await ensure_chat_member(session, user, chat_id) + records = await repository.get_for_chat(session, chat_id, attachment_ids) + if len(records) != len(attachment_ids): + raise not_found() + if any(record.status != "ready" for record in records): + raise conflict("Вложение ещё не готово") + + +async def reconcile_attachments( + session: AsyncSession, + storage: MinioStorage, + *, + batch_size: int = 20, + dry_run: bool = True, +) -> CleanupResult: + if not 1 <= batch_size <= 100: + raise invalid_request("batch_size должен быть от 1 до 100") + intents = await repository.stale_uploads( + session, storage.settings.cleanup_grace_seconds, batch_size + ) + candidates = deleted = 0 + for intent in intents: + if await repository.get_by_key(session, intent.object_key) is None: + candidates += 1 + if not dry_run: + await storage.delete_object(intent.object_key) + deleted += 1 + if not dry_run: + await repository.finish_upload(session, intent.id) + if dry_run: + await session.rollback() + else: + await session.commit() + return CleanupResult(len(intents), candidates, deleted) diff --git a/app/domain/auth.py b/app/domain/auth.py new file mode 100644 index 00000000..63f4f3cc --- /dev/null +++ b/app/domain/auth.py @@ -0,0 +1,84 @@ +import re +from datetime import UTC, datetime, timedelta + +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import security +from app.domain.users import to_user_schema +from app.errors.exceptions import unauthorized_basic, unauthorized_bearer +from app.schemas.session import AuthSession +from app.schemas.user import ( + PASSWORD_MAX_LENGTH, + PASSWORD_MIN_LENGTH, + USERNAME_PATTERN, + User, +) +from app.storage import sessions as sessions_storage +from app.storage import users as users_storage + +USERNAME_RE = re.compile(USERNAME_PATTERN) + + +async def create_session( + session: AsyncSession, + redis: Redis, + username: str, + password: str, + ttl_seconds: int, +) -> AuthSession: + """Проверяет учётные данные и выдаёт новый Bearer-токен.""" + if ( + USERNAME_RE.fullmatch(username) is None + or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH + or not password.strip() + ): + raise unauthorized_basic() + + user = await users_storage.get_user_by_username(session, username) + await session.commit() + + if user is None: + await security.dummy_verify_password(password) + raise unauthorized_basic() + + if not await security.verify_password(password, user.password_hash): + raise unauthorized_basic() + + token = security.generate_token() + expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds) + await sessions_storage.create_session( + redis, + token_hash=security.hash_token(token), + user_id=user.id, + expires_at=expires_at, + ) + auth_session = AuthSession( + token=token, expires_at=expires_at, user=to_user_schema(user) + ) + return auth_session + + +async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User: + """Возвращает владельца действующей сессии или бросает 401.""" + user_id = await sessions_storage.get_session_user_id( + redis, security.hash_token(token) + ) + if user_id is None: + raise unauthorized_bearer() + + user = await users_storage.get_user_by_id(session, user_id) + if user is None: + raise unauthorized_bearer() + + current_user = to_user_schema(user) + return current_user + + +async def revoke_session(redis: Redis, token: str) -> None: + """Отзывает текущую сессию; повторный отзыв даёт 401.""" + is_deleted = await sessions_storage.delete_session( + redis, security.hash_token(token) + ) + if not is_deleted: + raise unauthorized_bearer() diff --git a/app/domain/chats.py b/app/domain/chats.py new file mode 100644 index 00000000..c810a982 --- /dev/null +++ b/app/domain/chats.py @@ -0,0 +1,104 @@ +from uuid import UUID + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.pagination import ( + KIND_CHATS, + build_next_cursor, + decode_cursor, + split_page, +) +from app.errors.exceptions import invalid_request, not_found +from app.models.chat import Chat as ChatModel +from app.schemas.chat import Chat, ChatPage, CreateChat +from app.schemas.user import User +from app.storage import chats as chats_storage +from app.storage import users as users_storage +from app.storage.types import Position + +UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя" + + +def to_chat_schema(chat: ChatModel) -> Chat: + """Преобразует ORM-модель чата с загруженными участниками в схему.""" + chat_schema = Chat( + id=chat.id, + title=chat.title, + member_ids=[member.user_id for member in chat.members], + created_at=chat.created_at, + ) + return chat_schema + + +def validate_new_chat(creator: User, new_chat: CreateChat) -> None: + """Проверяет состав участников без обращения к базе.""" + member_ids = new_chat.member_ids + + if creator.id not in member_ids: + raise invalid_request("Создатель чата должен входить в member_ids") + + if len(set(member_ids)) != len(member_ids): + raise invalid_request("member_ids содержит дубликаты") + + +async def create_chat( + session: AsyncSession, creator: User, new_chat: CreateChat +) -> Chat: + """Создаёт чат с участниками атомарно; неверный состав даёт 400.""" + validate_new_chat(creator, new_chat) + + existing_ids = await users_storage.get_existing_user_ids( + session, new_chat.member_ids + ) + if existing_ids != set(new_chat.member_ids): + raise invalid_request(UNKNOWN_MEMBER_MESSAGE) + + try: + chat = await chats_storage.create_chat( + session, title=new_chat.title, member_ids=new_chat.member_ids + ) + await session.commit() + except IntegrityError: + await session.rollback() + raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None + + created_chat = to_chat_schema(chat) + return created_chat + + +async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat: + """Возвращает чат участнику; чужой и несуществующий чат дают 404.""" + chat = await chats_storage.get_chat_for_member(session, chat_id, user.id) + if chat is None: + raise not_found() + + found_chat = to_chat_schema(chat) + return found_chat + + +async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None: + """Бросает 404, если пользователь не состоит в чате или чата нет.""" + if not await chats_storage.is_chat_member(session, chat_id, user.id): + raise not_found() + + +async def list_chats( + session: AsyncSession, + user: User, + cursor: str | None, + limit: int, +) -> ChatPage: + """Возвращает страницу чатов пользователя по (created_at, id).""" + after: Position | None = None + if cursor is not None: + after = decode_cursor(cursor, KIND_CHATS, user.id) + + rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1) + page_rows, has_next = split_page(rows, limit) + items = [to_chat_schema(chat) for chat in page_rows] + page = ChatPage( + items=items, + next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next), + ) + return page diff --git a/app/domain/health.py b/app/domain/health.py new file mode 100644 index 00000000..aabd4d03 --- /dev/null +++ b/app/domain/health.py @@ -0,0 +1,33 @@ +"""Условия готовности реплики и ограниченная проверка зависимостей.""" + +import asyncio + +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.lifecycle import Lifecycle +from app.errors.exceptions import unavailable +from app.errors.handlers import operation_timeout +from app.schemas.base import Health +from app.storage.health import check_database, check_redis +from app.storage.s3 import MinioStorage + + +def liveness() -> Health: + return Health(status='ok') + + +async def readiness( + engine: AsyncEngine, redis: Redis, storage: MinioStorage | None, + lifecycle: Lifecycle, timeout_seconds: float, +) -> Health: + if lifecycle.draining: + raise unavailable() + checks = [check_database(engine), check_redis(redis)] + if storage is not None: + checks.append(storage.check_available()) + async with operation_timeout(timeout_seconds, unavailable()): + dependencies = await asyncio.gather(*checks, return_exceptions=True) + if not all(result is True for result in dependencies): + raise unavailable() + return Health(status='ok') diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py new file mode 100644 index 00000000..b30318ec --- /dev/null +++ b/app/domain/idempotency.py @@ -0,0 +1,72 @@ +"""Атомарная отправка сообщения с защитой от повторов.""" + +import hashlib +import json +import re +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.chats import ensure_chat_member +from app.domain.attachments import validate_message_attachments +from app.errors.exceptions import conflict, invalid_request, unavailable +from app.schemas.message import CreateMessage, Message +from app.schemas.user import User +from app.storage import idempotency as key_storage +from app.storage import messages as message_storage +from app.storage import attachments as attachment_storage + +IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def validate_idempotency_key(key: str | None) -> str: + """Проверяет обязательный ASCII-ключ запроса.""" + if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None: + raise invalid_request("Требуется корректный Idempotency-Key") + return key + + +def request_fingerprint(new_message: CreateMessage) -> str: + """Хеширует нормализованное содержимое запроса.""" + body = new_message.model_dump(mode="json") + # Сохраняет отпечатки текстовых запросов lab3 и равенство absent / []. + if not body["attachment_ids"]: + body.pop("attachment_ids") + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(canonical.encode()).hexdigest() + + +async def send_message_idempotent( + session: AsyncSession, + sender: User, + chat_id: UUID, + new_message: CreateMessage, + idempotency_key: str, + max_messages: int, + ttl_seconds: int, +) -> Message: + """Создаёт сообщение или возвращает результат прежнего запроса.""" + await ensure_chat_member(session, sender, chat_id) + scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key) + fingerprint = request_fingerprint(new_message) + + await key_storage.lock_key(session, scope) + previous = await key_storage.get_active_key(session, scope, ttl_seconds) + if previous is not None: + if previous.request_hash != fingerprint: + raise conflict("Idempotency-Key уже использован для другого сообщения") + message = await message_storage.get_message(session, previous.message_id) + else: + await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids) + if not await message_storage.reserve_message_slot(session, max_messages): + raise unavailable("Достигнут лимит количества сообщений") + message = await message_storage.create_message( + session, chat_id=chat_id, sender_id=sender.id, text=new_message.text + ) + await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids) + await key_storage.save_result(session, scope, fingerprint, message.id) + + links = await attachment_storage.message_attachment_ids(session, [message.id]) + result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]}) + await session.commit() + return result diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py new file mode 100644 index 00000000..6dffbc03 --- /dev/null +++ b/app/domain/image_pipeline.py @@ -0,0 +1,34 @@ +"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP. + +Pipeline запускается внутри ограниченного процесса worker. Исходные bytes +не перекодируются; варианты удаляют метаданные по документированной политике. +Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md. +""" + +from pathlib import Path + + +def create_thumbnail(source: Path, destination: Path) -> None: + """Создать JPEG, вписанный в 256×256 без увеличения. + + Применить EXIF orientation, alpha положить на белый; округлять размеры + вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160. + """ + raise NotImplementedError("Lab5: thumbnail ещё не реализован") + + +def create_crop(source: Path, destination: Path) -> None: + """Создать центрированный квадрат JPEG 128×128; увеличение допустимо. + + Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail. + """ + raise NotImplementedError("Lab5: crop ещё не реализован") + + +def create_optimized(source: Path, destination: Path) -> None: + """Создать WebP, вписанный в 1280×1280 без увеличения. + + Удалить метаданные и применить описанную политику качества. Уменьшение + размера любого маленького исходника не гарантируется контрактом. + """ + raise NotImplementedError("Lab5: optimized ещё не реализован") diff --git a/app/domain/image_validation.py b/app/domain/image_validation.py new file mode 100644 index 00000000..c004d295 --- /dev/null +++ b/app/domain/image_validation.py @@ -0,0 +1,23 @@ +"""Синхронная проверка изображения; вызывается только из ограниченного процесса.""" + +from pathlib import Path +import warnings + +from PIL import Image + +from app.errors.exceptions import payload_too_large, unsupported_media_type + + +def decode_image(path: Path, content_type: str, max_pixels: int) -> None: + Image.MAX_IMAGE_PIXELS = max_pixels + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(path, formats=["PNG", "JPEG"]) as image: + if image.width * image.height > max_pixels: + raise payload_too_large("Превышен лимит пикселей изображения") + image_format = image.format + if image_format is None or Image.MIME.get(image_format) != content_type: + raise unsupported_media_type("Содержимое не соответствует Content-Type") + image.verify() + # Для JPEG одной verify() недостаточно: декодировать весь оригинал. + with Image.open(path, formats=["PNG", "JPEG"]) as image: + image.load() diff --git a/app/domain/images.py b/app/domain/images.py new file mode 100644 index 00000000..4436656f --- /dev/null +++ b/app/domain/images.py @@ -0,0 +1,48 @@ +"""Проверка имени и запуск ограниченного декодера.""" + +import asyncio +from pathlib import Path +import sys +import unicodedata + +from anyio import CancelScope + +from app.core.s3 import S3Settings +from app.errors.exceptions import ( + image_validation_error, + invalid_request, + payload_too_large, +) +from app.errors.handlers import operation_timeout + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or any( + char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"} + for char in filename + ): + raise invalid_request("Недопустимое имя файла") + + +async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "app.core.image_worker", + str(path), + content_type, + str(settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + error = payload_too_large("Превышен бюджет проверки изображения") + try: + async with operation_timeout(settings.image_timeout_seconds, error): + code = await process.wait() + finally: + if process.returncode is None: + process.kill() + with CancelScope(shield=True): + await process.wait() + if code != 0: + raise image_validation_error(code) diff --git a/app/domain/messages.py b/app/domain/messages.py new file mode 100644 index 00000000..f35cbfb3 --- /dev/null +++ b/app/domain/messages.py @@ -0,0 +1,43 @@ +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.domain.chats import ensure_chat_member +from app.domain.pagination import ( + KIND_MESSAGES, + build_next_cursor, + decode_cursor, + split_page, +) +from app.schemas.message import Message, MessagePage +from app.schemas.user import User +from app.storage import messages as messages_storage +from app.storage import attachments as attachments_storage +from app.storage.types import Position + + +async def list_messages( + session: AsyncSession, + user: User, + chat_id: UUID, + cursor: str | None, + limit: int, +) -> MessagePage: + """Возвращает страницу истории чата по (created_at, id).""" + after: Position | None = None + if cursor is not None: + after = decode_cursor(cursor, KIND_MESSAGES, chat_id) + + await ensure_chat_member(session, user, chat_id) + + rows = await messages_storage.list_messages(session, chat_id, after, limit + 1) + page_rows, has_next = split_page(rows, limit) + links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows]) + items = [Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]}, + ) for message in page_rows] + page = MessagePage( + items=items, + next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next), + ) + return page diff --git a/app/domain/pagination.py b/app/domain/pagination.py new file mode 100644 index 00000000..9894437c --- /dev/null +++ b/app/domain/pagination.py @@ -0,0 +1,83 @@ +import base64 +import json +from collections.abc import Sequence +from datetime import datetime +from typing import Protocol, TypeVar +from uuid import UUID + +from app.errors.exceptions import invalid_request +from app.storage.types import Position + +KIND_CHATS = "chats" +KIND_MESSAGES = "messages" + +T = TypeVar("T") + + +class Positioned(Protocol): + """Объект с позицией в порядке (created_at, id).""" + + @property + def id(self) -> UUID: ... + + @property + def created_at(self) -> datetime: ... + + +def encode_cursor( + kind: str, + resource_id: UUID, + created_at: datetime, + item_id: UUID, +) -> str: + """Кодирует позицию и ресурс, к которому она привязана, в непрозрачный cursor.""" + payload = { + "kind": kind, + "resource_id": str(resource_id), + "created_at": created_at.isoformat(), + "item_id": str(item_id), + } + raw = json.dumps(payload).encode() + return base64.urlsafe_b64encode(raw).decode() + + +def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position: + """Возвращает позицию из cursor; битый или чужой cursor даёт 400.""" + try: + payload = json.loads(base64.urlsafe_b64decode(cursor)) + cursor_kind = payload["kind"] + cursor_resource_id = UUID(payload["resource_id"]) + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + except (ValueError, KeyError, TypeError, AttributeError): + raise invalid_request("Некорректный cursor") from None + + if ( + cursor_kind != kind + or cursor_resource_id != resource_id + or created_at.tzinfo is None + ): + raise invalid_request("Некорректный cursor") + + return created_at, item_id + + +def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]: + """Отделяет страницу от лишней строки limit + 1, по которой видно продолжение.""" + has_next = len(rows) > limit + items = rows[:limit] + return items, has_next + + +def build_next_cursor( + kind: str, + resource_id: UUID, + items: Sequence[Positioned], + has_next: bool, +) -> str | None: + """Возвращает cursor на последний элемент страницы или None, если она последняя.""" + if not has_next: + return None + + last_item = items[-1] + return encode_cursor(kind, resource_id, last_item.created_at, last_item.id) diff --git a/app/domain/users.py b/app/domain/users.py new file mode 100644 index 00000000..461bf0d9 --- /dev/null +++ b/app/domain/users.py @@ -0,0 +1,41 @@ +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import security +from app.errors.exceptions import conflict +from app.models.user import User as UserModel +from app.schemas.user import CreateUser, User +from app.storage import users as users_storage + +USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято" + + +def to_user_schema(user: UserModel) -> User: + """Преобразует ORM-модель в публичную схему без password_hash.""" + user_schema = User.model_validate(user) + return user_schema + + +async def register_user(session: AsyncSession, new_user: CreateUser) -> User: + """Регистрирует пользователя; занятый username даёт 409.""" + existing_user = await users_storage.get_user_by_username(session, new_user.username) + await session.commit() + if existing_user is not None: + raise conflict(USERNAME_TAKEN_MESSAGE) + + password_hash = await security.hash_password(new_user.password) + + try: + user = await users_storage.create_user( + session, + username=new_user.username, + display_name=new_user.display_name, + password_hash=password_hash, + ) + await session.commit() + except IntegrityError: + await session.rollback() + raise conflict(USERNAME_TAKEN_MESSAGE) from None + + created_user = to_user_schema(user) + return created_user diff --git a/app/endpoints/__init__.py b/app/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py new file mode 100644 index 00000000..94255c0c --- /dev/null +++ b/app/endpoints/attachments.py @@ -0,0 +1,52 @@ +"""HTTP-параметры и упаковка ответов; сценарии находятся в domain.""" + +from typing import Annotated +from urllib.parse import quote +from uuid import UUID + +from fastapi import APIRouter, Query, Request +from fastapi.responses import StreamingResponse +from starlette.background import BackgroundTask + +from app.domain import attachments +from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity +from app.schemas.attachment import Attachment + +router = APIRouter(prefix='/api/v1', tags=['attachments']) + + +@router.post('/chats/{chat_id}/attachments', status_code=201) +async def upload( + request: Request, chat_id: UUID, user: CurrentUser, session: DbSession, + storage: S3Storage, capacity: UploadCapacity, + filename: Annotated[str, Query(min_length=1, max_length=255)], +) -> Attachment: + return await attachments.receive_attachment( + session, storage, user, chat_id, capacity=capacity, filename=filename, + content_type=request.headers.get('content-type', '').lower(), + declared_length=request.headers.get('content-length'), chunks=request.stream(), + ) + + +@router.get('/attachments/{attachment_id}') +async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment: + return await attachments.get_attachment(session, user, attachment_id) + + +@router.get('/attachments/{attachment_id}/content') +async def content( + attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage, + variant: Annotated[str, Query()] = 'original', +) -> StreamingResponse: + record, stream = await attachments.download_attachment( + session, storage, user, attachment_id, variant=variant, + ) + return StreamingResponse( + stream, media_type=record.content_type, + headers={ + 'Content-Length': str(record.size_bytes), + 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''), + 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store', + }, + background=BackgroundTask(stream.aclose), + ) diff --git a/app/endpoints/auth.py b/app/endpoints/auth.py new file mode 100644 index 00000000..69f4535f --- /dev/null +++ b/app/endpoints/auth.py @@ -0,0 +1,32 @@ +from fastapi import APIRouter, Response + +from app.domain.auth import create_session, revoke_session +from app.endpoints.dependencies import ( + AppSettings, + BasicCredentials, + BearerToken, + DbSession, + RedisClient, +) +from app.schemas.session import AuthSession + +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) + + +@router.post("/sessions", status_code=201) +async def login( + credentials: BasicCredentials, + session: DbSession, + redis: RedisClient, + settings: AppSettings, +) -> AuthSession: + username, password = credentials + return await create_session( + session, redis, username, password, settings.session_ttl_seconds + ) + + +@router.delete("/sessions/current", status_code=204) +async def logout(token: BearerToken, redis: RedisClient) -> Response: + await revoke_session(redis, token) + return Response(status_code=204) diff --git a/app/endpoints/chats.py b/app/endpoints/chats.py new file mode 100644 index 00000000..91f2b211 --- /dev/null +++ b/app/endpoints/chats.py @@ -0,0 +1,32 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Query + +from app.domain.chats import create_chat, get_chat, list_chats +from app.endpoints.dependencies import CurrentUser, DbSession +from app.schemas.chat import Chat, ChatPage, CreateChat + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", status_code=201) +async def create_new_chat( + new_chat: CreateChat, creator: CurrentUser, session: DbSession +) -> Chat: + return await create_chat(session, creator, new_chat) + + +@router.get("") +async def list_user_chats( + user: CurrentUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> ChatPage: + return await list_chats(session, user, cursor, limit) + + +@router.get("/{chat_id}") +async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat: + return await get_chat(session, user, chat_id) diff --git a/app/endpoints/dependencies.py b/app/endpoints/dependencies.py new file mode 100644 index 00000000..39ffdf25 --- /dev/null +++ b/app/endpoints/dependencies.py @@ -0,0 +1,112 @@ +import base64 +import binascii +from collections.abc import AsyncIterator +from typing import Annotated + +from fastapi import Depends, Request, Security +from fastapi.openapi.models import HTTPBase, HTTPBearer +from fastapi.security.base import SecurityBase +from redis.asyncio import Redis +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import Settings +from app.core.request_capacity import RequestCapacity +from app.domain.auth import authenticate_token +from app.errors.exceptions import unauthorized_basic, unauthorized_bearer +from app.schemas.user import User +from app.storage.s3 import MinioStorage + + +async def get_settings(request: Request) -> Settings: + settings: Settings = request.app.state.settings + return settings + + +async def get_db(request: Request) -> AsyncIterator[AsyncSession]: + """Одна AsyncSession на HTTP-запрос; незакоммиченная транзакция откатывается.""" + session_factory = request.app.state.session_factory + async with session_factory() as session: + yield session + + +async def get_redis(request: Request) -> Redis: + redis: Redis = request.app.state.redis + return redis + + +def get_s3(request: Request) -> MinioStorage | None: + return request.app.state.s3 + + +def get_upload_capacity(request: Request) -> RequestCapacity: + return request.app.state.upload_capacity + + +class AuthorizationHeader(SecurityBase): + """Отдаёт сырой заголовок Authorization и описывает схему для OpenAPI.""" + + def __init__(self, model: HTTPBase, scheme_name: str) -> None: + self.model = model + self.scheme_name = scheme_name + + async def __call__(self, request: Request) -> str | None: + return request.headers.get("Authorization") + + +basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth") +bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth") + + +def _split_authorization(authorization: str | None) -> tuple[str, str] | None: + if authorization is None: + return None + scheme, _, value = authorization.partition(" ") + if not value.strip(): + return None + return scheme.lower(), value.strip() + + +async def get_bearer_token( + authorization: Annotated[str | None, Security(bearer_auth)], +) -> str: + parts = _split_authorization(authorization) + if parts is None or parts[0] != "bearer": + raise unauthorized_bearer() + return parts[1] + + +async def get_basic_credentials( + authorization: Annotated[str | None, Security(basic_auth)], +) -> tuple[str, str]: + """(username, password) из `Authorization: Basic`; пароль может содержать ':'.""" + parts = _split_authorization(authorization) + if parts is None or parts[0] != "basic": + raise unauthorized_basic() + + try: + decoded = base64.b64decode(parts[1], validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + raise unauthorized_basic() from None + + username, separator, password = decoded.partition(":") + if not separator or not username: + raise unauthorized_basic() + return username, password + + +DbSession = Annotated[AsyncSession, Depends(get_db)] +RedisClient = Annotated[Redis, Depends(get_redis)] +AppSettings = Annotated[Settings, Depends(get_settings)] +S3Storage = Annotated[MinioStorage | None, Depends(get_s3)] +UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)] +BearerToken = Annotated[str, Depends(get_bearer_token)] +BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)] + + +async def get_current_user( + token: BearerToken, session: DbSession, redis: RedisClient +) -> User: + return await authenticate_token(session, redis, token) + + +CurrentUser = Annotated[User, Depends(get_current_user)] diff --git a/app/endpoints/health.py b/app/endpoints/health.py new file mode 100644 index 00000000..9c28c7d4 --- /dev/null +++ b/app/endpoints/health.py @@ -0,0 +1,20 @@ +from fastapi import APIRouter, Request + +from app.domain import health +from app.schemas.base import Health + +router = APIRouter(prefix='/health', tags=['health']) + + +@router.get('/live') +async def liveness() -> Health: + return health.liveness() + + +@router.get('/ready') +async def readiness(request: Request) -> Health: + state = request.app.state + return await health.readiness( + state.engine, state.redis, state.s3, state.lifecycle, + state.settings.readiness_timeout_seconds, + ) diff --git a/app/endpoints/messages.py b/app/endpoints/messages.py new file mode 100644 index 00000000..50dc1a18 --- /dev/null +++ b/app/endpoints/messages.py @@ -0,0 +1,38 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Header, Query + +from app.domain.idempotency import send_message_idempotent, validate_idempotency_key +from app.domain.messages import list_messages +from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession +from app.schemas.message import CreateMessage, Message, MessagePage + +router = APIRouter(prefix="/api/v1/chats", tags=["messages"]) + + +@router.post("/{chat_id}/messages", status_code=201) +async def send_new_message( + chat_id: UUID, + new_message: CreateMessage, + sender: CurrentUser, + session: DbSession, + settings: AppSettings, + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, +) -> Message: + return await send_message_idempotent( + session, sender, chat_id, new_message, + validate_idempotency_key(idempotency_key), + settings.max_messages, settings.idempotency_ttl_seconds, + ) + + +@router.get("/{chat_id}/messages") +async def get_history_messages( + chat_id: UUID, + user: CurrentUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> MessagePage: + return await list_messages(session, user, chat_id, cursor, limit) diff --git a/app/endpoints/users.py b/app/endpoints/users.py new file mode 100644 index 00000000..c502d010 --- /dev/null +++ b/app/endpoints/users.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter + +from app.domain.users import register_user +from app.endpoints.dependencies import CurrentUser, DbSession +from app.schemas.user import CreateUser, User + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", status_code=201) +async def create_user(new_user: CreateUser, session: DbSession) -> User: + return await register_user(session, new_user) + + +@router.get("/me") +async def get_me(user: CurrentUser) -> User: + return user diff --git a/app/errors/__init__.py b/app/errors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py new file mode 100644 index 00000000..b434529e --- /dev/null +++ b/app/errors/exceptions.py @@ -0,0 +1,109 @@ +from collections.abc import Mapping + +from PIL import Image, UnidentifiedImageError + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__( + self, + status_code: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers + + +def invalid_request(message: str = "Неверный запрос") -> ApiError: + """Создаёт ошибку некорректного или недопустимого запроса.""" + return ApiError(400, "invalid_request", message) + + +def unauthorized( + message: str = "Нет действительной аутентификации", + www_authenticate: str | None = None, +) -> ApiError: + """Создаёт ошибку отсутствующей или недействительной аутентификации.""" + headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None + return ApiError(401, "unauthorized", message, headers) + + +def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError: + """401 для POST /auth/sessions (одинаковый для неверного username и пароля).""" + return unauthorized(message, BASIC_CHALLENGE) + + +def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError: + """401 для бизнес-endpoint: нет, истёк, отозван или неизвестен токен.""" + return unauthorized(message, BEARER_CHALLENGE) + + +def not_found(message: str = "Объект не найден") -> ApiError: + """Создаёт ошибку, указывающую на отсутствие запрошенного ресурса.""" + return ApiError(404, "not_found", message) + + +def conflict(message: str = "Конфликт") -> ApiError: + """Создаёт ошибку конфликта с текущим состоянием ресурса.""" + return ApiError(409, "conflict", message) + + +def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError: + """Создаёт ошибку, указывающую на превышение допустимого размера запроса.""" + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type( + message: str = "Требуется Content-Type: application/json", +) -> ApiError: + """Создаёт ошибку неподдерживаемого формата или типа содержимого запроса.""" + return ApiError(415, "unsupported_media_type", message) + + +def rate_limited(retry_after: int = 1) -> ApiError: + """Создаёт ошибку превышения допустимой частоты запросов.""" + return ApiError( + 429, + "rate_limited", + "Слишком много запросов", + {"Retry-After": str(max(1, retry_after))}, + ) + + +def unavailable(message: str = "Сервис временно недоступен") -> ApiError: + """Создаёт ошибку временной недоступности сервиса.""" + return ApiError(503, "unavailable", message) + + +def internal_error() -> ApiError: + """Создаёт ошибку внутренней ошибки сервера.""" + return ApiError(500, "internal_error", "Внутренняя ошибка сервера") + + +IMAGE_WORKER_ERRORS = ( + ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning, + MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError, +) + + +def image_worker_error_code(error: Exception) -> int: + code = 2 + resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError)) + declared_limit = isinstance(error, ApiError) and error.status_code == 413 + if resource_error or declared_limit: + code = 3 + return code + + +def image_validation_error(code: int) -> ApiError: + error = unsupported_media_type("Повреждённое изображение или несовпадение MIME") + if code == 3 or code < 0: + error = payload_too_large("Превышен лимит пикселей или ресурсов изображения") + return error diff --git a/app/errors/handlers.py b/app/errors/handlers.py new file mode 100644 index 00000000..9dd48190 --- /dev/null +++ b/app/errors/handlers.py @@ -0,0 +1,225 @@ +import asyncio +import logging +from contextlib import asynccontextmanager +from collections.abc import Mapping +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from minio.error import MinioException +from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import TimeoutError as RedisTimeoutError +from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect +from urllib3.exceptions import HTTPError as StorageHTTPError + +from app.core.instance import INSTANCE_ID_HEADER +from app.core.request_id import REQUEST_ID_HEADER +from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable + +logger = logging.getLogger(__name__) + +TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57") +TRANSIENT_SQLSTATES = frozenset({"55P03"}) +RECOVERABLE_STORAGE_ERRORS = ( + ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError, +) + + +@asynccontextmanager +async def operation_timeout(seconds: float, error: ApiError): + """Единое преобразование таймаутов; владелец операции задаёт ожидаемую ошибку.""" + try: + async with asyncio.timeout(seconds): + yield + except TimeoutError: + raise error from None + + +async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse: + return await api_error_handler(request, invalid_request("Загрузка прервана клиентом")) + + +def log_recoverable_error(operation: str, error: Exception) -> None: + """Согласованный безопасный лог восстановления без содержимого SDK-ошибки.""" + logger.warning('%s deferred: %s', operation, type(error).__name__) + + +def _get_request_id(request: Request) -> str: + request_id = getattr(request.state, "request_id", None) + + if request_id is None: + request_id = str(uuid4()) + request.state.request_id = request_id + + return request_id + + +def _build_response( + request: Request, + status_code: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, +) -> JSONResponse: + request_id = _get_request_id(request) + + response_headers = { + **(headers or {}), + REQUEST_ID_HEADER: request_id, + } + instance_id = getattr(request.app.state, "instance_id", None) + if instance_id is not None: + response_headers[INSTANCE_ID_HEADER] = instance_id + + body = { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } + + return JSONResponse( + status_code=status_code, + content=body, + headers=response_headers, + ) + + +async def api_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, ApiError) + + return _build_response( + request, + exc.status_code, + exc.code, + exc.message, + exc.headers, + ) + + +async def routing_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, StarletteHTTPException) + + error = invalid_request() + if exc.status_code == 404: + error = not_found() + return _build_response( + request, + exc.status_code, + error.code, + error.message, + exc.headers, + ) + + +async def validation_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + assert isinstance(exc, RequestValidationError) + + fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()}) + message = "Неверный запрос" + + if fields: + message = f"{message}: {', '.join(fields)}" + + return _build_response( + request, + 400, + "invalid_request", + message, + ) + + +async def dependency_unavailable_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + """PostgreSQL/Redis/S3 недоступны либо исчерпан пул: 503 без внутренних деталей.""" + request_id = _get_request_id(request) + logger.error( + "Dependency unavailable request_id=%s: %s", + request_id, + type(exc).__name__, + ) + + error = unavailable() + return _build_response( + request, + error.status_code, + error.code, + error.message, + {"Retry-After": "1"}, + ) + + +def _is_transient_database_error(exc: DBAPIError) -> bool: + sqlstate = getattr(exc.orig, "sqlstate", None) or "" + return ( + sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES) + or sqlstate in TRANSIENT_SQLSTATES + ) + + +async def database_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + """Временные ошибки PostgreSQL (таймаут запроса, блокировки, нехватка ресурсов) — 503.""" + assert isinstance(exc, DBAPIError) + + handler = unhandled_error_handler + if _is_transient_database_error(exc): + handler = dependency_unavailable_handler + return await handler(request, exc) + + +async def unhandled_error_handler( + request: Request, + exc: Exception, +) -> JSONResponse: + request_id = _get_request_id(request) + + logger.error( + "Unhandled error request_id=%s", + request_id, + exc_info=exc, + ) + + error = internal_error() + + return _build_response( + request, + error.status_code, + error.code, + error.message, + ) + + +def register_exception_handlers(app: FastAPI) -> None: + app.add_exception_handler(ApiError, api_error_handler) + app.add_exception_handler(ClientDisconnect, client_disconnect_handler) + app.add_exception_handler(MinioException, dependency_unavailable_handler) + app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler) + app.add_exception_handler(StarletteHTTPException, routing_error_handler) + app.add_exception_handler(RequestValidationError, validation_error_handler) + app.add_exception_handler(OperationalError, dependency_unavailable_handler) + app.add_exception_handler(InterfaceError, dependency_unavailable_handler) + app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler) + app.add_exception_handler(DBAPIError, database_error_handler) + app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler) + app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler) + app.add_exception_handler(OSError, dependency_unavailable_handler) + app.add_exception_handler(Exception, unhandled_error_handler) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..7bf806c9 --- /dev/null +++ b/app/main.py @@ -0,0 +1,92 @@ +import logging +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +import uvicorn +from fastapi import Depends, FastAPI + +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.instance import InstanceIdMiddleware, resolve_instance_id +from app.core.lifecycle import GracefulServer, Lifecycle +from app.core.redis import create_redis_client +from app.core.request_capacity import acquire_request_slot, create_request_capacity +from app.core.request_id import RequestIdMiddleware +from app.endpoints.auth import router as auth_router +from app.endpoints.attachments import router as attachments_router +from app.endpoints.chats import router as chats_router +from app.endpoints.health import router as health_router +from app.endpoints.messages import router as messages_router +from app.endpoints.users import router as users_router +from app.errors.handlers import register_exception_handlers +from app.storage.s3 import create_s3_storage + + +def configure_logging(settings: Settings) -> None: + logging.basicConfig( + level=settings.log_level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + + +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or Settings() # type: ignore + configure_logging(settings) + + engine = create_db_engine(settings) + redis = create_redis_client(settings) + s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None + + @asynccontextmanager + async def lifespan(_: FastAPI) -> AsyncGenerator[None]: + yield + if s3 is not None: + await s3.aclose() + await redis.aclose() + await engine.dispose() + + application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan) + + application.state.settings = settings + application.state.engine = engine + application.state.session_factory = create_session_factory(engine) + application.state.redis = redis + application.state.s3 = s3 + application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads) + application.state.request_capacity = create_request_capacity(settings.max_in_flight) + application.state.instance_id = resolve_instance_id(settings.instance_id) + application.state.lifecycle = Lifecycle() + application.add_middleware(RequestIdMiddleware) + application.add_middleware( + InstanceIdMiddleware, instance_id=application.state.instance_id + ) + register_exception_handlers(application) + + limited = [Depends(acquire_request_slot)] + application.include_router(health_router) + application.include_router(users_router, dependencies=limited) + application.include_router(auth_router, dependencies=limited) + application.include_router(chats_router, dependencies=limited) + application.include_router(messages_router, dependencies=limited) + application.include_router(attachments_router, dependencies=limited) + + return application + + +def main() -> None: + settings = Settings() # type: ignore + application = create_app(settings) + config = uvicorn.Config( + application, + host=settings.app_host, + port=settings.app_port, + log_level=settings.log_level.lower(), + timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds, + ) + GracefulServer( + config, application.state.lifecycle, settings.shutdown_drain_seconds + ).run() + + +if __name__ == "__main__": + main() diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..6679c3ef --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,10 @@ +from app.models.base import Base +from app.models.attachment import Attachment, MessageAttachment, UploadIntent +from app.models.chat import Chat +from app.models.chat_member import ChatMember +from app.models.counter import Counter +from app.models.idempotency_key import IdempotencyKey +from app.models.message import Message +from app.models.user import User + +__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 00000000..3b7c678d --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,57 @@ +"""Метаданные оригиналов и журнал операций до записи в S3.""" + +import uuid +from datetime import datetime + +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"), + CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", name="content_type" + ), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), + CheckConstraint("status = 'ready'", name="status"), + Index("ix_attachments_chat_id", "chat_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id")) + object_key: Mapped[str] = mapped_column(String(128), unique=True) + filename: Mapped[str] = mapped_column(String(255)) + content_type: Mapped[str] = mapped_column(String(32)) + size_bytes: Mapped[int] + sha256: Mapped[str] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(16), server_default="ready") + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + + message_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("attachments.id"), primary_key=True + ) + position: Mapped[int] + + +class UploadIntent(Base): + """Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных.""" + + __tablename__ = "upload_intents" + __table_args__ = (Index("ix_upload_intents_created_at", "created_at"),) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True) + object_key: Mapped[str] = mapped_column(String(128), unique=True) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 00000000..15a748e5 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,27 @@ +import uuid +from datetime import datetime +from typing import Any, ClassVar + +from sqlalchemy import DateTime, MetaData +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.types import TypeEngine + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + """Базовый класс ORM-моделей.""" + + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = { + uuid.UUID: PG_UUID(as_uuid=True), + datetime: DateTime(timezone=True), + } diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 00000000..ffff7ec2 --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,32 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import String, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat_member import ChatMember + from app.models.message import Message + + +class Chat(Base): + """Чат с фиксированным составом участников.""" + + __tablename__ = "chats" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + title: Mapped[str] = mapped_column(String(100)) + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + members: Mapped[list["ChatMember"]] = relationship( + back_populates="chat", + cascade="all, delete-orphan", + passive_deletes=True, + lazy="selectin", + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="chat", lazy="raise" + ) diff --git a/app/models/chat_member.py b/app/models/chat_member.py new file mode 100644 index 00000000..567ffd66 --- /dev/null +++ b/app/models/chat_member.py @@ -0,0 +1,26 @@ +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey, Index +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat import Chat + from app.models.user import User + + +class ChatMember(Base): + """Членство пользователя в чате.""" + + __tablename__ = "chat_members" + __table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),) + + chat_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + + chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise") + user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise") diff --git a/app/models/counter.py b/app/models/counter.py new file mode 100644 index 00000000..5f6706b5 --- /dev/null +++ b/app/models/counter.py @@ -0,0 +1,13 @@ +from sqlalchemy import BigInteger, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class Counter(Base): + """Именованный счётчик с атомарным обновлением.""" + + __tablename__ = "counters" + + name: Mapped[str] = mapped_column(String(64), primary_key=True) + value: Mapped[int] = mapped_column(BigInteger) diff --git a/app/models/idempotency_key.py b/app/models/idempotency_key.py new file mode 100644 index 00000000..5b1d3551 --- /dev/null +++ b/app/models/idempotency_key.py @@ -0,0 +1,24 @@ +"""Сохранённый результат отправки сообщения.""" + +import uuid +from datetime import datetime + +from sqlalchemy import ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class IdempotencyKey(Base): + """Ключ запроса и связанное сообщение.""" + + __tablename__ = "idempotency_keys" + __table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),) + + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True) + method: Mapped[str] = mapped_column(String(8), primary_key=True) + path: Mapped[str] = mapped_column(String(255), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_hash: Mapped[str] = mapped_column(String(64)) + message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id")) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job.py b/app/models/job.py new file mode 100644 index 00000000..95a1af01 --- /dev/null +++ b/app/models/job.py @@ -0,0 +1,16 @@ +"""Место для будущих ORM-моделей job, outbox и вариантов lab5. + +Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4 +до реализации миграции. Планируемые данные: + +* job: job_id, attachment_id, pipeline_version, attempt/max_attempts, + status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps; +* outbox: event_id, job_id, payload/version, состояние публикации и её lease; +* variant: attachment_id, pipeline_version, variant, приватный object_key, + content_type, size_bytes, sha256, размеры; +* replay audit: операторское действие и связь с прежней логической job. + +Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker +не публикует metadata поверх новой версии; точная схема зависит от трека 5. +Изменение ready-only constraints attachments — отдельная новая миграция. +""" diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 00000000..25dc8cf3 --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,30 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import ForeignKey, Index, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat import Chat + from app.models.user import User + + +class Message(Base): + """Сообщение в чате.""" + + __tablename__ = "messages" + __table_args__ = ( + Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id")) + text: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + + chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise") + sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise") diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 00000000..74b690e6 --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,31 @@ +import uuid +from datetime import datetime +from typing import TYPE_CHECKING + +from sqlalchemy import String, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + +if TYPE_CHECKING: + from app.models.chat_member import ChatMember + from app.models.message import Message + + +class User(Base): + """Пользователь и его учётные данные.""" + + __tablename__ = "users" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + username: Mapped[str] = mapped_column(String(32), unique=True) + display_name: Mapped[str] = mapped_column(String(100)) + password_hash: Mapped[str] = mapped_column(String(255)) + created_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + memberships: Mapped[list["ChatMember"]] = relationship( + back_populates="user", lazy="raise" + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="sender", lazy="raise" + ) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 00000000..2f038940 --- /dev/null +++ b/app/schemas/__init__.py @@ -0,0 +1,6 @@ +from app.schemas.chat import Chat +from app.schemas.message import Message +from app.schemas.session import AuthSession +from app.schemas.user import User + +__all__ = ["AuthSession", "Chat", "Message", "User"] diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py new file mode 100644 index 00000000..74574dd9 --- /dev/null +++ b/app/schemas/attachment.py @@ -0,0 +1,23 @@ +"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту.""" + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import Field + +from app.schemas.base import Base + + +class Attachment(Base): + chat_id: UUID + owner_id: UUID + filename: str = Field(min_length=1, max_length=255) + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int = Field(ge=1, le=10_485_760) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + status: Literal["ready"] = "ready" + variants: list[object] = Field(default_factory=list, max_length=0) + error_code: None = None + created_at: datetime + updated_at: datetime diff --git a/app/schemas/base.py b/app/schemas/base.py new file mode 100644 index 00000000..ca7f4de8 --- /dev/null +++ b/app/schemas/base.py @@ -0,0 +1,35 @@ +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator + + +class Base(BaseModel): + model_config = ConfigDict(extra="forbid", from_attributes=True) + + id: UUID + + +class Health(BaseModel): + model_config = ConfigDict(extra="forbid") + + status: Literal["ok"] + + +class BaseCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator("*", mode="before") + @classmethod + def reject_invalid_strings(cls, value: object) -> object: + """Отклоняет строки из одних пробелов и строки с NUL, который не хранит PostgreSQL.""" + if isinstance(value, str): + if not value.strip(): + raise ValueError("Строка не может состоять только из пробелов") + if "\x00" in value: + raise ValueError("Строка не может содержать символ NUL") + return value + + +class BasePage(BaseModel): + model_config = ConfigDict(extra="forbid") diff --git a/app/schemas/chat.py b/app/schemas/chat.py new file mode 100644 index 00000000..c0436dab --- /dev/null +++ b/app/schemas/chat.py @@ -0,0 +1,22 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import Field + +from app.schemas.base import Base, BaseCreate, BasePage + + +class Chat(Base): + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + created_at: datetime + + +class CreateChat(BaseCreate): + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + +class ChatPage(BasePage): + items: list[Chat] + next_cursor: str | None diff --git a/app/schemas/job.py b/app/schemas/job.py new file mode 100644 index 00000000..d9c71ca2 --- /dev/null +++ b/app/schemas/job.py @@ -0,0 +1,10 @@ +"""Место для будущей схемы сообщения contracts/jobs.schema.json. + +Пока классы не реализованы и сообщения не принимаются/не публикуются. +Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id, +pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339). +Допустимы traceparent и tracestate; дополнительные поля запрещены. +job_id сохраняется при redelivery; очередная управляемая попытка отражается +в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются. +Pydantic-схема должна соответствовать публичному JSON Schema, не менять его. +""" diff --git a/app/schemas/message.py b/app/schemas/message.py new file mode 100644 index 00000000..a085e03e --- /dev/null +++ b/app/schemas/message.py @@ -0,0 +1,41 @@ +from datetime import datetime +from typing import Self +from uuid import UUID + +from pydantic import Field, field_validator, model_validator + +from app.schemas.base import Base, BaseCreate, BasePage + + +class Message(Base): + chat_id: UUID + sender_id: UUID + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + created_at: datetime + + +class CreateMessage(BaseCreate): + text: str = Field(max_length=4000) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator("*", mode="before") + @classmethod + def reject_invalid_strings(cls, value: object) -> object: + if isinstance(value, str): + if "\x00" in value or (value != "" and not value.strip()): + raise ValueError("Некорректная строка сообщения") + return value + + @model_validator(mode="after") + def require_content(self) -> Self: + if not self.text and not self.attachment_ids: + raise ValueError("Нужен текст или вложение") + if len(self.attachment_ids) != len(set(self.attachment_ids)): + raise ValueError("attachment_ids содержит дубликаты") + return self + + +class MessagePage(BasePage): + items: list[Message] + next_cursor: str | None diff --git a/app/schemas/session.py b/app/schemas/session.py new file mode 100644 index 00000000..1cae0d21 --- /dev/null +++ b/app/schemas/session.py @@ -0,0 +1,17 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from app.schemas.user import User + + +class AuthSession(BaseModel): + """Ответ POST /auth/sessions (в OpenAPI — Session).""" + + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: Literal["Bearer"] = "Bearer" + expires_at: datetime + user: User diff --git a/app/schemas/user.py b/app/schemas/user.py new file mode 100644 index 00000000..16548516 --- /dev/null +++ b/app/schemas/user.py @@ -0,0 +1,25 @@ +from datetime import datetime + +from pydantic import Field + +from app.schemas.base import Base, BaseCreate + +USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$" +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +class User(Base): + """Публичное представление пользователя. Пароль/хеш сюда не попадают никогда.""" + + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + created_at: datetime + + +class CreateUser(BaseCreate): + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field( + min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False + ) diff --git a/app/storage/__init__.py b/app/storage/__init__.py new file mode 100644 index 00000000..c881665f --- /dev/null +++ b/app/storage/__init__.py @@ -0,0 +1 @@ +"""Доступ к данным PostgreSQL и Redis.""" diff --git a/app/storage/attachments.py b/app/storage/attachments.py new file mode 100644 index 00000000..3ef1765f --- /dev/null +++ b/app/storage/attachments.py @@ -0,0 +1,99 @@ +"""SQL-запросы метаданных и журнала; commit выполняется в domain.""" + +from datetime import timedelta +from uuid import UUID + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.attachment import Attachment, MessageAttachment, UploadIntent + + +async def create_attachment(session: AsyncSession, **values) -> Attachment: + record = Attachment(**values) + session.add(record) + await session.flush() + return record + + +async def get_attachment( + session: AsyncSession, attachment_id: UUID +) -> Attachment | None: + return await session.get(Attachment, attachment_id) + + +async def get_by_key(session: AsyncSession, key: str) -> Attachment | None: + return await session.scalar(select(Attachment).where(Attachment.object_key == key)) + + +async def get_for_chat( + session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID] +) -> list[Attachment]: + records = [] + if attachment_ids: + records = list( + await session.scalars( + select(Attachment).where( + Attachment.id.in_(attachment_ids), + Attachment.chat_id == chat_id, + ) + ) + ) + return records + + +async def bind_attachments( + session: AsyncSession, message_id: UUID, attachment_ids: list[UUID] +) -> None: + session.add_all( + MessageAttachment(message_id=message_id, attachment_id=id, position=position) + for position, id in enumerate(attachment_ids) + ) + await session.flush() + + +async def message_attachment_ids( + session: AsyncSession, message_ids: list[UUID] +) -> dict[UUID, list[UUID]]: + result: dict[UUID, list[UUID]] = {id: [] for id in message_ids} + if not message_ids: + return result + rows = await session.execute( + select(MessageAttachment) + .where(MessageAttachment.message_id.in_(message_ids)) + .order_by(MessageAttachment.position) + ) + for link in rows.scalars(): + result[link.message_id].append(link.attachment_id) + return result + + +async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None: + session.add(UploadIntent(id=id, object_key=key)) + await session.flush() + + +async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None: + return await session.scalar( + select(UploadIntent).where(UploadIntent.id == id).with_for_update() + ) + + +async def finish_upload(session: AsyncSession, id: UUID) -> None: + await session.execute(delete(UploadIntent).where(UploadIntent.id == id)) + + +async def stale_uploads( + session: AsyncSession, grace_seconds: int, limit: int +) -> list[UploadIntent]: + result = await session.scalars( + select(UploadIntent) + .where( + UploadIntent.created_at + < func.clock_timestamp() - timedelta(seconds=grace_seconds), + ) + .order_by(UploadIntent.created_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) + return list(result) diff --git a/app/storage/broker.py b/app/storage/broker.py new file mode 100644 index 00000000..3145d2f8 --- /dev/null +++ b/app/storage/broker.py @@ -0,0 +1,27 @@ +"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран. + +Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные +поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity, +формат delivery handle и интерфейсы уточнить после выбора брокера/версии. +Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/. +""" + +from collections.abc import Mapping + + +async def publish_job(payload: Mapping[str, object]) -> str: + """Опубликовать устойчивое сообщение и вернуть идентификатор доставки. + + Дождаться подтверждения сохранения брокером; неопределённый исход разрешает + повтор с тем же job_id. Не считать публикацию обработкой изображения. + """ + raise NotImplementedError("Lab5: публикация в брокер ещё не реализована") + + +async def acknowledge_job(delivery_id: str) -> None: + """Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля. + + Retry сначала устойчиво планируется, затем закрывается старая доставка. + Конкретный delivery handle зависит от брокера; сигнатура предварительная. + """ + raise NotImplementedError("Lab5: ack ещё не реализован") diff --git a/app/storage/chats.py b/app/storage/chats.py new file mode 100644 index 00000000..f88ee0cc --- /dev/null +++ b/app/storage/chats.py @@ -0,0 +1,71 @@ +from collections.abc import Sequence +from uuid import UUID + +from sqlalchemy import exists, select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.chat import Chat +from app.models.chat_member import ChatMember +from app.storage.types import Position + + +async def create_chat( + session: AsyncSession, + *, + title: str, + member_ids: Sequence[UUID], +) -> Chat: + """Добавляет чат вместе с участниками.""" + chat = Chat( + title=title, + members=[ChatMember(user_id=user_id) for user_id in member_ids], + ) + session.add(chat) + await session.flush() + return chat + + +async def get_chat_for_member( + session: AsyncSession, + chat_id: UUID, + user_id: UUID, +) -> Chat | None: + """Возвращает чат, если пользователь в нём состоит, иначе None.""" + stmt = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(Chat.id == chat_id, ChatMember.user_id == user_id) + ) + result = await session.execute(stmt) + chat = result.scalar_one_or_none() + return chat + + +async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool: + """Возвращает True, если пользователь состоит в чате; одна проверка по PK.""" + stmt = select( + exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id) + ) + result = await session.execute(stmt) + is_member = bool(result.scalar_one()) + return is_member + + +async def list_chats_for_user( + session: AsyncSession, + user_id: UUID, + after: Position | None, + limit: int, +) -> list[Chat]: + """Возвращает чаты пользователя по (created_at, id) строго после позиции after.""" + stmt = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(ChatMember.user_id == user_id) + ) + if after is not None: + stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after)) + stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit) + result = await session.execute(stmt) + chats = list(result.scalars().all()) + return chats diff --git a/app/storage/health.py b/app/storage/health.py new file mode 100644 index 00000000..d6913fe6 --- /dev/null +++ b/app/storage/health.py @@ -0,0 +1,15 @@ +"""Простые проверки соединения с общими хранилищами.""" + +from redis.asyncio import Redis +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + + +async def check_database(engine: AsyncEngine) -> bool: + async with engine.connect() as connection: + await connection.execute(text('SELECT 1')) + return True + + +async def check_redis(client: Redis) -> bool: + return bool(await client.ping()) diff --git a/app/storage/idempotency.py b/app/storage/idempotency.py new file mode 100644 index 00000000..5cfcf709 --- /dev/null +++ b/app/storage/idempotency.py @@ -0,0 +1,63 @@ +"""Хранение и блокировка ключей в PostgreSQL.""" + +from datetime import timedelta +import hashlib +import json +from uuid import UUID + +from sqlalchemy import BigInteger, func, literal, select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.idempotency_key import IdempotencyKey + +Scope = tuple[UUID, str, str, str] + + +async def lock_key(session: AsyncSession, scope: Scope) -> None: + """Сериализует один ключ до конца транзакции.""" + raw = json.dumps([str(scope[0]), *scope[1:]]).encode() + lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True) + await session.execute( + select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger))) + ) + + +async def get_active_key( + session: AsyncSession, scope: Scope, ttl_seconds: int +) -> IdempotencyKey | None: + """Читает результат, пока срок ключа не истёк.""" + stmt = select(IdempotencyKey).where( + tuple_( + IdempotencyKey.user_id, + IdempotencyKey.method, + IdempotencyKey.path, + IdempotencyKey.key, + ) == tuple_(*scope), + IdempotencyKey.created_at + > func.clock_timestamp() - timedelta(seconds=ttl_seconds), + ) + return await session.scalar(stmt) + + +async def save_result( + session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID +) -> None: + """Сохраняет новый результат или заменяет истёкший.""" + stmt = insert(IdempotencyKey).values( + user_id=scope[0], method=scope[1], path=scope[2], key=scope[3], + request_hash=request_hash, message_id=message_id, + ) + await session.execute( + stmt.on_conflict_do_update( + index_elements=[ + IdempotencyKey.user_id, IdempotencyKey.method, + IdempotencyKey.path, IdempotencyKey.key, + ], + set_={ + "request_hash": request_hash, + "message_id": message_id, + "created_at": func.clock_timestamp(), + }, + ) + ) diff --git a/app/storage/jobs.py b/app/storage/jobs.py new file mode 100644 index 00000000..2f35056e --- /dev/null +++ b/app/storage/jobs.py @@ -0,0 +1,37 @@ +"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет. + +Будущие ORM-операции захвата, lease, retry и результата должны защищаться +от конкурирующих попыток. Границы транзакций находятся в domain/. +Тип результата захвата уточняется после определения модели job. +""" + +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + + +async def claim_job(session: AsyncSession, job_id: UUID) -> None: + """Условно захватить допустимое задание и сохранить lease/номер попытки. + + Ready и актуальная чужая аренда не переходят обратно в processing. + Будущий результат должен различать захват, завершённый дубль и занятость. + """ + raise NotImplementedError("Lab5: захват job ещё не реализован") + + +async def complete_job(session: AsyncSession, job_id: UUID) -> None: + """Атомарно записать metadata вариантов и ready после сохранения объектов. + + Предусловия lease/attempt/pipeline_version и структура результата будут + добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage. + """ + raise NotImplementedError("Lab5: фиксация результата ещё не реализована") + + +async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None: + """Записать безопасный код сбоя без stack trace и secrets. + + Domain выбирает queued с next_attempt_at либо конечный failed согласно + классификации ошибки и лимиту попыток; storage лишь сохраняет решение. + """ + raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано") diff --git a/app/storage/messages.py b/app/storage/messages.py new file mode 100644 index 00000000..e0f75af5 --- /dev/null +++ b/app/storage/messages.py @@ -0,0 +1,62 @@ +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.counter import Counter +from app.models.message import Message +from app.storage.types import Position + +MESSAGES_COUNTER = "messages" + + +async def get_message(session: AsyncSession, message_id: UUID) -> Message | None: + return await session.get(Message, message_id) + + +async def create_message( + session: AsyncSession, + *, + chat_id: UUID, + sender_id: UUID, + text: str, +) -> Message: + """Добавляет сообщение; id и created_at назначает сервер.""" + message = Message(chat_id=chat_id, sender_id=sender_id, text=text) + session.add(message) + await session.flush() + return message + + +async def list_messages( + session: AsyncSession, + chat_id: UUID, + after: Position | None, + limit: int, +) -> list[Message]: + """Возвращает сообщения чата по (created_at, id) строго после позиции after.""" + stmt = select(Message).where(Message.chat_id == chat_id) + if after is not None: + stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after)) + stmt = stmt.order_by(Message.created_at, Message.id).limit(limit) + result = await session.execute(stmt) + messages = list(result.scalars().all()) + return messages + + +async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool: + """Атомарно увеличивает счётчик сообщений, если лимит не исчерпан.""" + stmt = ( + insert(Counter) + .values(name=MESSAGES_COUNTER, value=1) + .on_conflict_do_update( + index_elements=[Counter.name], + set_={"value": Counter.value + 1}, + where=Counter.value < max_messages, + ) + .returning(Counter.value) + ) + result = await session.execute(stmt) + is_reserved = result.scalar_one_or_none() is not None + return is_reserved diff --git a/app/storage/outbox.py b/app/storage/outbox.py new file mode 100644 index 00000000..7010f4e4 --- /dev/null +++ b/app/storage/outbox.py @@ -0,0 +1,23 @@ +"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker. + +Outbox — таблица намерений публикации в той же транзакции, что и job. +Повторная публикация после неопределённого confirm допустима; job_id стабилен. +Форма записей, захват пачки и lease publisher уточняются вместе с моделью. +""" + +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + + +async def add_job_event(session: AsyncSession, job_id: UUID) -> None: + """Сохранить намерение публикации job без commit и обращения к брокеру.""" + raise NotImplementedError("Lab5: запись outbox ещё не реализована") + + +async def mark_event_published(session: AsyncSession, event_id: UUID) -> None: + """Отметить событие опубликованным после устойчивого подтверждения брокера. + + Не помечать job завершённой: publisher confirm отличается от consumer ack. + """ + raise NotImplementedError("Lab5: отметка outbox ещё не реализована") diff --git a/app/storage/s3.py b/app/storage/s3.py new file mode 100644 index 00000000..b3355a31 --- /dev/null +++ b/app/storage/s3.py @@ -0,0 +1,180 @@ +"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции.""" + +import asyncio +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import TypeVar +import time +from urllib.parse import urlsplit + +from anyio import CancelScope +from minio import Minio +import urllib3 + +from app.core.s3 import S3Settings + +T = TypeVar("T") + + +async def run_sync( + operation: Callable[..., T], *args, discard_result=None, **kwargs +) -> T: + """При отмене дождаться потока: иначе PUT может догнать компенсацию удаления.""" + task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs)) + try: + result = await asyncio.shield(task) + except asyncio.CancelledError: + with CancelScope(shield=True): + await settle_cancelled_call(task, discard_result) + raise + return result + + +async def settle_cancelled_call( + task: asyncio.Task, discard_result: Callable | None +) -> None: + """wait не переносит исключение SDK; повторная отмена не прерывает компенсацию.""" + while not task.done(): + with suppress(asyncio.CancelledError): + await asyncio.wait([task]) + completed = not task.cancelled() and task.exception() is None + if completed and discard_result is not None: + discard_result(task.result()) + + +@dataclass(frozen=True) +class ObjectInfo: + """Поля S3 сохраняют необязательность SDK; согласованность проверяет domain.""" + + key: str + size_bytes: int | None + content_type: str | None + last_modified: datetime | None + + +def close_response(response) -> None: + response.close() + response.release_conn() + + +class ObjectStream: + """Закрывается даже если StreamingResponse не успел начать итерацию.""" + + def __init__(self, storage, response): + self.storage = storage + self.response = response + self.deadline = time.monotonic() + storage.settings.download_timeout_seconds + self.closed = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self.closed: + raise StopAsyncIteration + try: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + async with asyncio.timeout(remaining): + chunk = await self.storage._call( + lambda: self.response.read(self.storage.settings.chunk_bytes) + ) + if not chunk: + await self.aclose() + raise StopAsyncIteration + except BaseException: + await self.aclose() + raise + return chunk + + async def aclose(self): + if not self.closed: + self.closed = True + close_response(self.response) + + +class MinioStorage: + def __init__(self, settings: S3Settings) -> None: + self.settings = settings + # Клиент создаётся в event loop один раз, не конкурентно в worker threads. + url = urlsplit(settings.endpoint) + self._http = urllib3.PoolManager( + timeout=urllib3.Timeout( + connect=settings.connect_timeout_seconds, + read=settings.read_timeout_seconds, + ), + retries=urllib3.Retry(total=0), + maxsize=32, + ) + self._health_http = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=0.25, read=0.5), + retries=urllib3.Retry(total=0), + ) + self._client = Minio( + url.netloc, + secure=url.scheme == "https", + region=settings.region, + access_key=settings.access_key_id_file.read_text().strip(), + secret_key=settings.secret_access_key_file.read_text().strip(), + http_client=self._http, + ) + + async def _call(self, operation: Callable, *, discard_result=None): + return await run_sync(operation, discard_result=discard_result) + + async def put_file( + self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str + ) -> None: + def put(): + with path.open("rb") as source: + # Один PUT: размер части равен максимальному разрешённому файлу. + self._client.put_object( + self.settings.bucket, + key, + source, + size_bytes, + content_type=content_type, + metadata={"sha256": sha256}, + part_size=10_485_760, + num_parallel_uploads=1, + ) + + await self._call(put) + + async def open_object(self, key: str) -> ObjectStream: + response = await self._call( + lambda: self._client.get_object(self.settings.bucket, key), + discard_result=close_response, + ) + return ObjectStream(self, response) + + async def stat_object(self, key: str) -> ObjectInfo: + def stat(): + info = self._client.stat_object(self.settings.bucket, key) + return ObjectInfo(key, info.size, info.content_type, info.last_modified) + + return await self._call(stat) + + async def delete_object(self, key: str) -> None: + await self._call(lambda: self._client.remove_object(self.settings.bucket, key)) + + async def check_available(self) -> bool: + status = await self._call( + lambda: self._health_http.request( + "GET", + self.settings.endpoint.rstrip("/") + "/minio/health/ready", + ).status + ) + return status == 200 + + async def aclose(self) -> None: + self._http.clear() + self._health_http.clear() + + +def create_s3_storage(settings: S3Settings) -> MinioStorage: + return MinioStorage(settings) diff --git a/app/storage/sessions.py b/app/storage/sessions.py new file mode 100644 index 00000000..d8a30f29 --- /dev/null +++ b/app/storage/sessions.py @@ -0,0 +1,38 @@ +from datetime import datetime +from uuid import UUID + +from redis.asyncio import Redis + +SESSION_KEY_PREFIX = "session:" + + +def session_key(token_hash: str) -> str: + """Возвращает ключ Redis для хеша токена.""" + return f"{SESSION_KEY_PREFIX}{token_hash}" + + +async def create_session( + redis: Redis, + *, + token_hash: str, + user_id: UUID, + expires_at: datetime, +) -> None: + """Добавляет сессию с абсолютным временем истечения.""" + await redis.set(session_key(token_hash), str(user_id), pxat=expires_at) + + +async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None: + """Возвращает id владельца действующей сессии или None.""" + raw_user_id = await redis.get(session_key(token_hash)) + if isinstance(raw_user_id, bytes): + raw_user_id = raw_user_id.decode() + user_id = UUID(raw_user_id) if raw_user_id else None + return user_id + + +async def delete_session(redis: Redis, token_hash: str) -> bool: + """Удаляет сессию; возвращает True, если она существовала.""" + deleted_count = await redis.delete(session_key(token_hash)) + is_deleted = deleted_count > 0 + return is_deleted diff --git a/app/storage/types.py b/app/storage/types.py new file mode 100644 index 00000000..e0357cfa --- /dev/null +++ b/app/storage/types.py @@ -0,0 +1,6 @@ +"""Общие типы слоя хранения.""" + +from datetime import datetime +from uuid import UUID + +Position = tuple[datetime, UUID] diff --git a/app/storage/users.py b/app/storage/users.py new file mode 100644 index 00000000..45214e9b --- /dev/null +++ b/app/storage/users.py @@ -0,0 +1,47 @@ +from collections.abc import Collection +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.user import User + + +async def create_user( + session: AsyncSession, + *, + username: str, + display_name: str, + password_hash: str, +) -> User: + """Добавляет пользователя; занятый username даёт IntegrityError при flush.""" + user = User( + username=username, display_name=display_name, password_hash=password_hash + ) + session.add(user) + await session.flush() + return user + + +async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None: + """Возвращает пользователя по id или None.""" + user = await session.get(User, user_id) + return user + + +async def get_user_by_username(session: AsyncSession, username: str) -> User | None: + """Возвращает пользователя по username или None.""" + stmt = select(User).where(User.username == username) + result = await session.execute(stmt) + user = result.scalar_one_or_none() + return user + + +async def get_existing_user_ids( + session: AsyncSession, user_ids: Collection[UUID] +) -> set[UUID]: + """Возвращает те из user_ids, что существуют, одним запросом.""" + stmt = select(User.id).where(User.id.in_(user_ids)) + result = await session.execute(stmt) + existing_ids = set(result.scalars().all()) + return existing_ids diff --git a/compose.test.yaml b/compose.test.yaml new file mode 100644 index 00000000..c20fa587 --- /dev/null +++ b/compose.test.yaml @@ -0,0 +1,10 @@ +# Override для отдельного тестового стенда (make test-stand-up): +# публикует PostgreSQL и Redis на 127.0.0.1, чтобы pytest из student-tests/ +# мог подключаться к ним напрямую. В основном стенде эти порты закрыты. +services: + postgres: + ports: + - "127.0.0.1:${DB_HOST_PORT:?Set DB_HOST_PORT in .env}:5432" + redis: + ports: + - "127.0.0.1:${REDIS_HOST_PORT:?Set REDIS_HOST_PORT in .env}:6379" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..ac342c52 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,295 @@ +name: mtusi-messenger-lab5 + +x-api-image: &api-image + build: + context: . + args: + API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env} + image: ${API_IMAGE:?Set API_IMAGE in .env} + +x-base-env: &base-env + APP_HOST: ${APP_HOST:?Set APP_HOST in .env} + APP_PORT: ${APP_PORT:?Set APP_PORT in .env} + APP_ENV: ${APP_ENV:?Set APP_ENV in .env} + LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env} + MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env} + MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env} + DB_HOST: ${DB_HOST:?Set DB_HOST in .env} + DB_PORT: ${DB_PORT:?Set DB_PORT in .env} + DB_NAME: ${DB_NAME:?Set DB_NAME in .env} + DB_USER: ${DB_USER:?Set DB_USER in .env} + DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env} + DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env} + SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env} + GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env} + READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env} + +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env} + S3_REGION: ${S3_REGION:?Set S3_REGION in .env} + S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env} + S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env} + S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env} + S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env} + S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env} + S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env} + S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env} + S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env} + S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env} + S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env} + +services: + minio: + build: + context: . + dockerfile: infra/minio/Dockerfile + target: server + image: mtusi-messenger-minio:lab5 + command: ["server", "/data", "--console-address", "127.0.0.1:9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + MINIO_BROWSER: ${MINIO_BROWSER:?Set MINIO_BROWSER in .env} + secrets: + - source: minio_root_user + target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + - source: minio_root_password + target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + volumes: + - miniodata:/data + networks: [objects] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + minio-bootstrap: + build: + context: . + dockerfile: infra/minio/Dockerfile + target: bootstrap + image: mtusi-messenger-minio-bootstrap:lab5 + environment: + S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env} + S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + secrets: + - source: minio_root_user + target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env} + - source: minio_root_password + target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env} + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [objects] + restart: "no" + depends_on: + minio: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + postgres: + image: postgres:16.10-bookworm + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USER:?Set DB_USER in .env} + POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env} + POSTGRES_PASSWORD_FILE: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env} + secrets: + - source: db_password + target: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + redis: + image: redis:7.4.2-bookworm + restart: unless-stopped + entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"] + secrets: + - redis_password + volumes: + - ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + - ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro + - redisdata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"] + interval: 3s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + + migrate: + <<: *api-image + command: ["alembic", "upgrade", "head"] + environment: *base-env + secrets: + - db_password + depends_on: + postgres: + condition: service_healthy + restart: "no" + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + api: + <<: *api-image + restart: unless-stopped + environment: + <<: [*base-env, *s3-env] + REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env} + REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env} + REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env} + REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env} + REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env} + REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env} + REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env} + SESSION_STORE: ${SESSION_STORE:?Set SESSION_STORE in .env} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env} + secrets: + - db_password + - redis_password + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m,mode=1777 + deploy: + replicas: ${API_REPLICAS:?Set API_REPLICAS in .env} + resources: + limits: + cpus: "1.0" + memory: 1536M + healthcheck: + test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"] + interval: 3s + timeout: 3s + start_period: 10s + retries: 2 + stop_signal: SIGTERM + stop_grace_period: 8s + depends_on: + minio: + condition: service_healthy + minio-bootstrap: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + cleanup: + <<: *api-image + command: ["python", "-m", "app.core.cleanup", "--apply"] + restart: unless-stopped + environment: + <<: [*base-env, *s3-env] + secrets: + - db_password + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + minio-bootstrap: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + nginx: + image: nginx:1.28.0-alpine + restart: unless-stopped + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"] + interval: 5s + timeout: 3s + retries: 5 + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + +volumes: + pgdata: + redisdata: + miniodata: + +networks: + default: + objects: + internal: true + +secrets: + db_password: + file: ./secrets/db_password + redis_password: + file: ./secrets/redis_password + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/infra/README.md b/infra/README.md index 5a39e862..fda1b846 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,13 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Перенесённая база lab4 + +В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap, +migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления. +Роли broker, outbox publisher и worker пока не реализованы. +core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди. +Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md. +Настройки environment в Compose задаются через .env; образец — .env.example. +Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам. diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..fd7cc06c --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24.8-bookworm AS compiler +ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4 +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z + +FROM python:3.12.15-slim-bookworm AS runtime +RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data +USER minio + +FROM runtime AS server +COPY --from=compiler /out/minio /usr/local/bin/minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" \ + org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z" +VOLUME ["/data"] +ENTRYPOINT ["/usr/local/bin/minio"] + +FROM runtime AS bootstrap +COPY --from=compiler /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.py /bootstrap.py +ENTRYPOINT ["python", "/bootstrap.py"] diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py new file mode 100644 index 00000000..3b8c4b29 --- /dev/null +++ b/infra/minio/bootstrap.py @@ -0,0 +1,117 @@ +"""Идемпотентный private bucket и отдельная ограниченная учётная запись API.""" + +import json +import os +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import time +from urllib.request import urlopen + + +def secret(name): + value = Path(os.environ[name.upper() + "_FILE"]).read_text().strip() + if not value: + raise ValueError("Пустой файл секрета") + return value + + +def main(): + endpoint = os.environ["S3_ENDPOINT"] + bucket = os.environ["S3_BUCKET"] + root_user, root_password = secret("minio_root_user"), secret("minio_root_password") + access, password = secret("s3_access_key_id"), secret("s3_secret_access_key") + if access == root_user: + raise ValueError("API не должен использовать root credentials") + deadline = time.monotonic() + 60 + while True: + try: + with urlopen(endpoint + "/minio/health/ready", timeout=2) as response: + if response.status == 200: + break + except OSError: + pass + if time.monotonic() >= deadline: + raise RuntimeError("MinIO не готов за 60 секунд") + time.sleep(1) + with TemporaryDirectory(prefix="minio-bootstrap-") as directory: + config = Path(directory) + # Credentials не передаются mc alias set через аргументы или окружение. + config_file = config / "config.json" + config_file.write_text( + json.dumps( + { + "version": "10", + "aliases": { + "storage": { + "url": endpoint, + "accessKey": root_user, + "secretKey": root_password, + "api": "S3v4", + "path": "auto", + } + }, + } + ) + ) + config_file.chmod(0o600) + + def mc(*args): + result = subprocess.run( + ["mc", "--config-dir", directory, *args], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) + if result.returncode: + raise RuntimeError( + "Ошибка bootstrap MinIO; значения credentials скрыты" + ) + + mc("mb", "--ignore-existing", "storage/" + bucket) + mc("anonymous", "set", "none", "storage/" + bucket) + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": [f"arn:aws:s3:::{bucket}"], + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": [f"arn:aws:s3:::{bucket}/originals/*"], + }, + ], + } + policy_file = config / "policy.json" + policy_file.write_text(json.dumps(policy)) + mc( + "admin", + "policy", + "create", + "storage", + "messenger-attachments", + str(policy_file), + ) + # mc не поддерживает stdin для user add: вывод и ошибки команды скрыты. + mc("admin", "user", "add", "storage", access, password) + mc( + "admin", + "policy", + "attach", + "storage", + "messenger-attachments", + "--user", + access, + ) + print("MinIO private bucket and API policy configured", flush=True) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print("MinIO bootstrap failed: " + type(error).__name__, flush=True) + raise SystemExit(1) from None diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..0d62297b --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,140 @@ +# Единая точка входа: round-robin между API-репликами, без sticky sessions. +# Директивы рассчитаны на закреплённый nginx:1.28.0-alpine. + +worker_processes 2; + +events { + worker_connections 1024; +} + +http { + # В логе используем request_id ответа API, для собственных ошибок — id nginx. + map $upstream_http_x_request_id $response_request_id { + "" $request_id; + default $upstream_http_x_request_id; + } + + log_format requests escape=json + '{"time":"$time_iso8601","request_id":"$response_request_id",' + '"method":"$request_method","uri":"$uri","status":$status,' + '"bytes":$body_bytes_sent,"request_time":$request_time,' + '"upstream":"$upstream_addr","upstream_status":"$upstream_status",' + '"upstream_time":"$upstream_response_time",' + '"instance_id":"$upstream_http_x_instance_id",' + '"limit_status":"$limit_req_status"}'; + + access_log /dev/stdout requests; + error_log /dev/stderr warn; + server_tokens off; + + # DNS Docker возвращает адреса всех реплик сервиса api. + resolver 127.0.0.11 valid=5s ipv6=off; + resolver_timeout 1s; + + # Общий для worker-процессов лимит на IP, который видит nginx. + # Клиентский X-Forwarded-For не используется в качестве ключа лимита. + limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s; + + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_comp_level 3; + gzip_types application/json; + + upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=1 fail_timeout=5s; + + # Кэш свободных соединений на worker, а не лимит запросов к API. + keepalive 16; + keepalive_timeout 4s; + } + + server { + listen 8080; + + client_header_timeout 10s; + client_body_timeout 10s; + # Размер ограничивает API при чтении, включая chunked-передачу. + client_max_body_size 0; + client_body_buffer_size 64k; + send_timeout 10s; + keepalive_timeout 30s; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 500ms; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + + # Пассивное обнаружение отказов; Docker healthcheck сам не управляет LB. + # non_idempotent не включён: уже отправленные POST не повторяем. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + # Окно перехода на другой backend, не общий deadline ответа. + proxy_next_upstream_timeout 2s; + + # JSON и заголовки ошибок API передаются без подмены. + proxy_intercept_errors off; + error_page 429 = @rate_limited; + error_page 502 503 504 = @unavailable; + + # Проверка самого LB для healthcheck в compose; не проксируется в API. + location = /lb-health { + access_log off; + default_type text/plain; + return 200 "ok\n"; + } + + # Проверки готовности/жизни не расходуют бюджет бизнес-запросов. + location /health/ { + proxy_pass http://api_backend; + } + + location / { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + limit_req_log_level warn; + proxy_pass http://api_backend; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 65s; + proxy_send_timeout 15s; + # Ошибка одного upload не должна скрыть работающую реплику для чатов. + proxy_next_upstream off; + proxy_pass http://api_backend; + } + + location ~ ^/api/v1/attachments/[^/]+/content$ { + limit_req zone=per_client burst=100 nodelay; + limit_req_status 429; + proxy_buffering off; + proxy_read_timeout 65s; + proxy_pass http://api_backend; + } + + # Это ответы балансировщика: X-Instance-Id реплики у них отсутствует. + location @rate_limited { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header Retry-After 1 always; + return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}'; + } + + location @unavailable { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header Retry-After 1 always; + return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}'; + } + } +} diff --git a/infra/redis/redis.conf b/infra/redis/redis.conf new file mode 100644 index 00000000..f2ae982f --- /dev/null +++ b/infra/redis/redis.conf @@ -0,0 +1,5 @@ +dir /data +appendonly yes +appendfsync always +maxmemory 256mb +maxmemory-policy noeviction diff --git a/infra/redis/start-redis.sh b/infra/redis/start-redis.sh new file mode 100644 index 00000000..49ee706b --- /dev/null +++ b/infra/redis/start-redis.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +password_hash=$(tr -d '\r\n' /run/redis/users.acl +chown redis:redis /run/redis/users.acl +chmod 600 /run/redis/users.acl +exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 00000000..d914a274 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,68 @@ +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from app.core.config import Settings +from app.models import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + +target_metadata = Base.metadata + + +def _database_url() -> str: + explicit = config.get_main_option("sqlalchemy.url") + if explicit: + return explicit + return Settings().get_database_url() # type: ignore + + +def run_migrations_offline() -> None: + context.configure( + url=_database_url(), + target_metadata=target_metadata, + literal_binds=True, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + section = config.get_section(config.config_ini_section, {}) + section["sqlalchemy.url"] = _database_url().replace("%", "%%") + connectable = async_engine_from_config( + section, prefix="sqlalchemy.", poolclass=pool.NullPool + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 00000000..17dcba0e --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/.gitkeep b/migrations/versions/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py new file mode 100644 index 00000000..ea3c65fc --- /dev/null +++ b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py @@ -0,0 +1,43 @@ +"""Добавляет результаты идемпотентных запросов.""" + +from alembic import op +import sqlalchemy as sa + +revision = "49d2c947a3f1" +down_revision = "abd273076068" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Создаёт таблицу ключей без изменения существующих данных.""" + op.create_table( + "idempotency_keys", + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("method", sa.String(8), nullable=False), + sa.Column("path", sa.String(255), nullable=False), + sa.Column("key", sa.String(128), nullable=False), + sa.Column("request_hash", sa.String(64), nullable=False), + sa.Column("message_id", sa.UUID(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), nullable=False, + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users") + ), + sa.ForeignKeyConstraint( + ["message_id"], ["messages.id"], + name=op.f("fk_idempotency_keys_message_id_messages"), + ), + sa.PrimaryKeyConstraint( + "user_id", "method", "path", "key", name=op.f("pk_idempotency_keys") + ), + ) + op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]) + + +def downgrade() -> None: + """Удаляет таблицу результатов запросов.""" + op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys") + op.drop_table("idempotency_keys") diff --git a/migrations/versions/8f31b94a2d10_add_attachments.py b/migrations/versions/8f31b94a2d10_add_attachments.py new file mode 100644 index 00000000..b221dccb --- /dev/null +++ b/migrations/versions/8f31b94a2d10_add_attachments.py @@ -0,0 +1,61 @@ +"""Attachment, связи сообщений и устойчивый журнал загрузок.""" + +from alembic import op +import sqlalchemy as sa + +revision = "8f31b94a2d10" +down_revision = "49d2c947a3f1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "attachments", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("owner_id", sa.UUID(), nullable=False), + sa.Column("object_key", sa.String(128), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("status", sa.String(16), server_default="ready", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")), + sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")), + sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")), + sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")), + sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")), + sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")), + ) + op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"]) + op.create_table( + "message_attachments", + sa.Column("message_id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")), + sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")), + sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")), + ) + op.create_table( + "upload_intents", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("object_key", sa.String(128), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")), + sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")), + ) + op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_upload_intents_created_at", table_name="upload_intents") + op.drop_table("upload_intents") + op.drop_table("message_attachments") + op.drop_index("ix_attachments_chat_id", table_name="attachments") + op.drop_table("attachments") diff --git a/migrations/versions/abd273076068_add_counters.py b/migrations/versions/abd273076068_add_counters.py new file mode 100644 index 00000000..9c809b06 --- /dev/null +++ b/migrations/versions/abd273076068_add_counters.py @@ -0,0 +1,33 @@ +"""add counters + +Revision ID: abd273076068 +Revises: db06fd3126ff +Create Date: 2026-10-26 22:01:01.910396 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = 'abd273076068' +down_revision: Union[str, None] = 'db06fd3126ff' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('counters', + sa.Column('name', sa.String(length=64), nullable=False), + sa.Column('value', sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint('name', name=op.f('pk_counters')) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('counters') + # ### end Alembic commands ### diff --git a/migrations/versions/db06fd3126ff_create_initial_schema.py b/migrations/versions/db06fd3126ff_create_initial_schema.py new file mode 100644 index 00000000..eec6e51b --- /dev/null +++ b/migrations/versions/db06fd3126ff_create_initial_schema.py @@ -0,0 +1,107 @@ +"""create initial schema + +Revision ID: db06fd3126ff +Revises: +Create Date: 2026-10-26 17:09:16.675339 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "db06fd3126ff" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "chats", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("title", sa.String(length=100), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")), + ) + op.create_table( + "users", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("username", sa.String(length=32), nullable=False), + sa.Column("display_name", sa.String(length=100), nullable=False), + sa.Column("password_hash", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("username", name=op.f("uq_users_username")), + ) + op.create_table( + "chat_members", + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.ForeignKeyConstraint( + ["chat_id"], + ["chats.id"], + name=op.f("fk_chat_members_chat_id_chats"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users") + ), + sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")), + ) + op.create_index( + "ix_chat_members_user_id_chat_id", + "chat_members", + ["user_id", "chat_id"], + unique=False, + ) + op.create_table( + "messages", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("sender_id", sa.UUID(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats") + ), + sa.ForeignKeyConstraint( + ["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), + ) + op.create_index( + "ix_messages_chat_id_created_at_id", + "messages", + ["chat_id", "created_at", "id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages") + op.drop_table("messages") + op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members") + op.drop_table("chat_members") + op.drop_table("users") + op.drop_table("chats") + # ### end Alembic commands ### diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..b7dd8fa8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = student-tests +python_files = test_*.py +asyncio_mode = auto +asyncio_default_fixture_loop_scope = session +asyncio_default_test_loop_scope = session diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..f171cf56 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,52 @@ +alembic==1.20.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.143.3 +h11==0.16.0 +httpcore2==2.13.1 +httpx2==2.13.1 +idna==3.20 +iniconfig==2.3.1 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.12.0 +Mako==1.4.3 +MarkupSafe==3.0.4 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +opentelemetry-api==1.45.1 +packaging==26.3 +pathable==0.4.4 +pluggy==1.7.0 +pydantic==2.14.1 +pydantic_core==2.50.1 +Pygments==2.21.0 +pytest==9.1.1 +PyYAML==6.0.3 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +SQLAlchemy==2.1.4 +starlette==1.8.0 +truststore==0.10.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.54.0 +asyncpg==0.32.0 +argon2-cffi==25.1.0 +redis==8.1.0 +greenlet==3.5.6 +pydantic-settings==2.16.0 +pytest-asyncio==1.4.1 +httpx==0.28.1 +minio==7.2.20 +Pillow==13.0.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..592ea408 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Команды управления учебным стендом.""" diff --git a/scripts/check_grade4.py b/scripts/check_grade4.py new file mode 100644 index 00000000..a2c90bcd --- /dev/null +++ b/scripts/check_grade4.py @@ -0,0 +1,355 @@ +"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда.""" + +import asyncio +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import subprocess +from threading import Event +import time +from uuid import uuid4 + +import asyncpg + +from scripts.deployment import ComposeProject, ROOT, deployment_lock +from scripts.failover_all import exercise, failure_kind, history +from scripts.rolling_update import update +from tests.client import Client, Failure, arguments, check, run + +PROBE = """import json, sys, urllib.request, urllib.error +payload = json.load(sys.stdin) +results = [] +for item in payload: + data = json.dumps(item['body']).encode() if 'body' in item else None + request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET')) + try: + response = urllib.request.urlopen(request, timeout=7) + except urllib.error.HTTPError as error: + response = error + with response: + results.append({'status': response.status, 'body': json.loads(response.read())}) +print(json.dumps(results)) +""" + + +def probe(project, runner, requests): + """Обращается к внутреннему API, передавая токен только через stdin.""" + result = subprocess.run( + ["docker", "exec", "-i", runner, "python", "-c", PROBE], + input=json.dumps(requests), + text=True, + capture_output=True, + timeout=10, + ) + check(result.returncode == 0, "internal HTTP probe failed") + return json.loads(result.stdout) + + +def audit(project): + """Проверяет фактические лимиты, порты и область доступа к секретам.""" + allowed = { + "postgres": {"db_password"}, + "redis": {"redis_password"}, + "migrate": {"db_password"}, + "api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"}, + "minio": {"minio_root_user", "minio_root_password"}, + "minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"}, + "cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"}, + "nginx": set(), + } + ids = project.run("ps", "--all", "--quiet").splitlines() + containers = json.loads(project.docker("inspect", *ids)) + for item in containers: + service = item["Config"]["Labels"]["com.docker.compose.service"] + check(service in allowed, "unexpected service") + host = item["HostConfig"] + check( + host["Memory"] > 0 and host["NanoCpus"] > 0, + service + ": missing CPU/RAM limit", + ) + env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]} + check( + not env_names + & { + "DB_PASSWORD", + "REDIS_PASSWORD", + "POSTGRES_PASSWORD", + "REDISCLI_AUTH", + "DATABASE_URL", + "MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", + }, + service + ": password in environment", + ) + secrets = { + Path(mount["Destination"]).name + for mount in item["Mounts"] + if mount["Destination"].startswith("/run/secrets/") + } + check(secrets == allowed[service], service + ": incorrect secret scope") + if service in {"api", "minio"}: + check(not host["PortBindings"], service + " publishes ports") + for mount in item["Mounts"]: + if mount["Destination"].startswith("/run/secrets/"): + check(not mount["RW"], "writable secret mount") + check( + "container_name" not in project.config["services"][service], + "fixed container name", + ) + check( + len(containers) == len(allowed) - 1 + len(project.replicas()), + "unexpected container count", + ) + print( + "PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords", + flush=True, + ) + + +async def graceful(project, base_url): + """Удерживает реальный POST в БД и завершает его после SIGTERM.""" + replicas = project.replicas() + target, runner = replicas[:2] + client = Client(base_url) + actor, _, _, chat = await asyncio.to_thread(client.bootstrap) + path = f"/api/v1/chats/{chat['id']}/messages" + key = uuid4().hex + raw = json.dumps([actor["id"], "POST", path, key]).encode() + lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True) + database = project.config["services"]["postgres"] + env = database["environment"] + conn = await asyncpg.connect( + host="127.0.0.1", + port=int(database["ports"][0]["published"]), + database=env["POSTGRES_DB"], + user=env["POSTGRES_USER"], + password=(ROOT / "secrets/db_password").read_text().strip(), + timeout=3, + ) + network = project.config['networks']['default']['name'] + address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}' + address = project.docker( + "inspect", + "--format", + address_template, + target.id, + ) + check(bool(address), 'API has no address in the application network') + port = project.config["services"]["api"]["environment"]["APP_PORT"] + direct = f"http://{address}:{port}" + started = time.monotonic() + task = None + try: + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id) + task = asyncio.create_task( + asyncio.to_thread( + probe, + project, + runner.id, + [ + { + "url": direct + path, + "method": "POST", + "body": {"text": "graceful-" + key}, + "headers": { + "Authorization": "Bearer " + actor["token"], + "Idempotency-Key": key, + "Content-Type": "application/json", + }, + } + ], + ) + ) + deadline = time.monotonic() + 3 + while not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())" + ): + check(time.monotonic() < deadline, "POST did not reach database lock") + await asyncio.sleep(0.02) + started = time.monotonic() + await asyncio.to_thread( + project.docker, "kill", "--signal", "SIGTERM", target.id + ) + results = await asyncio.to_thread( + probe, + project, + runner.id, + [ + {"url": direct + "/health/ready"}, + {"url": direct + "/health/live"}, + {"url": direct + "/api/v1/users/me"}, + ], + ) + check( + [item["status"] for item in results] == [503, 200, 503], + "incorrect drain readiness/liveness/new work", + ) + await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started))) + result = (await task)[0] + check(result["status"] == 201, "in-flight POST failed during SIGTERM") + await asyncio.to_thread(project.docker, "wait", target.id, timeout=10) + state = json.loads( + project.docker("inspect", "--format", "{{json .State}}", target.id) + ) + check( + not state["OOMKilled"] and state["ExitCode"] != 137, + "forced shutdown instead of graceful completion", + ) + check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget") + items = await asyncio.to_thread(history, client, actor, chat) + check( + result["body"]["id"] in {item["id"] for item in items}, + "acknowledged in-flight message lost", + ) + print( + f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s", + flush=True, + ) + finally: + await conn.close() + if task: + await asyncio.gather(task, return_exceptions=True) + if any(item.id == target.id and item.running for item in project.replicas()): + await asyncio.to_thread(project.docker, "stop", target.id) + await asyncio.to_thread(project.docker, "start", target.id) + await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas}) + await asyncio.to_thread(project.wait_routed, target.instance_id, base_url) + + +class ObservedProject(ComposeProject): + """Подсчитывает миграции и проверяет готовность перед заменой.""" + + def __init__(self): + self.migrations = 0 + self.replacements = 0 + self.replacement_started = [] + super().__init__() + + def run(self, *args, **kwargs): + if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"): + self.migrations += 1 + return super().run(*args, **kwargs) + + def docker(self, *args, **kwargs): + if args[0] == "stop": + healthy = [ + item + for item in self.replicas() + if item.id != args[1] and item.running and item.health == "healthy" + ] + check(len(healthy) >= 2, "rolling update reduces healthy capacity") + self.replacements += 1 + self.replacement_started.append(time.monotonic()) + return super().docker(*args, **kwargs) + + +def rolling(base_url): + """Заменяет образ под постоянной записью с проверкой сохранности.""" + project = ObservedProject() + before = project.replicas() + client = Client(base_url) + actor, _, _, chat = client.bootstrap() + stop = Event() + results, errors = [], [] + + def worker(): + local = Client(base_url, timeout=2) + key = uuid4().hex + while not stop.is_set(): + started = time.monotonic() + try: + result = local.message(actor, chat, "rolling-" + key, key=key) + results.append( + (started, time.monotonic(), result["json"]["id"], result["elapsed"]) + ) + key = uuid4().hex + except Failure as error: + errors.append(failure_kind(error)) + stop.wait(0.08) + + with ThreadPoolExecutor(max_workers=2) as pool: + jobs = [pool.submit(worker) for _ in range(2)] + try: + time.sleep(1) + started = time.monotonic() + update(project, base_url, revision="verification-" + uuid4().hex[:12]) + finished = time.monotonic() + time.sleep(1) + finally: + stop.set() + for job in jobs: + job.result(timeout=5) + after = project.replicas() + check( + project.migrations == 1 and project.replacements == len(before), + "migration/replacement count incorrect", + ) + check( + {item.image for item in before}.isdisjoint({item.image for item in after}), + "new image was not deployed", + ) + check( + {item.id for item in before}.isdisjoint({item.id for item in after}), + "old containers remain", + ) + writes = [row for row in results if started <= row[0] <= finished] + check(bool(writes), "no writes during update") + check( + any(project.replacement_started[0] <= row[0] <= finished for row in writes), + "no writes during container replacement", + ) + items = history(client, actor, chat) + check( + {row[2] for row in results} <= {item["id"] for item in items}, + "rolling update lost confirmed messages", + ) + check( + len(items) == len({item["text"] for item in items}), + "rolling update created duplicate messages", + ) + latencies = sorted(row[3] for row in writes) + print( + json.dumps( + { + "scenario": "rolling update", + "replacements": project.replacements, + "migration_runs": project.migrations, + "successful_writes": len(writes), + "failed_attempts": len(errors), + "errors_by_kind": dict(Counter(errors)), + "elapsed_seconds": round(finished - started, 3), + "successful_rps": round(len(writes) / (finished - started), 3), + "p95_seconds": round( + latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3 + ), + } + ), + flush=True, + ) + + +def main(): + """Выполняет проверки только на отдельном тестовом проекте.""" + + def configure(parser): + parser.add_argument( + "--phase", choices=("all", "update-and-failover"), default="all" + ) + + args = arguments("Grade 4 integration checks", configure) + project = ComposeProject() + check(project.name.endswith("-test"), "use isolated test Compose project") + with deployment_lock(project): + audit(project) + if args.phase == "all": + asyncio.run(graceful(project, args.base_url)) + rolling(args.base_url) + args.duration, args.concurrency = 10, 2 + exercise(project, args) + audit(project) + + +if __name__ == "__main__": + run(main) diff --git a/scripts/check_idempotency.py b/scripts/check_idempotency.py new file mode 100644 index 00000000..ae9dfa20 --- /dev/null +++ b/scripts/check_idempotency.py @@ -0,0 +1,39 @@ +"""Проверяет конкурентные повторы через балансировщик.""" + +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +from tests.client import Client, arguments, check, run + + +def main() -> None: + """Отправляет один ключ параллельно через разные реплики.""" + args = arguments("Concurrent idempotency through load balancer") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = uuid4().hex + body = "parallel-" + key + + def send(_): + return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key) + + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send, range(24))) + ids = {result["json"]["id"] for result in results} + replicas = {result["headers"]["X-Instance-Id"] for result in results} + check(len(ids) == 1, "parallel retries created duplicate messages") + check(len(replicas) >= 2, "parallel retries did not reach two API replicas") + history = client.call( + "GET", + f"/api/v1/chats/{chat['id']}/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"]["items"] + check( + len(history) == 1 and history[0] == results[0]["json"], "history has duplicates" + ) + print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/check_nginx.py b/scripts/check_nginx.py new file mode 100644 index 00000000..e4881fc5 --- /dev/null +++ b/scripts/check_nginx.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения.""" + +from concurrent.futures import ThreadPoolExecutor +import gzip +import json +from pathlib import Path +import subprocess +import tempfile +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +IMAGE = "nginx:1.28.0-alpine" +PAYLOAD = json.dumps({"text": "message " * 256}).encode() + + +def check(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +class Stand: + def __init__(self, directory: str) -> None: + self.directory = Path(directory) + self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12] + self.base_url = "" + backend = self.directory / "backend.conf" + backend.write_text( + "worker_processes 1;\n" + "events { worker_connections 1024; }\n" + "http {\n" + " log_format backend 'BACKEND $request_method $uri';\n" + " access_log /dev/stdout backend;\n" + " server {\n" + " listen 8080;\n" + " default_type application/json;\n" + " add_header X-Instance-Id $hostname always;\n" + " add_header X-Request-Id $request_id always;\n" + " add_header Retry-After 7 always;\n" + ' location /health/ { return 200 \'{"status":"ok"}\'; }\n' + " location = /api/limited {\n" + ' return 429 \'{"error":{"code":"rate_limited",' + '"message":"upstream limit","request_id":"$request_id"}}\';\n' + " }\n" + " location = /api/post-failure {\n" + ' return 503 \'{"error":{"code":"unavailable",' + '"message":"upstream failure","request_id":"$request_id"}}\';\n' + " }\n" + " location / { return 200 '" + PAYLOAD.decode() + "'; }\n" + " }\n" + "}\n" + ) + self.compose_file = self.directory / "compose.json" + self.compose_file.write_text( + json.dumps( + { + "services": { + "api": { + "image": IMAGE, + "volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"], + }, + "nginx": { + "image": IMAGE, + "volumes": [ + f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro" + ], + "ports": ["127.0.0.1::8080"], + }, + } + } + ) + ) + + def compose(self, *args: str) -> str: + # Явный файл, уникальный проект и пустой env-file исключают основной стенд. + result = subprocess.run( + [ + "docker", + "compose", + "--env-file", + "/dev/null", + "-f", + str(self.compose_file), + "-p", + self.project, + *args, + ], + cwd=self.directory, + capture_output=True, + text=True, + timeout=45, + ) + if result.returncode: + raise RuntimeError(result.stdout + result.stderr) + return result.stdout + result.stderr + + def call(self, path: str, *, method: str = "GET", encoding: str = "identity"): + request = Request( + self.base_url + path, + method=method, + headers={"Accept-Encoding": encoding}, + data=b"{}" if method == "POST" else None, + ) + try: + response = urlopen(request, timeout=3) + except HTTPError as error: + response = error + except (URLError, TimeoutError): + return 0, {}, b"" + with response: + return ( + response.status, + {key.lower(): value for key, value in response.headers.items()}, + response.read(), + ) + + def wait_instances(self, count: int) -> set[str]: + seen: set[str] = set() + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + status, headers, _ = self.call("/health/live") + if status == 200: + seen.add(headers["x-instance-id"]) + if len(seen) == count: + return seen + time.sleep(0.05) + raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}") + + +def check_error(response, status: int, code: str) -> None: + actual_status, headers, body = response + check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}") + check( + headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON" + ) + error = json.loads(body)["error"] + check(error["code"] == code, "Неверный код ошибки") + check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает") + + +def run_checks(stand: Stand) -> None: + stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx") + address = stand.compose("port", "nginx", "8080").strip() + stand.base_url = "http://" + address + stand.wait_instances(2) + print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip()) + stand.compose("exec", "-T", "nginx", "nginx", "-t") + print("PASS: nginx -t и балансировка двух реплик") + + status, headers, body = stand.call("/api/large", encoding="gzip") + check( + status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON" + ) + check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое") + check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ") + check( + "accept-encoding" in headers.get("vary", "").lower(), + "Нет Vary: Accept-Encoding", + ) + check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён") + small = stand.call("/health/live", encoding="gzip") + check( + small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат" + ) + print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено") + + limited = stand.call("/api/limited") + check_error(limited, 429, "rate_limited") + check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения") + check("x-instance-id" in limited[1], "Потерян id backend на ошибке API") + print("PASS: JSON и заголовки ошибки API передаются без подмены") + + failure = stand.call("/api/post-failure", method="POST") + check_error(failure, 503, "unavailable") + backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api") + check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился") + print("PASS: отправленный POST не повторяется на другой реплике") + + with ThreadPoolExecutor(max_workers=32) as pool: + responses = list(pool.map(lambda _: stand.call("/api/large"), range(500))) + rejected = [response for response in responses if response[0] == 429] + check(bool(rejected), "Не удалось вызвать ограничение частоты") + check( + all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке" + ) + for response in rejected: + check_error(response, 429, "rate_limited") + check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика") + check("x-instance-id" not in response[1], "LB выдал себя за API-реплику") + with ThreadPoolExecutor(max_workers=16) as pool: + health = list(pool.map(lambda _: stand.call("/health/ready"), range(100))) + check(all(response[0] == 200 for response in health), "Лимит затронул health") + check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна") + print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен") + + stand.compose( + "up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api" + ) + instances = stand.wait_instances(3) + print("PASS: третья реплика обнаружена через DNS без reload nginx") + + target = sorted(instances)[0] + containers = stand.compose("ps", "-q", "api").splitlines() + matching = [container for container in containers if container.startswith(target)] + check(len(matching) == 1, "Не найдена тестовая реплика для остановки") + # Все id взяты только из уникального временного Compose-проекта. + started = time.monotonic() + subprocess.run( + ["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True + ) + recovery = None + for _ in range(20): + status, headers, _ = stand.call("/health/live") + if status == 200: + check(headers["x-instance-id"] != target, "Остановленная реплика отвечает") + if recovery is None: + recovery = time.monotonic() - started + check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с") + print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с") + + stand.compose("stop", "--timeout", "1", "api") + unavailable = stand.call("/health/ready") + check_error(unavailable, 503, "unavailable") + check( + unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API" + ) + check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики") + check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB") + print("PASS: недоступность всех backend возвращает JSON 503") + + logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx") + entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] + check(bool(entries), "Нет JSON access log") + check( + all("request_id" in entry and "upstream_time" in entry for entry in entries), + "Неполный лог", + ) + check( + any(entry["limit_status"] == "REJECTED" for entry in entries), + "Лимит не отражён в логе", + ) + print("PASS: JSON access log содержит время backend и результат лимита") + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory: + stand = Stand(directory) + try: + run_checks(stand) + except Exception: + print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True) + raise + finally: + stand.compose("down", "--remove-orphans", "--timeout", "1") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_s3.py b/scripts/check_s3.py new file mode 100644 index 00000000..c8422781 --- /dev/null +++ b/scripts/check_s3.py @@ -0,0 +1,137 @@ +"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ.""" + +import json +import base64 +from pathlib import Path +import subprocess +import time + +from scripts.deployment import ComposeProject +from tests.client import Client, arguments, check, run + +PROBE = r''' +import asyncio, json, sys +from pathlib import Path +from uuid import UUID +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.storage.s3 import create_s3_storage +from app.models.attachment import Attachment +from minio.error import S3Error +async def main(): + settings = Settings() + engine = create_db_engine(settings) + storage = create_s3_storage(settings.s3) + try: + async with create_session_factory(engine)() as session: + record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id'])) + key = record.object_key + results = {} + for method in ('GET', 'PUT'): + request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key, + method=method, data=b'x' if method == 'PUT' else None) + try: + with urlopen(request, timeout=3) as response: + results[method] = response.status + except HTTPError as error: + results[method] = error.code + error.close() + try: + storage._client.get_bucket_policy(settings.s3.bucket) + results['admin_denied'] = False + except S3Error as error: + results['admin_denied'] = error.code == 'AccessDenied' + try: + buckets = storage._client.list_buckets() + results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets) + except S3Error as error: + results['other_buckets_denied'] = error.code == 'AccessDenied' + try: + with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1): + results['console_unreachable'] = False + except HTTPError as error: + results['console_unreachable'] = False + error.close() + except URLError: + results['console_unreachable'] = True + results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*')) + results['root_files_absent'] = not any(Path('/run/secrets', name).exists() + for name in ('minio_root_user', 'minio_root_password')) + print(json.dumps(results)) + finally: + await storage.aclose() + await engine.dispose() +asyncio.run(main()) +''' + + +def main(): + def configure(parser): + parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта') + parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД') + args = arguments('MinIO private bucket and failure checks', configure) + project = ComposeProject() + client = Client(args.base_url, args.ca_file, timeout=40) + alice, _, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + # Повторный bootstrap не должен очистить bucket или сменить используемые credentials. + for _ in range(2): + project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100) + replicas = project.replicas() + check(bool(replicas), 'API not running') + result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE], + input=json.dumps({'attachment_id': attachment['id']}), + text=True, capture_output=True, timeout=20) + check(result.returncode == 0, 'private bucket probe failed (details hidden)') + statuses = json.loads(result.stdout) + check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed') + check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access') + check(statuses['console_unreachable'], 'console reachable from API network') + check(statuses['temp_originals_absent'], 'API retained temporary originals') + check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports') + env = project.config['services']['minio']['environment'] + check(env['MINIO_BROWSER'] == 'off', 'browser enabled') + check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback') + path = '/api/v1/attachments/' + attachment['id'] + '/content' + check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original') + if args.with_faults: + project.run('stop', 'cleanup') + try: + fault_script = Path(__file__).with_name('s3_faults.py').read_text() + result = subprocess.run( + ['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script], + input=json.dumps({'user': alice['user'], 'chat_id': chat['id'], + 'image': base64.b64encode(original).decode()}), + text=True, capture_output=True, timeout=60, + ) + check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)') + check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed') + finally: + project.run('start', 'cleanup') + check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original') + if args.with_outage: + try: + project.run('stop', 'minio') + client.call('GET', path, expected=503, actor=alice) + finally: + project.run('start', 'minio') + deadline = time.monotonic() + 60 + while True: + try: + client.call('GET', '/health/ready', schema='Health') + break + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart') + print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console' + + (', MinIO outage/recovery' if args.with_outage else '')) + if args.with_faults: + print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py new file mode 100644 index 00000000..f4ff9ef1 --- /dev/null +++ b/scripts/collect_evidence.py @@ -0,0 +1,204 @@ +"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence.""" + +from datetime import datetime, timezone +import hashlib +from importlib.metadata import version +import json +import os +from pathlib import Path +import platform +import re +import shlex +import subprocess +from tempfile import TemporaryDirectory +from zoneinfo import ZoneInfo + +ROOT = Path(__file__).resolve().parents[1] +PROJECT = 'mtusi-messenger-lab5-test' +EVIDENCE = ROOT / 'evidence' +ENVIRONMENT = { + **os.environ, + 'COMPOSE_PROJECT_NAME': PROJECT, + 'COMPOSE_FILE': 'compose.yaml:compose.test.yaml', + 'LB_HOST_PORT': '28080', +} +COMPOSE = ['docker', 'compose', '--env-file', '.env'] +NONSECRET_SETTINGS = { + 'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT', + 'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS', + 'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS', + 'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS', + 'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS', + 'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS', + 'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS', + 'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES', + 'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS', + 'S3_CLEANUP_INTERVAL_SECONDS', +} +SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}' +IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}' + + +def timestamp(): + return datetime.now(timezone.utc).isoformat() + + +def invoke(command): + return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + +def read_command(command): + result = invoke(command) + if result.returncode: + raise RuntimeError('Не выполнена команда: ' + shlex.join(command)) + return result.stdout.strip() + + +def source_manifest(): + names = { + '.dockerignore', '.env.example', + 'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml', + 'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json', + } + for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'): + names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*') + if path.is_file() and '__pycache__' not in path.parts + and path.suffix != '.md' and not path.name.startswith('.')) + return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names)) + + +def sanitize(text, secrets, staging): + clean = text.replace(str(ROOT), '.').replace(str(staging), '') + for secret in secrets: + clean = clean.replace(secret, '') + clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 ', clean) + clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1', clean) + return clean + + +def container_snapshot(images): + ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines() + rows = [] + if ids: + output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids]) + rows = [json.loads(line) for line in output.splitlines()] + for row in rows: + image = row['image_id'] + if image not in images: + images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image])) + return rows + + +def remaining_resources(): + return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter', + f'label=com.docker.compose.project={PROJECT}']).splitlines() + for resource in ('container', 'network', 'volume')} + + +def extract_metrics(output, upload_file): + rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')] + pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output) + if pytest is None: + raise RuntimeError('В выводе нет успешного итогового результата pytest') + return { + 'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])}, + 'uploads': json.loads(upload_file.read_text()), + 'public_failover': [row for row in rows if 'observed_instances_before' in row], + 'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'], + 'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row], + } + + +def main(): + if any(remaining_resources().values()): + raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно') + started = timestamp() + manifest = source_manifest() + secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()] + secrets = [value for value in secrets if value] + snapshots, images, commands, combined, nginx_output = {}, {}, [], [], '' + info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}'])) + config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json'])) + services = {name: { + 'image': service.get('image'), + 'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}), + 'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS}, + 'ports': service.get('ports', []), 'networks': list(service.get('networks', {})), + 'secret_names': [entry['source'] for entry in service.get('secrets', [])], + 'tmpfs': service.get('tmpfs', []), + } for name, service in config['services'].items()} + with TemporaryDirectory(prefix='lab4-evidence-') as folder: + staging = Path(folder) + upload_file = staging / 'uploads.json' + + def execute(command): + begin = timestamp() + result = invoke(command) + output = sanitize(result.stdout, secrets, staging) + display = sanitize(shlex.join(command), secrets, staging) + commands.append({'command': display, 'started_at_utc': begin, + 'finished_at_utc': timestamp(), 'exit_code': result.returncode}) + print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True) + if result.returncode: + print(output[-5000:], flush=True) + raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы') + return f'$ {display}\n{output}\n' + + cleanup_output = '' + try: + combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}'])) + snapshots['started'] = container_snapshot(images) + combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short'])) + combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}'])) + snapshots['before_restart'] = container_snapshot(images) + combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}'])) + snapshots['after_restart'] = container_snapshot(images) + combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}'])) + snapshots['after_failover'] = container_snapshot(images) + combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults'])) + combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)])) + combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080'])) + snapshots['after_update_and_failures'] = container_snapshot(images) + nginx_output = execute(['make', 'test-nginx']) + finally: + cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans']) + remaining = remaining_resources() + if any(remaining.values()): + raise RuntimeError('После очистки остались ресурсы тестового проекта') + if source_manifest() != manifest: + raise RuntimeError('Во время прогона изменились проверяемые исходники') + output = ''.join(combined) + metrics = extract_metrics(output, upload_file) + run = { + 'started_at_utc': started, 'finished_at_utc': timestamp(), + 'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(), + 'base_commit': read_command(['git', 'rev-parse', 'HEAD']), + 'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])), + 'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(), + 'project': PROJECT, 'runtime': {'python': platform.python_version(), + 'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')}, + 'compose': read_command(['docker', 'compose', 'version'])}, + 'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')}, + 'services': services, 'container_snapshots': snapshots, 'images': images, + 'commands': commands, 'remaining_test_resources': remaining, + 'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z', + 'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z', + } + artifacts = { + 'test-output.txt': output, 'nginx-output.txt': nginx_output, + 'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest, + 'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n', + 'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n', + } + for name, content in artifacts.items(): + if any(secret in content for secret in secrets): + raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы') + EVIDENCE.mkdir(exist_ok=True) + for name, content in artifacts.items(): + (EVIDENCE / name).write_text(content) + print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/deployment.py b/scripts/deployment.py new file mode 100644 index 00000000..a2bf38d8 --- /dev/null +++ b/scripts/deployment.py @@ -0,0 +1,107 @@ +"""Общие операции с репликами текущего Compose-проекта.""" + +from contextlib import contextmanager +from dataclasses import dataclass +import fcntl +import json +import os +from pathlib import Path +import ssl +import subprocess +import tempfile +import time +from urllib.error import HTTPError, URLError +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class Replica: + """Состояние контейнера API.""" + + id: str + image: str + instance_id: str + running: bool + health: str | None + + +class ComposeProject: + """Работает только с выбранным Compose-проектом.""" + + def __init__(self) -> None: + self.command = ["docker", "compose", "--env-file", ".env"] + self.config = json.loads(self.run("config", "--format", "json")) + self.name = self.config["name"] + + def docker(self, *args: str, timeout: int = 90, env=None) -> str: + """Выполняет Docker-команду с ограничением времени.""" + result = subprocess.run( + ["docker", *args], cwd=ROOT, text=True, capture_output=True, + timeout=timeout, env={**os.environ, **(env or {})}, + ) + if result.returncode: + raise RuntimeError(result.stderr.strip() or result.stdout.strip()) + return result.stdout.strip() + + def run(self, *args: str, timeout: int = 90, env=None) -> str: + """Выполняет команду Compose с текущими overrides.""" + return self.docker(*self.command[1:], *args, timeout=timeout, env=env) + + def replicas(self) -> list[Replica]: + """Читает состояние API без окружения контейнеров.""" + ids = self.run("ps", "--all", "--quiet", "api").splitlines() + if not ids: + return [] + template = ( + '{"id":{{json .Id}},"image":{{json .Image}},' + '"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},' + '"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}' + ) + rows = self.docker("inspect", "--format", template, *ids).splitlines() + configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip() + return [ + Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"]) + for row in map(json.loads, rows) + ] + + def wait_healthy(self, ids: set[str], timeout: float = 40) -> None: + """Ожидает готовность конкретных реплик.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + current = {replica.id: replica for replica in self.replicas()} + if not ids <= current.keys(): + raise RuntimeError("Контейнер API исчез во время развёртывания") + if all(current[id].running and current[id].health == "healthy" for id in ids): + return + if any(not current[id].running for id in ids): + raise RuntimeError("Контейнер API завершился до готовности") + time.sleep(0.25) + raise TimeoutError("API не стал healthy за отведённое время") + + def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None: + """Ожидает ответа реплики через балансировщик.""" + context = ssl.create_default_context(cafile=ca_file) + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + try: + with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response: + if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id: + return + except (HTTPError, URLError, TimeoutError, OSError): + pass + time.sleep(0.1) + raise TimeoutError("Реплика не появилась в ответах балансировщика") + + +@contextmanager +def deployment_lock(project: ComposeProject): + """Исключает параллельные изменения одного стенда.""" + path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock" + with path.open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None + yield diff --git a/scripts/expiry_demo.sh b/scripts/expiry_demo.sh new file mode 100755 index 00000000..cc32541d --- /dev/null +++ b/scripts/expiry_demo.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +BASE_URL="${BASE_URL:-http://localhost:8080}" +TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}" +API="$BASE_URL/api/v1" +USERNAME="expiry_$(date +%s)" +PASSWORD="expiry-password-123" + +code() { + curl -s -o /dev/null -w '%{http_code}' "$@" +} + +curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \ + -d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}" +SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD") +TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') +EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])') +AUTH="Authorization: Bearer $TOKEN" + +echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT" +echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")" +sleep $((TTL + 1)) +printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")" +echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")" +NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') +echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")" diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..9d571c42 --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,132 @@ +"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных. + +Данные создаются в одной транзакции и откатываются в конце: база не меняется. +""" + +import asyncio +from uuid import uuid4 + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection + +from app.core.config import Settings +from app.core.database import create_db_engine + +TARGET_MESSAGES = 10_000 +NOISE_MESSAGES = 40_000 +PAGE = 51 + +HISTORY_FIRST_PAGE = """ +SELECT * FROM messages +WHERE chat_id = :chat_id +ORDER BY created_at, id +LIMIT :page +""" + +HISTORY_AFTER_CURSOR = """ +SELECT * FROM messages +WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id) +ORDER BY created_at, id +LIMIT :page +""" + + +async def seed(connection: AsyncConnection) -> dict[str, object]: + user_id, target_id, noise_id = uuid4(), uuid4(), uuid4() + await connection.execute( + text( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (:id, :username, 'Explain', 'x')" + ), + {"id": user_id, "username": "explain_" + uuid4().hex[:8]}, + ) + await connection.execute( + text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"), + {"a": target_id, "b": noise_id}, + ) + await connection.execute( + text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"), + {"a": target_id, "b": noise_id, "u": user_id}, + ) + for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)): + await connection.execute( + text( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, " + "now() - make_interval(secs => :count - g) " + "FROM generate_series(1, :count) AS g" + ), + {"chat_id": chat_id, "user_id": user_id, "count": count}, + ) + await connection.execute(text("ANALYZE messages")) + + middle = ( + await connection.execute( + text( + "SELECT created_at, id FROM messages WHERE chat_id = :c " + "ORDER BY created_at, id OFFSET :o LIMIT 1" + ), + {"c": target_id, "o": TARGET_MESSAGES // 2}, + ) + ).one() + return { + "chat_id": target_id, + "after_created_at": middle.created_at, + "after_id": middle.id, + "page": PAGE, + } + + +async def explain( + connection: AsyncConnection, title: str, query: str, params: dict[str, object] +) -> None: + rows = await connection.execute( + text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params + ) + print(f"== {title}") + print(query.strip()) + print() + for row in rows: + print(row[0]) + print() + + +async def main() -> None: + engine = create_db_engine(Settings()) # type: ignore[call-arg] + async with engine.connect() as connection: + transaction = await connection.begin() + params = await seed(connection) + total = ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() + print( + f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}" + ) + print() + + first_page = {"chat_id": params["chat_id"], "page": PAGE} + await explain( + connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page + ) + await explain( + connection, + "page after cursor in the middle of history", + HISTORY_AFTER_CURSOR, + params, + ) + + await connection.execute(text("SET LOCAL enable_indexscan = off")) + await connection.execute(text("SET LOCAL enable_bitmapscan = off")) + await explain( + connection, + "same page with index scans disabled (for comparison)", + HISTORY_AFTER_CURSOR, + params, + ) + + await transaction.rollback() + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/failover_all.py b/scripts/failover_all.py new file mode 100644 index 00000000..79745d7e --- /dev/null +++ b/scripts/failover_all.py @@ -0,0 +1,148 @@ +"""Сценарий поочерёдного отказа всех API-реплик под записью.""" + +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import json +import os +import re +import subprocess +from threading import Event, Lock +import time +from urllib.parse import quote +from uuid import uuid4 + +from scripts.deployment import ComposeProject, ROOT, deployment_lock +from tests.client import Client, Failure, arguments, check, run + + +def failure_kind(error: Failure) -> str: + """Различает HTTP-отказы, ошибки соединения и проверки контракта.""" + match = re.search(r"received (\d{3})$", str(error)) + if match: + return "http_" + match.group(1) + return "connection" if "connection/TLS failure" in str(error) else "contract" + + +def snapshot(project: ComposeProject, phase: str, target: str) -> list: + """Печатает состояние API без окружения и секретов.""" + replicas = project.replicas() + print(json.dumps({ + "scenario": "replica_failure", "phase": phase, "target": target, + "replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas], + }), flush=True) + return replicas + + +def history(client: Client, actor: dict, chat: dict) -> list[dict]: + """Читает всю историю без повторов курсора.""" + path = f"/api/v1/chats/{chat['id']}/messages?limit=100" + items = [] + cursors = set() + for _ in range(1000): + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + check(cursor not in cursors, "cursor cycle") + cursors.add(cursor) + path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}" + raise Failure("history exceeded safety bound") + + +def exercise(project: ComposeProject, args) -> None: + """Останавливает каждую реплику и восстанавливает её перед следующей.""" + replicas = project.replicas() + check(len(replicas) >= 2, "at least two API replicas are required") + project.wait_healthy({item.id for item in replicas}) + client = Client(args.base_url, args.ca_file, timeout=2) + for item in replicas: + project.wait_routed(item.instance_id, client.base, args.ca_file) + actor, _, _, chat = client.bootstrap() + confirmed = set() + + for target in replicas: + snapshot(project, "before", target.instance_id) + stop = Event() + lock = Lock() + successes = [] + errors = [] + + def worker(): + local = Client(client.base, args.ca_file, timeout=2) + key = uuid4().hex + while not stop.is_set(): + started = time.monotonic() + try: + result = local.message(actor, chat, "failover-" + key, key=key) + with lock: + successes.append((started, time.monotonic(), result["json"]["id"])) + key = uuid4().hex + except Failure as error: + with lock: + errors.append(failure_kind(error)) + time.sleep(0.1) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + jobs = [pool.submit(worker) for _ in range(args.concurrency)] + try: + time.sleep(2) + stop_started = time.monotonic() + subprocess.run( + [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id}, + check=True, timeout=10, capture_output=True, + ) + stop_finished = time.monotonic() + stopped = snapshot(project, "stopped", target.instance_id) + check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped") + check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API") + time.sleep(args.duration) + finally: + stop.set() + try: + for job in jobs: + job.result(timeout=5) + finally: + project.docker("start", target.id) + project.wait_healthy({item.id for item in replicas}) + project.wait_routed(target.instance_id, client.base, args.ca_file) + snapshot(project, "restored", target.instance_id) + + elapsed = max(end for _, end, _ in successes) - started if successes else 0 + recovered = [end for begin, end, _ in successes if begin >= stop_finished] + check(bool(recovered), "no confirmed writes after failure") + recovery = min(recovered) - stop_started + check(recovery <= 10, "writes did not recover within 10 seconds") + confirmed.update(id for _, _, id in successes) + items = history(client, actor, chat) + check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost") + texts = [item["text"] for item in items] + check(len(texts) == len(set(texts)), "uncertain retry created duplicates") + print(json.dumps({ + "instance": target.instance_id, + "successful_writes": len(successes), + "failed_attempts": len(errors), + "errors_by_kind": dict(Counter(errors)), + "elapsed_seconds": round(elapsed, 3), + "successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0, + "first_recovery_seconds": round(recovery, 3), + }), flush=True) + + +def main() -> None: + """Готовит параметры эксперимента отказа реплик.""" + def configure(parser): + parser.add_argument("--duration", type=float, default=15) + parser.add_argument("--concurrency", type=int, default=2) + + args = arguments("Fail each API replica under write load", configure) + check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload") + project = ComposeProject() + with deployment_lock(project): + exercise(project, args) + + +if __name__ == "__main__": + run(main) diff --git a/scripts/failure_demo.sh b/scripts/failure_demo.sh new file mode 100755 index 00000000..b5c7332d --- /dev/null +++ b/scripts/failure_demo.sh @@ -0,0 +1,64 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +BASE_URL="${BASE_URL:-http://localhost:8080}" +API="$BASE_URL/api/v1" +USERNAME="demo_$(date +%s)" +PASSWORD="demo-password-123" + +compose() { + docker compose --env-file .env "$@" +} + +code() { + curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@" +} + +wait_healthy() { + attempts=0 + until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; } + sleep 1 + done +} + +curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \ + -d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}" +TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') +AUTH="Authorization: Bearer $TOKEN" + +echo "== baseline" +echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" + +echo +echo "== redis stopped" +compose stop redis >/dev/null 2>&1 +echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" +echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")" +echo "GET /users/me (no token) -> $(code "$API/users/me")" +echo "GET /health/ready -> $(code "$BASE_URL/health/ready")" +echo "GET /health/live -> $(code "$BASE_URL/health/live")" +compose start redis >/dev/null 2>&1 +wait_healthy redis +echo "-- redis started again (AOF restored)" +echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" + +echo +echo "== postgres stopped" +compose stop postgres >/dev/null 2>&1 +echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" +echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")" +echo "GET /health/ready -> $(code "$BASE_URL/health/ready")" +echo "GET /health/live -> $(code "$BASE_URL/health/live")" +compose start postgres >/dev/null 2>&1 +wait_healthy postgres +echo "-- postgres started again" +echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" + +echo +echo "== logout" +echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")" +echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" diff --git a/scripts/init_local.py b/scripts/init_local.py new file mode 100644 index 00000000..baa14022 --- /dev/null +++ b/scripts/init_local.py @@ -0,0 +1,30 @@ +"""Создаёт только отсутствующие локальные настройки и файловые секреты.""" + +import os +from pathlib import Path +import secrets + + +def main() -> None: + root = Path(__file__).resolve().parents[1] + destination = root / ".env" + if not destination.exists(): + with destination.open("x") as target: + target.write((root / ".env.example").read_text()) + destination.chmod(0o600) + directory = root / "secrets" + directory.mkdir(mode=0o700, exist_ok=True) + for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): + path = directory / name + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) + except FileExistsError: + continue + with os.fdopen(descriptor, "w") as target: + target.write(secrets.token_urlsafe(24)) + print("Создан файл " + str(path.relative_to(root))) + print("Настройки готовы; существующие значения сохранены") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..1d313c98 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +compose() { + docker compose --env-file .env "$@" +} + +wait_healthy() { + for service in "$@"; do + attempts=0 + until states=$(compose ps --format '{{.Health}}' "$service") \ + && [ -n "$states" ] \ + && [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 60 ]; then + echo "$service не стал healthy за 60 секунд" >&2 + exit 1 + fi + sleep 1 + done + done +} + +compose stop cleanup +compose restart postgres redis minio +wait_healthy postgres redis minio +compose restart api +wait_healthy api +compose restart nginx +wait_healthy nginx +compose start cleanup diff --git a/scripts/rolling_update.py b/scripts/rolling_update.py new file mode 100644 index 00000000..20aebda8 --- /dev/null +++ b/scripts/rolling_update.py @@ -0,0 +1,92 @@ +"""Поочерёдно заменяет API с одной дополнительной репликой.""" + +import argparse +import sys + +from scripts.deployment import ComposeProject, deployment_lock + + +def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None: + """Собирает образ, применяет миграции и заменяет реплики.""" + original = project.replicas() + if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original): + raise RuntimeError("До обновления нужны минимум две здоровые API-реплики") + old_images = {item.image for item in original} + if len(old_images) != 1: + raise RuntimeError("Сначала восстановите единый образ работающих реплик") + old_image = old_images.pop() + profile = project.docker( + "image", "inspect", old_image, + "--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}', + ) + if profile != "file-v1": + raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up") + count = len(original) + print("Сборка нового образа API", flush=True) + build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else () + project.run("build", *build_args, "api", timeout=300) + tag = project.config["services"]["api"]["image"] + new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}") + print("Однократное применение совместимых миграций", flush=True) + project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate") + if old_image == new_image: + print("Образ не изменился; замена контейнеров не требуется") + return + + try: + for old in original: + project.run( + "up", "-d", "--no-deps", "--no-build", "--pull", "never", + "--no-recreate", "--scale", f"api={count + 1}", "api", + ) + current = project.replicas() + additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image] + if not additions: + raise RuntimeError("Новая реплика не создана") + project.wait_healthy({item.id for item in additions}) + for added in additions: + project.wait_routed(added.instance_id, base_url, ca_file) + survivors = {item.id for item in project.replicas() if item.id != old.id} + project.wait_healthy(survivors) + project.docker("stop", old.id) + project.docker("rm", old.id) + print(f"Заменена реплика {old.instance_id}", flush=True) + final = project.replicas() + if len(final) != count or any(item.image != new_image for item in final): + raise RuntimeError("Состав реплик после обновления не соответствует плану") + project.wait_healthy({item.id for item in final}) + except Exception: + print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr) + project.run( + "up", "-d", "--no-deps", "--no-build", "--pull", "never", + "--force-recreate", "--scale", f"api={count}", "api", + env={"API_IMAGE": old_image}, + ) + project.wait_healthy({item.id for item in project.replicas()}) + raise + + dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines() + if old_image in dangling: + references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}") + if not references: + try: + project.docker("image", "rm", old_image) + except RuntimeError: + print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr) + print(f"Готово: {count} здоровых реплик нового образа") + + +def main() -> None: + """Запускает обновление с блокировкой проекта.""" + parser = argparse.ArgumentParser(description="Rolling update of API replicas") + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--ca-file", default=None) + parser.add_argument("--revision", default=None) + args = parser.parse_args() + project = ComposeProject() + with deployment_lock(project): + update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision) + + +if __name__ == "__main__": + main() diff --git a/scripts/s3_faults.py b/scripts/s3_faults.py new file mode 100644 index 00000000..07a4bc25 --- /dev/null +++ b/scripts/s3_faults.py @@ -0,0 +1,108 @@ +"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы.""" + +import asyncio +import base64 +from datetime import datetime, timedelta, timezone +import json +import sys +from uuid import UUID + +from minio.error import S3Error +from sqlalchemy import select +from sqlalchemy.exc import OperationalError + +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.domain import attachments as domain +from app.errors.exceptions import unavailable +from app.models.attachment import UploadIntent +from app.schemas.user import User +from app.storage import attachments as repository +from app.storage.s3 import create_s3_storage + + +async def object_exists(storage, key): + exists = True + try: + await storage.stat_object(key) + except S3Error as error: + if error.code not in {'NoSuchKey', 'NoSuchObject'}: + raise + exists = False + return exists + + +async def main(): + payload = json.load(sys.stdin) + settings = Settings() + engine = create_db_engine(settings) + factory = create_session_factory(engine) + storage = create_s3_storage(settings.s3) + user = User.model_validate(payload['user']) + chat_id = UUID(payload['chat_id']) + original = base64.b64decode(payload['image']) + keys = [] + create_record = repository.create_attachment + put_file, delete_object = storage.put_file, storage.delete_object + + async def capture_put(key, path, **metadata): + keys.append(key) + await put_file(key, path, **metadata) + + async def fail_metadata(*args, **kwargs): + raise OperationalError('injected metadata failure after real S3 PUT', {}, None) + + async def fail_delete(*args): + raise unavailable('injected compensation failure') + + async def chunks(): + yield original + + async def attempt(): + raised = False + async with factory() as session: + try: + await domain.upload_attachment(session, storage, user, chat_id, + filename='fault.png', content_type='image/png', chunks=chunks()) + except OperationalError: + raised = True + assert raised, 'injected database failure was not raised' + + storage.put_file = capture_put + repository.create_attachment = fail_metadata + try: + await attempt() + assert not await object_exists(storage, keys[-1]), 'compensation left original' + async with factory() as session: + intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1])) + assert intent is None, 'compensated upload left intent' + storage.delete_object = fail_delete + await attempt() + key = keys[-1] + assert await object_exists(storage, key), 'fault did not leave real S3 object' + storage.delete_object = delete_object + async with factory() as session: + result = await domain.reconcile_attachments(session, storage, dry_run=False) + assert result.deleted == 0, 'fresh object deleted before grace period' + intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key)) + assert intent is not None, 'durable intent missing' + intent.created_at = datetime.now(timezone.utc) - timedelta(days=2) + await session.commit() + dry_run = await domain.reconcile_attachments(session, storage) + assert dry_run.candidates == 1 and dry_run.deleted == 0 + assert await object_exists(storage, key), 'dry-run removed original' + applied = await domain.reconcile_attachments(session, storage, dry_run=False) + assert applied.deleted == 1 and not await object_exists(storage, key) + repeated = await domain.reconcile_attachments(session, storage, dry_run=False) + assert repeated.examined == repeated.deleted == 0 + print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True, + 'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True})) + finally: + repository.create_attachment = create_record + storage.put_file, storage.delete_object = put_file, delete_object + await storage.aclose() + await engine.dispose() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..0dcd4da0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +case "$TARGET_INSTANCE_ID" in + *[!a-zA-Z0-9_.:-]*|'') exit 1 ;; +esac + +target="" +for container in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \ + | sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then + if [ -n "$target" ]; then + echo 'Instance id matches multiple API containers' >&2 + exit 1 + fi + target="$container" + fi +done + +if [ -z "$target" ]; then + echo 'Target API instance not found in this Compose project' >&2 + exit 1 +fi + +echo "Stopping API container $target" >&2 +docker stop --timeout 8 "$target" >/dev/null diff --git a/scripts/upload_load.py b/scripts/upload_load.py new file mode 100644 index 00000000..91445202 --- /dev/null +++ b/scripts/upload_load.py @@ -0,0 +1,113 @@ +"""Повторяемые параллельные upload и память cgroup API, без записи токенов.""" + +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import hashlib +from io import BytesIO +import json +from pathlib import Path +import random +import subprocess +from threading import Barrier +import time + +from PIL import Image + +from scripts.deployment import ComposeProject +from tests.client import Client, Failure, arguments, check, run + +MONITOR = r''' +import json, pathlib, sys, threading +root = pathlib.Path('/sys/fs/cgroup') +def current(): + return int((root / 'memory.current').read_text()) +def peak(): + path = root / 'memory.peak' + return int(path.read_text()) if path.exists() else None +stop = threading.Event() +threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start() +initial, before = current(), peak() +sampled = initial +print('ready', flush=True) +while not stop.wait(0.02): + sampled = max(sampled, current()) +print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled, + 'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True) +''' + + +def main(): + def configure(parser): + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--rounds', type=int, default=3) + parser.add_argument('--output', type=Path) + args = arguments('Concurrent attachment uploads and API memory', configure) + check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10') + project = ComposeProject() + client = Client(args.base_url, args.ca_file, timeout=65) + alice, _, _, chat = client.bootstrap() + random_bytes = random.Random(42).randbytes(1500 * 1500 * 3) + image = Image.frombytes('RGB', (1500, 1500), random_bytes) + buffer = BytesIO() + image.save(buffer, format='PNG') + data = buffer.getvalue() + check(len(data) <= 10485760, 'fixture exceeds upload limit') + monitors = [] + measurements = {} + rows = [] + started = time.monotonic() + try: + for replica in project.replicas(): + process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + monitors.append((replica.instance_id, process)) + check(process.stdout.readline().strip() == 'ready', 'memory monitor failed') + check(bool(monitors), 'API not running') + for _ in range(args.rounds): + barrier = Barrier(args.concurrency) + def upload(index): + local = Client(args.base_url, args.ca_file, timeout=65) + barrier.wait(timeout=15) + request_started = time.monotonic() + try: + return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png", + expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data, + headers={'Content-Type': 'image/png'}, schema='Attachment') + except Failure: + # При раннем отказе сервер может закрыть ещё отправляемое тело. + return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started} + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + rows.extend(pool.map(upload, range(args.concurrency))) + finally: + elapsed = time.monotonic() - started + for alias, process in monitors: + output, _ = process.communicate(input='', timeout=10) + check(process.returncode == 0, 'memory monitor failed') + measurements[alias] = json.loads(output) + succeeded = [row for row in rows if row['status'] == 201] + check(bool(succeeded), 'all uploads were rejected') + digest = hashlib.sha256(data).hexdigest() + for row in succeeded: + check(row['json']['sha256'] == digest, 'upload checksum differs') + original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes'] + check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs') + latencies = sorted(row['elapsed'] for row in rows) + result = { + 'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows), + 'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest}, + 'statuses': dict(Counter(str(row['status']) for row in rows)), + 'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded), + 'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3), + 'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'], + 'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries', + } + encoded = json.dumps(result, ensure_ascii=False, indent=2) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(encoded + '\n') + print(encoded) + check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..ca5063f1 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,202 @@ +"""Фикстуры async-тестов на реальных PostgreSQL и Redis. + +Перед запуском: `make test-stand-up` (тестовый стенд из compose.yaml + compose.test.yaml +и база _test). +Переменные: + TEST_DATABASE_URL по умолчанию postgresql+asyncpg://messenger:messenger@localhost:5432/messenger_test + TEST_USE_MIGRATIONS=1 строить схему через `alembic upgrade head` вместо Base.metadata.create_all + TEST_REDIS_HOST/PORT/PASSWORD/DB по умолчанию localhost:6379, база 15 + DOCKER_TESTS=1 включить тесты, которые перезапускают контейнеры compose +""" + +import asyncio +import os +from collections.abc import AsyncIterator, Callable + +import httpx +import pytest +from alembic import command +from alembic.config import Config +from fastapi import FastAPI +from redis.asyncio import Redis +from redis.exceptions import RedisError +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool + +from app.core.config import Settings +from app.core.redis import create_redis_client +from app.main import create_app +from app.models import Base + +DEFAULT_TEST_DATABASE_URL = ( + "postgresql+asyncpg://messenger:messenger@localhost:5432/messenger_test" +) + + +def _database_url() -> str: + return os.environ.get("TEST_DATABASE_URL", DEFAULT_TEST_DATABASE_URL) + + +def _redis_settings() -> dict[str, object]: + return { + "redis_host": os.environ.get("TEST_REDIS_HOST", "localhost"), + "redis_port": int(os.environ.get("TEST_REDIS_PORT", "6379")), + "redis_password": os.environ.get("TEST_REDIS_PASSWORD") or None, + "redis_db": int(os.environ.get("TEST_REDIS_DB", "15")), + } + + +def _alembic_config(url: str) -> Config: + config = Config("alembic.ini") + config.set_main_option("sqlalchemy.url", url.replace("%", "%%")) + return config + + +async def upgrade_head(url: str) -> None: + """alembic command.upgrade внутри вызывает asyncio.run, поэтому — в отдельном потоке.""" + await asyncio.to_thread(command.upgrade, _alembic_config(url), "head") + + +async def reset_schema(engine: AsyncEngine) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.drop_all) + await connection.execute(text("DROP TABLE IF EXISTS alembic_version")) + + +async def close_app(app: FastAPI) -> None: + """ASGITransport не запускает lifespan, поэтому ресурсы приложения закрываются явно.""" + await app.state.redis.aclose() + await app.state.engine.dispose() + + +@pytest.fixture(scope="session") +def database_url() -> str: + return _database_url() + + +@pytest.fixture(scope="session") +async def db_engine(database_url: str) -> AsyncIterator[AsyncEngine]: + engine = create_async_engine(database_url, poolclass=NullPool) + try: + async with engine.connect() as connection: + await connection.execute(text("SELECT 1")) + except (SQLAlchemyError, OSError) as error: + pytest.exit( + f"PostgreSQL недоступна по {engine.url.render_as_string(hide_password=True)}: " + f"{type(error).__name__}. Выполните `make test-stand-up`.", + returncode=2, + ) + + await reset_schema(engine) + + if os.environ.get("TEST_USE_MIGRATIONS") == "1": + await upgrade_head(database_url) + else: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + yield engine + await engine.dispose() + + +@pytest.fixture(scope="session") +async def redis_client() -> AsyncIterator[Redis]: + settings = build_settings(_database_url()) + if settings.redis_db == 0: + pytest.exit("TEST_REDIS_DB=0 запрещён: тесты очищают базу Redis.", returncode=2) + + client = create_redis_client(settings) + try: + await client.ping() + except (RedisError, OSError) as error: + pytest.exit( + f"Redis недоступен: {type(error).__name__}. Выполните `make test-stand-up`.", + returncode=2, + ) + + yield client + await client.flushdb() + await client.aclose() + + +@pytest.fixture(autouse=True) +async def clean_db(db_engine: AsyncEngine, redis_client: Redis) -> None: + names = ", ".join(f'"{table.name}"' for table in Base.metadata.sorted_tables) + if names: + async with db_engine.begin() as connection: + await connection.execute(text(f"TRUNCATE {names} RESTART IDENTITY CASCADE")) + await redis_client.flushdb() + + +def build_settings(database_url: str, **overrides: object) -> Settings: + values: dict[str, object] = { + # Регрессия lab3 проверяет PostgreSQL/Redis; MinIO подменяется отдельно. + "s3_enabled": False, + "app_host": "127.0.0.1", + "app_port": 8080, + "app_env": "test", + "max_messages": 1000, + "max_in_flight": 32, + "log_level": "CRITICAL", + "database_url": database_url, + "session_ttl_seconds": 300, + "db_pool_size": 10, + "db_max_overflow": 10, + **_redis_settings(), + } + values.update(overrides) + return Settings(_env_file=None, **values) # type: ignore[arg-type] + + +@pytest.fixture +def settings(database_url: str) -> Settings: + return build_settings(database_url) + + +@pytest.fixture +def make_settings(database_url: str) -> Callable[..., Settings]: + def factory(**overrides: object) -> Settings: + url = str(overrides.pop("database_url", database_url)) + return build_settings(url, **overrides) + + return factory + + +def make_client( + app, *, raise_app_exceptions: bool = True, timeout: float = 30 +) -> httpx.AsyncClient: + """ASGITransport не запускает lifespan, поэтому ресурсы закрывает вызывающий (close_app).""" + return httpx.AsyncClient( + transport=httpx.ASGITransport( + app=app, raise_app_exceptions=raise_app_exceptions + ), + base_url="http://testserver", + timeout=timeout, + ) + + +@pytest.fixture +def app(settings: Settings): + application = create_app(settings) + + @application.get("/__test__/explode", include_in_schema=False) + async def explode() -> None: + raise RuntimeError("test-secret") + + return application + + +@pytest.fixture +async def client(app) -> AsyncIterator[httpx.AsyncClient]: + async with make_client(app) as test_client: + yield test_client + await close_app(app) + + +@pytest.fixture +async def client_no_raise(app) -> AsyncIterator[httpx.AsyncClient]: + async with make_client(app, raise_app_exceptions=False) as test_client: + yield test_client + await close_app(app) diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..97bf8f99 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,88 @@ +import base64 +from uuid import uuid4 + +from httpx import AsyncClient + +PASSWORD = "Correct-horse-battery-1" + + +def basic_header(username: str, password: str) -> dict[str, str]: + raw = f"{username}:{password}".encode() + return {"Authorization": "Basic " + base64.b64encode(raw).decode()} + + +def bearer(actor: dict) -> dict[str, str]: + return {"Authorization": "Bearer " + actor["token"]} + + +async def register( + client: AsyncClient, + username: str | None = None, + display_name: str = "Тестовый пользователь", + password: str = PASSWORD, +) -> dict: + """Регистрация без входа. Возвращает User + password (нужен для login).""" + actual_username = username or "user_" + uuid4().hex[:12] + response = await client.post( + "/api/v1/users", + json={ + "username": actual_username, + "display_name": display_name, + "password": password, + }, + ) + assert response.status_code == 201, response.text + return {**response.json(), "password": password} + + +async def login(client: AsyncClient, user: dict) -> dict: + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + assert response.status_code == 201, response.text + return {**user, "token": response.json()["token"]} + + +async def create_user(client: AsyncClient, **kwargs) -> dict: + """Регистрация + вход: актор с полями id, username, token, ...""" + return await login(client, await register(client, **kwargs)) + + +async def create_chat( + client: AsyncClient, + user: dict, + *members: dict, + title: str = "Тестовый чат", +) -> dict: + member_ids = [user["id"], *(member["id"] for member in members)] + response = await client.post( + "/api/v1/chats", + headers=bearer(user), + json={"title": title, "member_ids": member_ids}, + ) + assert response.status_code == 201, response.text + return response.json() + + +def idempotency_header(key: str | None = None) -> dict[str, str]: + return {"Idempotency-Key": key or uuid4().hex} + + +async def send_message( + client: AsyncClient, user: dict, chat: dict, text: str, key: str | None = None +): + """POST сообщения требует Idempotency-Key; по умолчанию — новый ключ.""" + return await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header(key)}, + json={"text": text}, + ) + + +def assert_request_id(response) -> str: + header_id = response.headers.get("X-Request-Id") + assert header_id + if response.status_code >= 400: + assert response.json()["error"]["request_id"] == header_id + return header_id diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..5661d148 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,365 @@ +"""Lab4: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" + +import asyncio +import hashlib +from io import BytesIO +from datetime import datetime, timedelta, timezone +from pathlib import Path +import struct +from uuid import UUID, uuid4 +import zlib + +import pytest +from PIL import Image +from sqlalchemy import func, select +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.requests import ClientDisconnect + +from app.core.s3 import S3Settings +from app.domain import attachments as domain +from app.errors.exceptions import unavailable +from app.models.attachment import Attachment, UploadIntent +from app.storage import attachments as repository +from app.storage.s3 import ObjectInfo +from helpers import bearer, create_chat, create_user, idempotency_header + +PNG = Path('tests/fixtures/sample.png').read_bytes() + + +class MemoryStorage: + """Только тестовый адаптер; production использует MinioStorage.""" + + def __init__(self): + self.settings = S3Settings(_env_file=None) + self.objects = {} + self.fail_put = self.fail_delete = self.down = False + + async def put_file(self, key, path, **metadata): + self.objects[key] = (path.read_bytes(), metadata) + if self.fail_put: + raise unavailable() + + async def delete_object(self, key): + if self.fail_delete: + raise unavailable() + self.objects.pop(key, None) + + async def stat_object(self, key): + if self.down: + raise unavailable() + info = None + if key in self.objects: + data, metadata = self.objects[key] + info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc)) + return info + + async def open_object(self, key): + async def chunks(): + yield self.objects[key][0] + return chunks() + + async def check_available(self): + return not self.down + + +@pytest.fixture +async def context(client, app): + storage = MemoryStorage() + app.state.s3 = storage + alice, bob, eve = [await create_user(client) for _ in range(3)] + chat = await create_chat(client, alice, bob) + return storage, alice, bob, eve, chat + + +async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'): + return await client.post( + f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename}, + headers={**bearer(actor), 'Content-Type': content_type}, content=data, + ) + + +async def test_upload_download_and_attachment_only_message(client, context): + storage, alice, bob, eve, chat = context + response = await upload(client, alice, chat) + assert response.status_code == 201, response.text + attachment = response.json() + assert attachment['status'] == 'ready' + assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest() + assert attachment['variants'] == [] and attachment['error_code'] is None + assert 'object_key' not in attachment + id = attachment['id'] + for actor in (alice, bob): + content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor)) + assert content.status_code == 200 and content.content == PNG + assert content.headers['content-type'] == 'image/png' + key = idempotency_header() + body = {'text': '', 'attachment_ids': [id]} + path = f"/api/v1/chats/{chat['id']}/messages" + message = await client.post(path, json=body, headers={**bearer(bob), **key}) + assert message.status_code == 201, message.text + repeat = await client.post(path, json=body, headers={**bearer(bob), **key}) + assert repeat.json() == message.json() + history = await client.get(path, headers=bearer(alice)) + assert history.json()['items'][0]['attachment_ids'] == [id] + for suffix in ('', '/content'): + denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve)) + assert denied.status_code == 404 + + +@pytest.mark.parametrize( + 'data,content_type', + [(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')], + ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'], +) +async def test_invalid_image_and_mime_are_415(client, context, data, content_type): + storage, alice, _, _, chat = context + response = await upload(client, alice, chat, data, content_type=content_type) + assert response.status_code == 415, response.text + assert not storage.objects + + +async def test_jpeg_is_stored_without_reencoding(client, context): + _, alice, _, _, chat = context + output = BytesIO() + Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG') + original = output.getvalue() + response = await upload(client, alice, chat, original, content_type='image/jpeg') + assert response.status_code == 201, response.text + id = response.json()['id'] + downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + assert downloaded.content == original + + +@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256]) +async def test_unsafe_filename_is_400(client, context, filename): + storage, alice, _, _, chat = context + response = await upload(client, alice, chat, filename=filename) + assert response.status_code == 400 + assert not storage.objects + + +async def test_oversize_chunked_upload_is_413(client, context): + storage, alice, _, _, chat = context + async def chunks(): + for _ in range(11): + yield b'x' * 1048576 + response = await upload(client, alice, chat, chunks()) + assert response.status_code == 413 + assert not storage.objects + + +async def test_oversize_content_length_is_413(client, context): + _, alice, _, _, chat = context + response = await client.post( + f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", + headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG, + ) + assert response.status_code == 413 + + +async def test_exact_byte_limit_accepts_valid_png(client, context): + _, alice, _, _, chat = context + maximum = 10_485_760 + # Валидный ancillary chunk вместо данных после IEND. + payload = bytes(maximum - len(PNG) - 12) + kind = b'npAD' + padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload)) + image = PNG[:-12] + padding + PNG[-12:] + response = await upload(client, alice, chat, image) + assert response.status_code == 201, response.text + assert response.json()['size_bytes'] == maximum + assert response.json()['sha256'] == hashlib.sha256(image).hexdigest() + + +async def test_exact_pixel_limit_accepts_valid_png(client, context): + _, alice, _, _, chat = context + output = BytesIO() + with Image.new('RGB', (8000, 5000), 'blue') as image: + image.save(output, format='PNG') + response = await upload(client, alice, chat, output.getvalue()) + assert response.status_code == 201, response.text + + +async def test_pixel_limit_before_decoding(client, context): + _, alice, _, _, chat = context + def chunk(kind, data): + return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data)) + header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0) + image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'') + response = await upload(client, alice, chat, image) + assert response.status_code == 413, response.text + + +async def test_unknown_and_other_chat_attachment_are_404(client, context): + _, alice, bob, _, chat = context + attachment = (await upload(client, alice, chat)).json() + other_chat = await create_chat(client, alice, bob) + for id in (attachment['id'], str(uuid4())): + response = await client.post( + f"/api/v1/chats/{other_chat['id']}/messages", + json={'text': '', 'attachment_ids': [id]}, + headers={**bearer(alice), **idempotency_header()}, + ) + assert response.status_code == 404 + + +async def test_duplicate_ids_are_400(client, context): + _, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()['id'] + response = await client.post(f"/api/v1/chats/{chat['id']}/messages", + json={'text': '', 'attachment_ids': [id, id]}, + headers={**bearer(alice), **idempotency_header()}) + assert response.status_code == 400 + + +async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): + _, alice, _, _, chat = context + headers = {**bearer(alice), **idempotency_header()} + path = f"/api/v1/chats/{chat['id']}/messages" + first = await client.post(path, json={'text': 'same'}, headers=headers) + second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers) + assert first.status_code == second.status_code == 201 + assert first.json() == second.json() + + +async def test_s3_outage_is_503(client, context): + storage, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()['id'] + storage.down = True + response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + assert response.status_code == 503 + assert (await client.get('/health/ready')).status_code == 503 + + +async def test_uncertain_put_is_compensated(client, context, db_engine): + storage, alice, _, _, chat = context + storage.fail_put = True + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert not storage.objects + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_failure_after_s3_write_compensates(client, context, monkeypatch): + storage, alice, _, _, chat = context + async def fail(*args, **kwargs): + raise OperationalError('injected', {}, None) + monkeypatch.setattr(repository, 'create_attachment', fail) + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert not storage.objects + + +async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch): + storage, alice, _, _, chat = context + original = AsyncSession.commit + count = 0 + async def uncertain(session): + nonlocal count + count += 1 + await original(session) + if count == 2: + raise OperationalError('lost commit acknowledgement', {}, None) + monkeypatch.setattr(AsyncSession, 'commit', uncertain) + response = await upload(client, alice, chat) + assert response.status_code == 503 + assert len(storage.objects) == 1 + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(Attachment)) == 1 + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine): + storage, alice, _, _, chat = context + storage.fail_put = storage.fail_delete = True + assert (await upload(client, alice, chat)).status_code == 503 + assert len(storage.objects) == 1 + async with AsyncSession(db_engine) as session: + intent = await session.scalar(select(UploadIntent)) + intent.created_at = datetime.now(timezone.utc) - timedelta(days=2) + await session.commit() + storage.fail_put = storage.fail_delete = False + async with AsyncSession(db_engine) as session: + dry_run = await domain.reconcile_attachments(session, storage) + assert dry_run.candidates == 1 and dry_run.deleted == 0 + assert storage.objects + applied = await domain.reconcile_attachments(session, storage, dry_run=False) + assert applied.deleted == 1 and not storage.objects + repeated = await domain.reconcile_attachments(session, storage, dry_run=False) + assert repeated.deleted == 0 + + +async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine): + storage, _, _, _, _ = context + old_id, fresh_id = uuid4(), uuid4() + async with AsyncSession(db_engine) as session: + session.add_all([ + UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)), + UploadIntent(id=fresh_id, object_key='originals/fresh'), + ]) + await session.commit() + async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner: + await repository.lock_upload(uploader, old_id) + result = await domain.reconcile_attachments(cleaner, storage, dry_run=False) + assert result.examined == 0 + await uploader.rollback() + result = await domain.reconcile_attachments(cleaner, storage, dry_run=False) + assert result.examined == 1 + assert await cleaner.get(UploadIntent, fresh_id) is not None + + +async def test_client_disconnect_before_put_has_no_side_effects(client, context): + storage, alice, _, _, chat = context + async def interrupted(): + yield PNG[:16] + raise ClientDisconnect() + response = await upload(client, alice, chat, interrupted()) + assert response.status_code == 400 + assert not storage.objects + + +async def test_upload_deadline_cancels_validation(client, context, monkeypatch): + storage, alice, _, _, chat = context + storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02}) + async def stalled(*args): + await asyncio.sleep(10) + monkeypatch.setattr(domain, 'validate_image', stalled) + response = await upload(client, alice, chat) + assert response.status_code == 400 + assert not storage.objects + + +async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch): + storage, alice, _, _, chat = context + entered = asyncio.Event() + original = storage.put_file + async def interrupted(*args, **kwargs): + await original(*args, **kwargs) + entered.set() + await asyncio.sleep(10) + monkeypatch.setattr(storage, 'put_file', interrupted) + task = asyncio.create_task(upload(client, alice, chat)) + await asyncio.wait_for(entered.wait(), timeout=10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert not storage.objects + async with AsyncSession(db_engine) as session: + assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0 + + +async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): + storage, alice, _, _, chat = context + id = (await upload(client, alice, chat)).json()['id'] + async with AsyncSession(db_engine) as session: + attachment = await session.get(Attachment, UUID(id)) + key = attachment.object_key + # Искусственно оставляем старый журнал у уже подтверждённого объекта. + session.add(UploadIntent(id=uuid4(), object_key=key, + created_at=datetime.now(timezone.utc) - timedelta(days=2))) + await session.commit() + result = await domain.reconcile_attachments(session, storage, dry_run=False) + assert result.examined == 1 and result.deleted == 0 + assert key in storage.objects diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..6746320f --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,298 @@ +import asyncio +import hashlib +from datetime import datetime + +from httpx import AsyncClient + +from app.main import create_app +from conftest import close_app, make_client +from helpers import ( + PASSWORD, + assert_request_id, + basic_header, + bearer, + create_user, + login, + register, +) + + +async def test_login_returns_opaque_bearer_session(client: AsyncClient): + user = await register(client) + + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + + assert response.status_code == 201 + body = response.json() + assert set(body) == {"token", "token_type", "expires_at", "user"} + assert body["token_type"] == "Bearer" + assert len(body["token"]) >= 43 + assert body["token"] not in (user["id"], user["username"]) + assert body["user"]["id"] == user["id"] + assert body["expires_at"].endswith("Z") + assert datetime.fromisoformat(body["expires_at"]) + assert_request_id(response) + + +async def test_each_login_issues_a_different_token(client: AsyncClient): + user = await register(client) + first = await login(client, user) + second = await login(client, user) + + assert first["token"] != second["token"] + + +async def test_password_with_colon_works(client: AsyncClient): + user = await register(client, password="pass:with:colons-123") + + assert (await login(client, user))["token"] + + +async def test_unicode_password_works(client: AsyncClient): + user = await register(client, password="Пароль-с-юникодом-🔑-123") + + assert (await login(client, user))["token"] + + +async def test_wrong_password_and_unknown_user_look_identical(client: AsyncClient): + user = await register(client) + + wrong_password = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + unknown_user = await client.post( + "/api/v1/auth/sessions", + headers=basic_header("no_such_user", "wrong-password-123"), + ) + + for response in (wrong_password, unknown_user): + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert response.headers["WWW-Authenticate"].startswith( + 'Basic realm="messenger"' + ) + assert_request_id(response) + assert ( + wrong_password.json()["error"]["message"] + == unknown_user.json()["error"]["message"] + ) + + +async def test_login_requires_basic_scheme(client: AsyncClient): + user = await register(client) + session = await login(client, user) + + for headers in ({}, bearer(session), {"Authorization": "Basic !!!not-base64"}): + response = await client.post("/api/v1/auth/sessions", headers=headers) + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Basic") + + +async def test_business_endpoints_require_bearer(client: AsyncClient): + user = await register(client) + + for headers in ( + {}, + basic_header(user["username"], user["password"]), + {"Authorization": "Bearer "}, + {"Authorization": "Bearer unknown-token-value"}, + {"X-User-Id": user["id"]}, + ): + response = await client.get("/api/v1/users/me", headers=headers) + assert response.status_code == 401, headers + assert response.json()["error"]["code"] == "unauthorized" + assert response.headers["WWW-Authenticate"].startswith( + 'Bearer realm="messenger"' + ) + assert_request_id(response) + + +async def test_x_user_id_cannot_override_bearer_identity(client: AsyncClient): + alice = await create_user(client) + eve = await create_user(client) + + response = await client.get( + "/api/v1/users/me", + headers={**bearer(alice), "X-User-Id": eve["id"]}, + ) + + assert response.status_code == 200 + assert response.json()["id"] == alice["id"] + + +async def test_logout_revokes_only_current_session(client: AsyncClient): + user = await register(client) + first = await login(client, user) + second = await login(client, user) + + assert ( + await client.delete("/api/v1/auth/sessions/current", headers=bearer(first)) + ).status_code == 204 + + assert ( + await client.get("/api/v1/users/me", headers=bearer(first)) + ).status_code == 401 + assert ( + await client.get("/api/v1/users/me", headers=bearer(second)) + ).status_code == 200 + + +async def test_logout_twice_returns_401(client: AsyncClient): + session = await create_user(client) + + first = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + again = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + + assert first.status_code == 204 + assert first.content == b"" + assert again.status_code == 401 + + +async def test_session_expires_after_ttl(make_settings): + app = create_app(make_settings(session_ttl_seconds=2)) + async with make_client(app) as client: + session = await create_user(client) + assert ( + await client.get("/api/v1/users/me", headers=bearer(session)) + ).status_code == 200 + + await asyncio.sleep(3) + + response = await client.get("/api/v1/users/me", headers=bearer(session)) + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Bearer") + await close_app(app) + + +async def test_token_is_stored_only_as_sha256_key(client: AsyncClient, redis_client): + session = await create_user(client) + + keys = [key async for key in redis_client.scan_iter("session:*")] + token_hash = hashlib.sha256(session["token"].encode()).hexdigest() + + assert keys == [f"session:{token_hash}"] + assert await redis_client.get(keys[0]) == session["id"] + assert all(session["token"] not in key for key in keys) + + +async def test_password_is_not_stored_in_plaintext(client: AsyncClient, db_engine): + from sqlalchemy import text + + await register(client, username="hash_check") + + async with db_engine.connect() as connection: + result = await connection.execute( + text("SELECT password_hash FROM users WHERE username = 'hash_check'") + ) + stored = result.scalar_one() + + assert PASSWORD not in stored + assert stored.startswith("$argon2id$") + + +async def test_login_rejects_out_of_range_password_like_wrong_password( + client: AsyncClient, +): + user = await register(client) + reference = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + + for password in ("x" * 11, "x" * 129, ""): + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], password), + ) + assert response.status_code == 401 + assert ( + response.json()["error"]["message"] == reference.json()["error"]["message"] + ) + assert ( + response.headers["WWW-Authenticate"] + == reference.headers["WWW-Authenticate"] + ) + + +async def test_malformed_basic_credentials_return_401(client: AsyncClient): + import base64 + + def raw(value: bytes) -> dict[str, str]: + return {"Authorization": "Basic " + base64.b64encode(value).decode()} + + for headers in ( + raw(b"no-colon-here"), + raw(b":password-without-user"), + raw(b"\xff\xfe:invalid-utf8"), + {"Authorization": "Basic"}, + {"Authorization": "Digest abc"}, + ): + response = await client.post("/api/v1/auth/sessions", headers=headers) + assert response.status_code == 401, headers + assert response.headers["WWW-Authenticate"].startswith("Basic") + + +async def test_unknown_user_takes_comparable_time_to_wrong_password( + client: AsyncClient, +): + import statistics + import time + + user = await register(client) + + async def measure(username: str) -> float: + samples = [] + for _ in range(5): + started = time.perf_counter() + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(username, "wrong-password-123"), + ) + samples.append(time.perf_counter() - started) + return statistics.median(samples) + + wrong_password = await measure(user["username"]) + unknown_user = await measure("no_such_user") + + assert unknown_user > wrong_password * 0.5 + + +async def test_invalid_basic_username_returns_401_not_500(client: AsyncClient): + user = await register(client) + reference = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], "wrong-password-123"), + ) + + for username in ("bad\u0000name", "UPPER", "no", "x" * 33, "имя", "a b c"): + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(username, "wrong-password-123"), + ) + assert response.status_code == 401, username + assert ( + response.json()["error"]["message"] == reference.json()["error"]["message"] + ) + assert ( + response.headers["WWW-Authenticate"] + == reference.headers["WWW-Authenticate"] + ) + + +async def test_whitespace_only_password_at_login_returns_401(client: AsyncClient): + user = await register(client) + + response = await client.post( + "/api/v1/auth/sessions", headers=basic_header(user["username"], " " * 12) + ) + + assert response.status_code == 401 + assert response.headers["WWW-Authenticate"].startswith("Basic") diff --git a/student-tests/test_chats.py b/student-tests/test_chats.py new file mode 100644 index 00000000..475c946a --- /dev/null +++ b/student-tests/test_chats.py @@ -0,0 +1,209 @@ +from urllib.parse import quote +from uuid import uuid4 + +import pytest +from httpx import AsyncClient + +from helpers import assert_request_id, bearer, create_chat, create_user + + +async def test_create_and_get_chat(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat = await create_chat(client, alice, bob) + + response = await client.get( + f"/api/v1/chats/{chat['id']}", + headers=bearer(bob), + ) + + assert response.status_code == 200 + assert response.json() == chat + assert_request_id(response) + + +async def test_chat_is_not_visible_to_outsider(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + eve = await create_user(client) + chat = await create_chat(client, alice, bob) + + response = await client.get( + f"/api/v1/chats/{chat['id']}", + headers=bearer(eve), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_chat_creation_requires_creator_in_members(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [bob["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize( + "payload", + [ + {"title": "", "member_ids": []}, + {"title": " ", "member_ids": []}, + ], +) +async def test_chat_payload_boundaries(client: AsyncClient, payload: dict): + alice = await create_user(client) + payload["member_ids"] = [alice["id"]] + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json=payload, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_duplicate_members_are_rejected(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], bob["id"], bob["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_unknown_member_is_rejected(client: AsyncClient): + alice = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_chat_exact_title_limit_is_accepted(client: AsyncClient): + alice = await create_user(client) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "x" * 100, "member_ids": [alice["id"]]}, + ) + + assert response.status_code == 201 + assert len(response.json()["title"]) == 100 + + +async def test_chat_list_hides_chats_without_membership(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat_for_bob = await create_chat(client, bob, title="Only Bob") + chat_for_alice = await create_chat(client, alice, title="Only Alice") + + response = await client.get( + "/api/v1/chats?limit=100", + headers=bearer(alice), + ) + + assert response.status_code == 200 + ids = {item["id"] for item in response.json()["items"]} + assert chat_for_alice["id"] in ids + assert chat_for_bob["id"] not in ids + + +async def test_nonexistent_chat_returns_404(client: AsyncClient): + alice = await create_user(client) + + response = await client.get( + "/api/v1/chats/00000000-0000-4000-8000-000000000000", + headers=bearer(alice), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_failed_chat_creation_leaves_no_partial_data( + client: AsyncClient, db_engine +): + """Атомарность: неизвестный участник -> нет ни чата, ни записей членства.""" + from sqlalchemy import text + + alice = await create_user(client) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]}, + ) + assert response.status_code == 400 + + async with db_engine.connect() as connection: + chats = ( + await connection.execute(text("SELECT count(*) FROM chats")) + ).scalar_one() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert chats == 0 and members == 0 + + +async def test_chat_list_pagination_and_order(client: AsyncClient): + alice = await create_user(client) + chats = [await create_chat(client, alice, title=f"chat-{i}") for i in range(5)] + + page1 = (await client.get("/api/v1/chats?limit=2", headers=bearer(alice))).json() + page2 = ( + await client.get( + f"/api/v1/chats?limit=2&cursor={quote(page1['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + page3 = ( + await client.get( + f"/api/v1/chats?limit=2&cursor={quote(page2['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + ids = [c["id"] for page in (page1, page2, page3) for c in page["items"]] + assert ids == [c["id"] for c in chats] + assert page3["next_cursor"] is None + + +async def test_chat_endpoints_require_bearer(client: AsyncClient): + assert (await client.get("/api/v1/chats")).status_code == 401 + assert ( + await client.post("/api/v1/chats", json={"title": "x", "member_ids": []}) + ).status_code in (401, 400) + + +async def test_nul_in_chat_title_is_rejected(client: AsyncClient): + alice = await create_user(client) + + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "chat\u0000title", "member_ids": [alice["id"]]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..66860669 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,266 @@ +"""Конкурентность на реальной БД: гонка регистрации и параллельная запись.""" + +import asyncio +from collections import Counter +from uuid import uuid4 + +from sqlalchemy import text + +from app.main import create_app +from conftest import close_app, make_client +from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register + + +async def test_parallel_registration_of_same_username_creates_exactly_one_user( + make_settings, +): + app = create_app( + make_settings(max_in_flight=64, db_pool_size=20, db_max_overflow=20) + ) + + async with make_client(app) as client: + responses = await asyncio.gather( + *( + client.post( + "/api/v1/users", + json={ + "username": "race_user", + "display_name": "Race", + "password": PASSWORD, + }, + ) + for _ in range(16) + ) + ) + await close_app(app) + + statuses = Counter(response.status_code for response in responses) + assert statuses == Counter({201: 1, 409: 15}), statuses + + +async def test_100_messages_from_10_clients(make_settings): + app = create_app( + make_settings( + max_messages=1000, max_in_flight=100, db_pool_size=20, db_max_overflow=20 + ) + ) + + async with make_client(app) as client: + users = [ + await create_user(client, username=f"load_{i}_{uuid4().hex[:6]}") + for i in range(10) + ] + created = await client.post( + "/api/v1/chats", + headers=bearer(users[0]), + json={"title": "Concurrency", "member_ids": [u["id"] for u in users]}, + ) + chat = created.json() + + async def worker(index: int, user: dict) -> list[dict]: + responses = await asyncio.gather( + *( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header()}, + json={"text": f"client-{index}-message-{n}"}, + ) + for n in range(10) + ) + ) + assert all(r.status_code == 201 for r in responses), [ + r.text for r in responses if r.status_code != 201 + ] + return [r.json() for r in responses] + + results = await asyncio.gather(*(worker(i, u) for i, u in enumerate(users))) + messages = [m for batch in results for m in batch] + + assert len(messages) == 100 + assert len({m["id"] for m in messages}) == 100 + assert Counter(m["sender_id"] for m in messages) == Counter( + {u["id"]: 10 for u in users} + ) + + history = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(users[0]) + ) + items = history.json()["items"] + assert len(items) == 100 + assert len({m["id"] for m in items}) == 100 + await close_app(app) + + +async def test_chat_creation_in_parallel_is_atomic(make_settings, db_engine): + app = create_app( + make_settings(max_in_flight=32, db_pool_size=20, db_max_overflow=20) + ) + + async with make_client(app) as client: + alice = await create_user(client) + bob = await create_user(client) + responses = await asyncio.gather( + *( + client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": f"c{i}", "member_ids": [alice["id"], bob["id"]]}, + ) + for i in range(10) + ) + ) + assert all(r.status_code == 201 for r in responses) + await close_app(app) + + async with db_engine.connect() as connection: + chats = ( + await connection.execute(text("SELECT count(*) FROM chats")) + ).scalar_one() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert chats == 10 and members == 20 + + +async def test_parallel_registration_of_distinct_usernames_all_succeed(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + responses = await asyncio.gather( + *( + client.post( + "/api/v1/users", + json={ + "username": f"distinct_{i}", + "display_name": "D", + "password": PASSWORD, + }, + ) + for i in range(12) + ) + ) + await close_app(app) + + assert [r.status_code for r in responses] == [201] * 12 + assert len({r.json()["id"] for r in responses}) == 12 + + +async def test_parallel_logins_issue_distinct_valid_tokens(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + user = await register(client) + responses = await asyncio.gather( + *( + client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + for _ in range(10) + ) + ) + tokens = [r.json()["token"] for r in responses] + checks = await asyncio.gather( + *( + client.get( + "/api/v1/users/me", headers={"Authorization": f"Bearer {token}"} + ) + for token in tokens + ) + ) + await close_app(app) + + assert all(r.status_code == 201 for r in responses) + assert len(set(tokens)) == 10 + assert all(r.status_code == 200 for r in checks) + + +async def test_parallel_logout_of_same_token_succeeds_once(make_settings): + app = create_app(make_settings(max_in_flight=64)) + async with make_client(app) as client: + session = await create_user(client) + responses = await asyncio.gather( + *( + client.delete("/api/v1/auth/sessions/current", headers=bearer(session)) + for _ in range(10) + ) + ) + await close_app(app) + + assert Counter(r.status_code for r in responses) == Counter({204: 1, 401: 9}) + + +async def test_message_limit_is_exact_under_concurrency(make_settings, db_engine): + app = create_app( + make_settings( + max_messages=5, max_in_flight=64, db_pool_size=20, db_max_overflow=20 + ) + ) + async with make_client(app) as client: + user = await create_user(client) + chat = ( + await client.post( + "/api/v1/chats", + headers=bearer(user), + json={"title": "limit", "member_ids": [user["id"]]}, + ) + ).json() + responses = await asyncio.gather( + *( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header()}, + json={"text": f"m{i}"}, + ) + for i in range(20) + ) + ) + await close_app(app) + + assert Counter(r.status_code for r in responses) == Counter({201: 5, 503: 15}) + async with db_engine.connect() as connection: + stored = ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() + counter = ( + await connection.execute( + text("SELECT value FROM counters WHERE name = 'messages'") + ) + ).scalar_one() + assert stored == counter == 5 + + +async def test_failed_chat_insert_rolls_back_whole_transaction( + make_settings, db_engine, monkeypatch +): + import app.domain.chats as chats_domain + + async def accept_everyone(_session, user_ids): + return set(user_ids) + + app = create_app(make_settings()) + async with make_client(app) as client: + alice = await create_user(client) + monkeypatch.setattr( + chats_domain.users_storage, "get_existing_user_ids", accept_everyone + ) + response = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "broken", "member_ids": [alice["id"], str(uuid4())]}, + ) + monkeypatch.undo() + after = await client.post( + "/api/v1/chats", + headers=bearer(alice), + json={"title": "ok", "member_ids": [alice["id"]]}, + ) + await close_app(app) + + assert response.status_code == 400 + assert after.status_code == 201 + async with db_engine.connect() as connection: + titles = (await connection.execute(text("SELECT title FROM chats"))).scalars() + members = ( + await connection.execute(text("SELECT count(*) FROM chat_members")) + ).scalar_one() + assert list(titles) == ["ok"] + assert members == 1 diff --git a/student-tests/test_constraints.py b/student-tests/test_constraints.py new file mode 100644 index 00000000..11aa1f3e --- /dev/null +++ b/student-tests/test_constraints.py @@ -0,0 +1,46 @@ +"""Ограничения на уровне БД: приложение — не единственный защитник данных.""" + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError + +from helpers import create_chat, create_user + + +async def test_duplicate_username_is_rejected_by_database(client, db_engine): + await create_user(client, username="unique_name") + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (gen_random_uuid(), 'unique_name', 'Other', 'hash')" + ) + ) + + +async def test_message_requires_existing_chat(client, db_engine): + alice = await create_user(client) + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO messages (id, chat_id, sender_id, text) " + "VALUES (gen_random_uuid(), gen_random_uuid(), :sender, 'x')" + ), + {"sender": alice["id"]}, + ) + + +async def test_duplicate_membership_is_rejected_by_database(client, db_engine): + alice = await create_user(client) + chat = await create_chat(client, alice) + + with pytest.raises(IntegrityError): + async with db_engine.begin() as connection: + await connection.execute( + text("INSERT INTO chat_members (chat_id, user_id) VALUES (:c, :u)"), + {"c": chat["id"], "u": alice["id"]}, + ) diff --git a/student-tests/test_errors.py b/student-tests/test_errors.py new file mode 100644 index 00000000..3fe5e939 --- /dev/null +++ b/student-tests/test_errors.py @@ -0,0 +1,76 @@ +import json +import logging + +from httpx import AsyncClient + +from helpers import assert_request_id + + +async def test_missing_token_returns_401(client: AsyncClient): + response = await client.get("/api/v1/users/me") + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert_request_id(response) + + +async def test_garbage_token_returns_401(client: AsyncClient): + response = await client.get( + "/api/v1/users/me", headers={"Authorization": "Bearer garbage"} + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthorized" + assert_request_id(response) + + +async def test_malformed_json_returns_400(client: AsyncClient): + response = await client.post( + "/api/v1/users", + content='{"username":"alice",', + headers={"Content-Type": "application/json"}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_not_found_returns_404(client: AsyncClient): + response = await client.get("/no-such-path") + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_method_not_allowed_preserves_405(client: AsyncClient): + response = await client.delete("/api/v1/users") + + assert response.status_code == 405 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_unhandled_exception_is_hidden_from_client( + client_no_raise: AsyncClient, caplog +): + with caplog.at_level(logging.ERROR, logger="app.errors.handlers"): + response = await client_no_raise.get("/__test__/explode") + + assert response.status_code == 500 + payload = response.json() + assert payload["error"]["code"] == "internal_error" + assert payload["error"]["message"] == "Внутренняя ошибка сервера" + assert "test-secret" not in json.dumps(payload, ensure_ascii=False) + assert "Traceback" not in json.dumps(payload, ensure_ascii=False) + assert "Unhandled error request_id=" in caplog.text + assert "test-secret" in caplog.text + assert_request_id(response) + + +async def test_success_response_contains_request_id(client: AsyncClient): + response = await client.get("/health/live") + + assert response.status_code == 200 + assert response.headers.get("X-Request-Id") diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..9338c3b4 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,138 @@ +"""Недоступность Redis и PostgreSQL: ограниченный 503, без анонимного доступа.""" + +import time + +from app.main import create_app +from conftest import close_app, make_client +from helpers import PASSWORD, assert_request_id, basic_header, bearer, create_user + +ME = "/api/v1/users/me" +DEAD_DATABASE_URL = "postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none" + + +def assert_unavailable(response) -> None: + assert response.status_code == 503, response.text + assert response.json()["error"]["code"] == "unavailable" + assert "secretpass" not in response.text + assert "Traceback" not in response.text + assert_request_id(response) + + +async def test_redis_down_gives_503_for_login_and_protected_endpoints(make_settings): + alive = create_app(make_settings()) + async with make_client(alive) as client: + session = await create_user(client) + await close_app(alive) + + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + started = time.perf_counter() + protected = await client.get(ME, headers=bearer(session)) + elapsed = time.perf_counter() - started + + login = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(session["username"], session["password"]), + ) + logout = await client.delete( + "/api/v1/auth/sessions/current", headers=bearer(session) + ) + ready = await client.get("/health/ready") + live = await client.get("/health/live") + + await close_app(app) + + for response in (protected, login, logout, ready): + assert_unavailable(response) + assert live.status_code == 200 + assert elapsed < 3 + + +async def test_redis_down_keeps_registration_working(make_settings): + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "no_redis", "display_name": "N", "password": PASSWORD}, + ) + await close_app(app) + + assert response.status_code == 201 + + +async def test_database_down_with_valid_session_gives_503(make_settings): + alive = create_app(make_settings()) + async with make_client(alive) as client: + session = await create_user(client) + await close_app(alive) + + app = create_app( + make_settings(database_url=DEAD_DATABASE_URL, db_connect_timeout_seconds=1) + ) + async with make_client(app, raise_app_exceptions=False) as client: + protected = await client.get(ME, headers=bearer(session)) + chats = await client.get("/api/v1/chats", headers=bearer(session)) + login = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(session["username"], session["password"]), + ) + await close_app(app) + + for response in (protected, chats, login): + assert_unavailable(response) + + +async def test_dependency_failure_never_authenticates_anonymous_requests(make_settings): + app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5)) + async with make_client(app, raise_app_exceptions=False) as client: + no_token = await client.get(ME) + garbage = await client.get(ME, headers={"Authorization": "Bearer garbage"}) + await close_app(app) + + assert no_token.status_code == 401 + assert_unavailable(garbage) + + +async def test_statement_timeout_gives_503(make_settings, monkeypatch): + from sqlalchemy import text + + import app.domain.users as users_domain + + async def slow_lookup(session, _username): + await session.execute(text("SELECT pg_sleep(1)")) + + app = create_app(make_settings(db_statement_timeout_ms=100)) + monkeypatch.setattr(users_domain.users_storage, "get_user_by_username", slow_lookup) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "slow_user", "display_name": "S", "password": PASSWORD}, + ) + await close_app(app) + + assert_unavailable(response) + assert response.headers["Retry-After"] == "1" + + +async def test_non_transient_database_error_stays_500(make_settings, monkeypatch): + from sqlalchemy import text + + import app.domain.users as users_domain + + async def broken_lookup(session, _username): + await session.execute(text("SELECT 1 / 0")) + + app = create_app(make_settings()) + monkeypatch.setattr( + users_domain.users_storage, "get_user_by_username", broken_lookup + ) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={"username": "broken_user", "display_name": "B", "password": PASSWORD}, + ) + await close_app(app) + + assert response.status_code == 500 + assert response.json()["error"]["code"] == "internal_error" + assert "division" not in response.text diff --git a/student-tests/test_health.py b/student-tests/test_health.py new file mode 100644 index 00000000..26d15896 --- /dev/null +++ b/student-tests/test_health.py @@ -0,0 +1,62 @@ +from httpx import AsyncClient + +from app.main import create_app +from conftest import close_app, make_client +from helpers import assert_request_id + + +async def test_liveness(client: AsyncClient): + response = await client.get("/health/live") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert_request_id(response) + + +async def test_readiness(client: AsyncClient): + response = await client.get("/health/ready") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert_request_id(response) + + +async def test_readiness_is_503_when_database_is_down(make_settings): + app = create_app( + make_settings( + database_url="postgresql+asyncpg://nobody:nopass@127.0.0.1:1/none", + db_connect_timeout_seconds=1, + ) + ) + async with make_client(app) as client: + assert (await client.get("/health/live")).status_code == 200 + + ready = await client.get("/health/ready") + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert_request_id(ready) + await close_app(app) + + +async def test_business_request_is_503_when_database_is_down(make_settings): + app = create_app( + make_settings( + database_url="postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none", + db_connect_timeout_seconds=1, + ) + ) + async with make_client(app, raise_app_exceptions=False) as client: + response = await client.post( + "/api/v1/users", + json={ + "username": "alice", + "display_name": "Alice", + "password": "Correct-horse-battery-1", + }, + ) + await close_app(app) + + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert "secretpass" not in response.text + assert_request_id(response) diff --git a/student-tests/test_history_plan.py b/student-tests/test_history_plan.py new file mode 100644 index 00000000..704c9ccd --- /dev/null +++ b/student-tests/test_history_plan.py @@ -0,0 +1,78 @@ +"""План запроса истории на >= 10 000 сообщений: индекс без Sort и полного скана.""" + +import json +from datetime import datetime +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + +from helpers import bearer, create_chat, create_user + +HISTORY_QUERY = """ +SELECT * FROM messages +WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id) +ORDER BY created_at, id +LIMIT 51 +""" + + +def walk(node: dict) -> list[dict]: + nodes = [node] + for child in node.get("Plans", []): + nodes.extend(walk(child)) + return nodes + + +async def seed_messages( + engine: AsyncEngine, chat_id: str, sender_id: str, count: int +) -> None: + async with engine.begin() as connection: + await connection.execute( + text( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g, + now() - make_interval(secs => :count - g) + FROM generate_series(1, :count) AS g + """ + ), + {"chat_id": chat_id, "sender_id": sender_id, "count": count}, + ) + await connection.execute(text("ANALYZE messages")) + + +async def test_history_page_uses_composite_index(client, db_engine: AsyncEngine): + alice = await create_user(client) + target = await create_chat(client, alice, title="target") + noise = await create_chat(client, alice, title="noise") + await seed_messages(db_engine, target["id"], alice["id"], 10_000) + await seed_messages(db_engine, noise["id"], alice["id"], 10_000) + + middle = ( + await client.get( + f"/api/v1/chats/{target['id']}/messages?limit=100", headers=bearer(alice) + ) + ).json()["items"][-1] + + async with db_engine.connect() as connection: + raw_plan = ( + await connection.execute( + text(f"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {HISTORY_QUERY}"), + { + "chat_id": UUID(target["id"]), + "after_created_at": datetime.fromisoformat(middle["created_at"]), + "after_id": UUID(middle["id"]), + }, + ) + ).scalar_one() + + plan = (raw_plan if isinstance(raw_plan, list) else json.loads(raw_plan))[0]["Plan"] + nodes = walk(plan) + node_types = {node["Node Type"] for node in nodes} + index_names = {node.get("Index Name") for node in nodes} + + assert "ix_messages_chat_id_created_at_id" in index_names + assert "Sort" not in node_types + assert "Seq Scan" not in node_types + assert all(node.get("Actual Rows", 0) <= 51 for node in nodes) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..9455e3d4 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,156 @@ +"""Повторы сообщений через приложения с общими хранилищами.""" + +import asyncio +from contextlib import AsyncExitStack + +import pytest +from sqlalchemy import text + +from app.errors.exceptions import unavailable +from app.main import create_app +from app.storage import idempotency as key_storage +from conftest import close_app, make_client +from helpers import bearer, create_chat, create_user, idempotency_header, send_message + + +@pytest.fixture +async def replicas(request, make_settings): + overrides = getattr(request, "param", {}) + apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")] + try: + async with AsyncExitStack() as stack: + clients = [await stack.enter_async_context(make_client(app)) for app in apps] + yield clients + finally: + for app in apps: + await close_app(app) + + +@pytest.mark.parametrize("key", [None, "", "a b", "a/b", "x" * 129]) +async def test_invalid_key_returns_400(client, key): + user = await create_user(client) + chat = await create_chat(client, user) + headers = bearer(user) + if key is not None: + headers["Idempotency-Key"] = key + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", headers=headers, json={"text": "hello"} + ) + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize("key", ["a", "x" * 128, "key_1.2:3-4"]) +async def test_valid_key_boundaries(client, key): + user = await create_user(client) + chat = await create_chat(client, user) + assert (await send_message(client, user, chat, "hello", key)).status_code == 201 + + +async def test_replay_and_conflict_across_replicas(replicas, db_engine): + first, second = replicas + user = await create_user(first) + chat = await create_chat(first, user) + original = await send_message(first, user, chat, "Привет", "same-key") + repeated = await second.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"}, + content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }', + ) + conflict = await send_message(second, user, chat, "другой текст", "same-key") + assert original.status_code == repeated.status_code == 201 + assert original.json() == repeated.json() + assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"] + assert conflict.status_code == 409 + async with db_engine.connect() as connection: + assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 + assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1 + + +async def test_parallel_replays_create_one_message(replicas, db_engine): + user = await create_user(replicas[0]) + chat = await create_chat(replicas[0], user) + responses = await asyncio.gather(*( + send_message(replicas[index % 2], user, chat, "parallel", "one-key") + for index in range(24) + )) + assert all(response.status_code == 201 for response in responses) + assert len({response.json()["id"] for response in responses}) == 1 + assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"} + async with db_engine.connect() as connection: + assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 + assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1 + + +async def test_key_scope_separates_users_and_chats(client): + alice = await create_user(client) + bob = await create_user(client) + shared = await create_chat(client, alice, bob) + other = await create_chat(client, alice) + responses = [ + await send_message(client, alice, shared, "a", "shared-key"), + await send_message(client, bob, shared, "b", "shared-key"), + await send_message(client, alice, other, "c", "shared-key"), + ] + assert all(response.status_code == 201 for response in responses) + assert len({response.json()["id"] for response in responses}) == 3 + + +async def test_expired_key_can_be_reused_without_removing_message(client, db_engine): + user = await create_user(client) + chat = await create_chat(client, user) + first = await send_message(client, user, chat, "old", "expiring-key") + async with db_engine.begin() as connection: + await connection.execute(text( + "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" + )) + second = await send_message(client, user, chat, "new", "expiring-key") + assert first.status_code == second.status_code == 201 + assert first.json()["id"] != second.json()["id"] + history = await client.get( + f"/api/v1/chats/{chat['id']}/messages", headers=bearer(user) + ) + assert [item["text"] for item in history.json()["items"]] == ["old", "new"] + + +async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeypatch): + user = await create_user(client) + chat = await create_chat(client, user) + original = key_storage.save_result + + async def fail_after_insert(*args): + await original(*args) + raise unavailable() + + monkeypatch.setattr(key_storage, "save_result", fail_after_insert) + failed = await send_message(client, user, chat, "retry", "retry-key") + assert failed.status_code == 503 + async with db_engine.connect() as connection: + for table in ("messages", "idempotency_keys", "counters"): + assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + monkeypatch.setattr(key_storage, "save_result", original) + recovered = await send_message(client, user, chat, "retry", "retry-key") + assert recovered.status_code == 201 + + +@pytest.mark.parametrize("replicas", [{"max_messages": 1}], indirect=True) +async def test_replay_works_when_message_limit_is_full(replicas): + user = await create_user(replicas[0]) + chat = await create_chat(replicas[0], user) + first = await send_message(replicas[0], user, chat, "last", "last-key") + replay = await send_message(replicas[1], user, chat, "last", "last-key") + rejected = await send_message(replicas[1], user, chat, "extra", "other-key") + assert first.status_code == replay.status_code == 201 + assert first.json() == replay.json() + assert rejected.status_code == 503 + + +async def test_replay_checks_session_and_membership(client, db_engine): + user = await create_user(client) + chat = await create_chat(client, user) + assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201 + async with db_engine.begin() as connection: + await connection.execute(text("DELETE FROM chat_members")) + assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404 + await client.delete("/api/v1/auth/sessions/current", headers=bearer(user)) + assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401 diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py new file mode 100644 index 00000000..8b0d932b --- /dev/null +++ b/student-tests/test_instance.py @@ -0,0 +1,106 @@ +"""Идентификатор реплики и заголовки ответов без PostgreSQL, Redis и Docker.""" + +from uuid import UUID + +import pytest +from fastapi import Request + +from app.core.instance import resolve_instance_id +from app.errors.exceptions import unavailable +from app.main import create_app +from conftest import close_app, make_client + + +@pytest.fixture(autouse=True) +def clean_db() -> None: + """Заменяет очистку БД из conftest: эти тесты не обращаются к хранилищам.""" + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ("api-explicit", "api-explicit"), + (" api-explicit ", "api-explicit"), + (None, "container-123"), + ("", "container-123"), + (" ", "container-123"), + ], +) +def test_resolve_prefers_configured_id_then_hostname(monkeypatch, configured, expected): + monkeypatch.setenv("HOSTNAME", " container-123 ") + + assert resolve_instance_id(configured) == expected + + +@pytest.mark.parametrize("hostname", [None, "", " "]) +def test_resolve_without_config_or_hostname_returns_uuid(monkeypatch, hostname): + if hostname is None: + monkeypatch.delenv("HOSTNAME", raising=False) + else: + monkeypatch.setenv("HOSTNAME", hostname) + + assert UUID(resolve_instance_id(None)).version == 4 + + +@pytest.fixture +async def instance_client(make_settings): + app = create_app( + make_settings( + instance_id="api-test", + database_url="postgresql+asyncpg://unused@127.0.0.1:1/unused", + redis_port=1, + ) + ) + + @app.get("/__test__/request-id", include_in_schema=False) + async def request_id(request: Request) -> dict[str, str]: + return {"request_id": request.state.request_id} + + @app.get("/__test__/explode", include_in_schema=False) + async def explode() -> None: + raise RuntimeError("test error") + + @app.get("/__test__/unavailable", include_in_schema=False) + async def dependency_failure() -> None: + raise unavailable() + + try: + async with make_client(app, raise_app_exceptions=False) as client: + yield client + finally: + await close_app(app) + + +@pytest.mark.parametrize( + ("method", "path", "status"), + [ + ("GET", "/health/live", 200), + ("POST", "/api/v1/users", 400), + ("GET", "/api/v1/users/me", 401), + ("GET", "/no-such-path", 404), + ("DELETE", "/health/live", 405), + ("GET", "/__test__/explode", 500), + ("GET", "/__test__/unavailable", 503), + ], +) +async def test_instance_header_is_present_on_success_and_errors( + instance_client, method, path, status +): + response = await instance_client.request(method, path) + + assert response.status_code == status + assert response.headers["X-Instance-Id"] == "api-test" + assert response.headers["X-Request-Id"] + if status >= 400: + assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + + +async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client): + first = await instance_client.get("/__test__/request-id") + second = await instance_client.get("/__test__/request-id") + + assert first.status_code == second.status_code == 200 + assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"] + for response in (first, second): + assert response.json()["request_id"] == response.headers["X-Request-Id"] diff --git a/student-tests/test_lifecycle.py b/student-tests/test_lifecycle.py new file mode 100644 index 00000000..1a659b74 --- /dev/null +++ b/student-tests/test_lifecycle.py @@ -0,0 +1,50 @@ +"""Готовность при завершении и ограничение времени проверки зависимостей.""" + +import asyncio +import time + +import pytest + +from app.domain import health +from app.main import create_app +from conftest import close_app, make_client + + +@pytest.fixture(autouse=True) +def clean_db() -> None: + """Эти проверки не обращаются к хранилищам.""" + + +async def test_draining_keeps_liveness_but_rejects_new_work(make_settings): + app = create_app(make_settings()) + app.state.lifecycle.start_draining() + try: + async with make_client(app) as client: + assert (await client.get('/health/live')).status_code == 200 + assert (await client.get('/health/ready')).status_code == 503 + assert (await client.get('/api/v1/users/me')).status_code == 503 + finally: + await close_app(app) + + +async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkeypatch): + cancelled = [] + + async def stalled(_): + try: + await asyncio.sleep(30) + finally: + cancelled.append(True) + + monkeypatch.setattr(health, 'check_database', stalled) + monkeypatch.setattr(health, 'check_redis', stalled) + app = create_app(make_settings(readiness_timeout_seconds=0.03)) + try: + async with make_client(app) as client: + started = time.monotonic() + response = await client.get('/health/ready') + assert response.status_code == 503 + assert time.monotonic() - started < 1 + assert len(cancelled) == 2 + finally: + await close_app(app) diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py new file mode 100644 index 00000000..3bd68c0d --- /dev/null +++ b/student-tests/test_limits.py @@ -0,0 +1,98 @@ +"""Ограничения нагрузки: MAX_IN_FLIGHT -> 429, MAX_MESSAGES -> 503.""" + +import asyncio + +import httpx + +from app.main import create_app +from conftest import close_app, make_client +from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header + + +def _assert_error(response: httpx.Response, status: int, code: str) -> None: + assert response.status_code == status + body = response.json() + assert body["error"]["code"] == code + assert response.headers.get("X-Request-Id") == body["error"]["request_id"] + + +async def test_message_limit_rejects_without_deleting_confirmed_messages(make_settings): + app = create_app(make_settings(max_messages=2)) + async with make_client(app) as client: + user = await create_user(client) + chat = await create_chat(client, user) + url = f"/api/v1/chats/{chat['id']}/messages" + + for index in range(2): + ok = await client.post( + url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"} + ) + assert ok.status_code == 201 + + rejected = await client.post( + url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"} + ) + _assert_error(rejected, 503, "unavailable") + + history = await client.get(f"{url}?limit=100", headers=bearer(user)) + assert history.status_code == 200 + assert [m["text"] for m in history.json()["items"]] == [ + "confirmed-0", + "confirmed-1", + ] + await close_app(app) + + +async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypatch): + import app.endpoints.messages as message_endpoint + + app = create_app(make_settings(max_in_flight=2)) + + original_send_message = message_endpoint.send_message_idempotent + accepted_count = 0 + all_slots_taken = asyncio.Event() + release = asyncio.Event() + + async def blocking_send_message(*args, **kwargs): + nonlocal accepted_count + accepted_count += 1 + if accepted_count == 2: + all_slots_taken.set() + await asyncio.wait_for(release.wait(), timeout=5) + return await original_send_message(*args, **kwargs) + + async with make_client(app, timeout=10) as client: + user = await create_user(client) + chat = await create_chat(client, user) + + monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message) + url = f"/api/v1/chats/{chat['id']}/messages" + + held = [ + asyncio.create_task( + client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"}) + ) + for i in range(2) + ] + + await asyncio.wait_for(all_slots_taken.wait(), timeout=2) + + rejected = await asyncio.gather( + *( + client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"}) + for i in range(4) + ) + ) + for response in rejected: + _assert_error(response, 429, "rate_limited") + assert response.headers.get("Retry-After") == "1" + + assert (await client.get("/health/live")).status_code == 200 + + release.set() + assert all(r.status_code == 201 for r in await asyncio.gather(*held)) + + recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"}) + assert recovered.status_code == 201 + assert_request_id(recovered) + await close_app(app) diff --git a/student-tests/test_messages.py b/student-tests/test_messages.py new file mode 100644 index 00000000..07fdc08f --- /dev/null +++ b/student-tests/test_messages.py @@ -0,0 +1,238 @@ +from urllib.parse import quote + +import pytest +from httpx import AsyncClient + +from helpers import ( + assert_request_id, + bearer, + create_chat, + create_user, + idempotency_header, + send_message, +) + + +async def _message(client: AsyncClient, user: dict, chat: dict, text: str): + return await send_message(client, user, chat, text) + + +async def test_send_and_list_message(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + chat = await create_chat(client, alice, bob) + + created = await _message(client, alice, chat, "Привет, мир 👋") + + assert created.status_code == 201 + message = created.json() + assert message["sender_id"] == alice["id"] + assert message["chat_id"] == chat["id"] + assert_request_id(created) + + listed = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", + headers=bearer(bob), + ) + + assert listed.status_code == 200 + assert listed.json()["items"] == [message] + assert listed.json()["next_cursor"] is None + assert_request_id(listed) + + +@pytest.mark.parametrize("text", ["", " ", "x" * 4001]) +async def test_invalid_message_text(client: AsyncClient, text: str): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, text) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_extra_message_field_is_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**bearer(alice), **idempotency_header()}, + json={"text": "hello", "sender_id": alice["id"]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_messages_are_private_to_chat_members(client: AsyncClient): + alice = await create_user(client) + eve = await create_user(client) + chat = await create_chat(client, alice) + await _message(client, alice, chat, "секрет") + + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages", + headers=bearer(eve), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + + +async def test_pagination_has_no_duplicates(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + created = [ + (await _message(client, alice, chat, f"message-{index}")).json() + for index in range(5) + ] + + page1 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2", + headers=bearer(alice), + ) + ).json() + page2 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page1['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + page3 = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page2['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + ids = [item["id"] for page in (page1, page2, page3) for item in page["items"]] + assert ids == [item["id"] for item in created] + assert len(ids) == len(set(ids)) == 5 + assert len(page1["items"]) == 2 + assert len(page2["items"]) == 2 + assert len(page3["items"]) == 1 + assert page3["next_cursor"] is None + + +async def test_cursor_cannot_be_used_for_another_chat(client: AsyncClient): + alice = await create_user(client) + chat_a = await create_chat(client, alice, title="A") + chat_b = await create_chat(client, alice, title="B") + + await _message(client, alice, chat_a, "a1") + await _message(client, alice, chat_a, "a2") + await _message(client, alice, chat_b, "b1") + await _message(client, alice, chat_b, "b2") + + page = ( + await client.get( + f"/api/v1/chats/{chat_a['id']}/messages?limit=1", + headers=bearer(alice), + ) + ).json() + + response = await client.get( + f"/api/v1/chats/{chat_b['id']}/messages?limit=1&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize( + "limit, expected_status", [(0, 400), (1, 200), (100, 200), (101, 400)] +) +async def test_limit_boundaries(client: AsyncClient, limit: int, expected_status: int): + alice = await create_user(client) + chat = await create_chat(client, alice) + await _message(client, alice, chat, "one") + + response = await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit={limit}", + headers=bearer(alice), + ) + + assert response.status_code == expected_status + assert_request_id(response) + + +async def test_message_exact_max_length_is_accepted(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, "x" * 4000) + + assert response.status_code == 201 + assert len(response.json()["text"]) == 4000 + + +async def test_nonexistent_chat_messages_return_404(client: AsyncClient): + alice = await create_user(client) + + response = await client.get( + "/api/v1/chats/00000000-0000-4000-8000-000000000000/messages", + headers=bearer(alice), + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "not_found" + assert_request_id(response) + + +async def test_unauthenticated_requests_are_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + url = f"/api/v1/chats/{chat['id']}/messages" + + assert (await client.get(url)).status_code == 401 + assert (await client.post(url, json={"text": "hi"})).status_code == 401 + + +async def test_messages_survive_new_login(client: AsyncClient): + """Сообщения принадлежат пользователю, а не сессии: после logout/login видны.""" + from helpers import login + + alice = await create_user(client) + chat = await create_chat(client, alice) + message = (await _message(client, alice, chat, "до выхода")).json() + await client.delete("/api/v1/auth/sessions/current", headers=bearer(alice)) + + alice_again = await login(client, alice) + listed = await client.get( + f"/api/v1/chats/{chat['id']}/messages", headers=bearer(alice_again) + ) + + assert listed.json()["items"] == [message] + + +async def test_history_order_is_created_at_then_id(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + created = [(await _message(client, alice, chat, f"m{i}")).json() for i in range(10)] + + listed = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(alice) + ) + ).json()["items"] + + assert [m["id"] for m in listed] == [m["id"] for m in created] + keys = [(m["created_at"], m["id"]) for m in listed] + assert keys == sorted(keys) + + +async def test_nul_in_message_text_is_rejected(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + + response = await _message(client, alice, chat, "hello\u0000world") + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py new file mode 100644 index 00000000..f13139d8 --- /dev/null +++ b/student-tests/test_migrations.py @@ -0,0 +1,67 @@ +"""Миграции: применение с нуля, идемпотентный повтор, совпадение со схемой моделей.""" + +import pytest +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from sqlalchemy import inspect, text + +from app.models import Base +from conftest import reset_schema, upgrade_head + +EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents"} + + +@pytest.fixture +async def migrated_engine(database_url, db_engine): + await reset_schema(db_engine) + await upgrade_head(database_url) + yield db_engine + + await reset_schema(db_engine) + async with db_engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + +async def test_upgrade_creates_all_tables(migrated_engine): + async with migrated_engine.connect() as connection: + tables = await connection.run_sync( + lambda sync: set(inspect(sync).get_table_names()) + ) + assert EXPECTED_TABLES <= tables + + +async def test_upgrade_twice_is_idempotent_and_keeps_data( + migrated_engine, database_url +): + async with migrated_engine.begin() as connection: + await connection.execute( + text( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (gen_random_uuid(), 'keep_me', 'Keep', 'hash')" + ) + ) + + await upgrade_head(database_url) + + async with migrated_engine.connect() as connection: + count = ( + await connection.execute( + text("SELECT count(*) FROM users WHERE username = 'keep_me'") + ) + ).scalar_one() + assert count == 1 + + +async def test_models_and_migrations_are_in_sync(migrated_engine): + def compare(sync_connection): + context = MigrationContext.configure( + sync_connection, opts={"compare_type": True} + ) + return compare_metadata(context, Base.metadata) + + async with migrated_engine.connect() as connection: + diff = await connection.run_sync(compare) + + assert diff == [], ( + f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" + ) diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py new file mode 100644 index 00000000..e45e5b1b --- /dev/null +++ b/student-tests/test_pagination.py @@ -0,0 +1,137 @@ +"""Курсоры: непрозрачность, привязка к ресурсу и устойчивость к подделке.""" + +import base64 +import json +from urllib.parse import quote + +import pytest +from httpx import AsyncClient + +from helpers import bearer, create_chat, create_user, send_message + + +def forged_cursor(**payload: str) -> str: + return base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + + +@pytest.mark.parametrize( + "cursor", + [ + "not-base64-%%%", + base64.urlsafe_b64encode(b"not json").decode(), + base64.urlsafe_b64encode(b"[1, 2, 3]").decode(), + forged_cursor(kind="messages"), + ], +) +async def test_garbage_cursor_returns_400(client: AsyncClient, cursor: str): + alice = await create_user(client) + chat = await create_chat(client, alice) + + for url in ( + f"/api/v1/chats?cursor={quote(cursor, safe='')}", + f"/api/v1/chats/{chat['id']}/messages?cursor={quote(cursor, safe='')}", + ): + response = await client.get(url, headers=bearer(alice)) + assert response.status_code == 400, url + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_cursor_without_timezone_is_rejected(client: AsyncClient): + alice = await create_user(client) + cursor = forged_cursor( + kind="chats", + resource_id=alice["id"], + created_at="2026-01-01T00:00:00", + item_id="00000000-0000-4000-8000-000000000000", + ) + + response = await client.get( + f"/api/v1/chats?cursor={quote(cursor, safe='')}", headers=bearer(alice) + ) + + assert response.status_code == 400 + + +async def test_chat_list_cursor_is_bound_to_its_owner(client: AsyncClient): + alice = await create_user(client) + bob = await create_user(client) + for index in range(2): + await create_chat(client, alice, bob, title=f"shared-{index}") + + page = (await client.get("/api/v1/chats?limit=1", headers=bearer(alice))).json() + response = await client.get( + f"/api/v1/chats?limit=1&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(bob), + ) + + assert response.status_code == 400 + + +async def test_messages_cursor_is_not_accepted_by_chat_list(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + for index in range(2): + await send_message(client, alice, chat, f"m{index}") + + page = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + response = await client.get( + f"/api/v1/chats?cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + + assert response.status_code == 400 + + +async def test_limit_can_change_between_pages(client: AsyncClient): + alice = await create_user(client) + chat = await create_chat(client, alice) + sent = [ + (await send_message(client, alice, chat, f"m{index}")).json() + for index in range(6) + ] + + first = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + rest = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100" + f"&cursor={quote(first['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + assert [m["id"] for m in first["items"] + rest["items"]] == [m["id"] for m in sent] + assert rest["next_cursor"] is None + + +async def test_message_sent_after_reading_last_page_is_seen_with_cursor( + client: AsyncClient, +): + alice = await create_user(client) + chat = await create_chat(client, alice) + await send_message(client, alice, chat, "first") + await send_message(client, alice, chat, "second") + + page = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice) + ) + ).json() + new = (await send_message(client, alice, chat, "third")).json() + rest = ( + await client.get( + f"/api/v1/chats/{chat['id']}/messages?limit=100" + f"&cursor={quote(page['next_cursor'], safe='')}", + headers=bearer(alice), + ) + ).json() + + assert [m["text"] for m in rest["items"]] == ["second", "third"] + assert rest["items"][-1] == new diff --git a/student-tests/test_s3_stream.py b/student-tests/test_s3_stream.py new file mode 100644 index 00000000..b168a626 --- /dev/null +++ b/student-tests/test_s3_stream.py @@ -0,0 +1,56 @@ +"""Освобождение HTTP-соединения и порядок завершения SDK при отмене.""" + +import asyncio +from threading import Event +from types import SimpleNamespace + +from app.core.s3 import S3Settings +from app.storage.s3 import ObjectStream, run_sync + + +async def test_sdk_thread_finishes_before_cancellation_returns(): + entered, release, finished = Event(), Event(), Event() + def operation(): + entered.set() + release.wait(timeout=3) + finished.set() + task = asyncio.create_task(run_sync(operation)) + await asyncio.to_thread(entered.wait, 2) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + release.set() + try: + await task + except asyncio.CancelledError: + pass + assert finished.is_set() + + +async def test_response_discarded_when_get_is_cancelled(): + entered, release = Event(), Event() + discarded = [] + response = object() + def operation(): + entered.set() + release.wait(timeout=3) + return response + task = asyncio.create_task(run_sync(operation, discard_result=discarded.append)) + await asyncio.to_thread(entered.wait, 2) + task.cancel() + release.set() + try: + await task + except asyncio.CancelledError: + pass + assert discarded == [response] + + +async def test_stream_closes_even_before_first_iteration(): + calls = [] + response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release')) + storage = SimpleNamespace(settings=S3Settings(_env_file=None)) + stream = ObjectStream(storage, response) + await stream.aclose() + await stream.aclose() + assert calls == ['close', 'release'] diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..e239e227 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,173 @@ +"""Сессии в Redis: TTL, абсолютный срок, отзыв, потеря и перезапуск хранилища.""" + +import asyncio +import hashlib +import os +import time +from datetime import datetime + +import pytest +from httpx import AsyncClient +from redis.asyncio import Redis + +from app.main import create_app +from conftest import close_app, make_client +from helpers import basic_header, bearer, create_user, register + +ME = "/api/v1/users/me" +LOGOUT = "/api/v1/auth/sessions/current" + + +def session_key(token: str) -> str: + return "session:" + hashlib.sha256(token.encode()).hexdigest() + + +async def test_session_key_ttl_matches_expires_at( + client: AsyncClient, redis_client: Redis +): + user = await register(client) + response = await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + body = response.json() + + expires_at = datetime.fromisoformat(body["expires_at"]) + expire_time_ms = await redis_client.pexpiretime(session_key(body["token"])) + + assert abs(expire_time_ms - expires_at.timestamp() * 1000) < 2 + assert 299_000 < await redis_client.pttl(session_key(body["token"])) <= 300_000 + + +async def test_session_ttl_is_absolute_and_not_extended_by_activity( + client: AsyncClient, redis_client: Redis +): + session = await create_user(client) + key = session_key(session["token"]) + expire_before = await redis_client.pexpiretime(key) + + for _ in range(5): + assert (await client.get(ME, headers=bearer(session))).status_code == 200 + + assert await redis_client.pexpiretime(key) == expire_before + + +async def test_logout_removes_session_key(client: AsyncClient, redis_client: Redis): + session = await create_user(client) + key = session_key(session["token"]) + assert await redis_client.exists(key) == 1 + + assert (await client.delete(LOGOUT, headers=bearer(session))).status_code == 204 + + assert await redis_client.exists(key) == 0 + assert (await client.get(ME, headers=bearer(session))).status_code == 401 + + +async def test_expired_session_is_rejected_everywhere(make_settings): + app = create_app(make_settings(session_ttl_seconds=1)) + async with make_client(app) as client: + session = await create_user(client) + assert (await client.get(ME, headers=bearer(session))).status_code == 200 + + await asyncio.sleep(1.5) + + for method, url in (("GET", ME), ("GET", "/api/v1/chats"), ("DELETE", LOGOUT)): + response = await client.request(method, url, headers=bearer(session)) + assert response.status_code == 401, (method, url) + assert response.headers["WWW-Authenticate"].startswith("Bearer") + await close_app(app) + + +async def test_relogin_after_expiry_issues_new_working_token(make_settings): + app = create_app(make_settings(session_ttl_seconds=1)) + async with make_client(app) as client: + user = await register(client) + first = ( + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + ).json() + await asyncio.sleep(1.5) + second = ( + await client.post( + "/api/v1/auth/sessions", + headers=basic_header(user["username"], user["password"]), + ) + ).json() + + assert first["token"] != second["token"] + assert (await client.get(ME, headers=bearer(first))).status_code == 401 + assert (await client.get(ME, headers=bearer(second))).status_code == 200 + await close_app(app) + + +async def test_tampered_token_is_rejected(client: AsyncClient): + session = await create_user(client) + token = session["token"] + tampered = token[:-1] + ("A" if token[-1] != "A" else "B") + + response = await client.get(ME, headers={"Authorization": f"Bearer {tampered}"}) + + assert response.status_code == 401 + + +async def test_session_of_deleted_user_is_rejected(client: AsyncClient, db_engine): + from sqlalchemy import text + + session = await create_user(client) + async with db_engine.begin() as connection: + await connection.execute( + text("DELETE FROM users WHERE id = :id"), {"id": session["id"]} + ) + + response = await client.get(ME, headers=bearer(session)) + + assert response.status_code == 401 + + +async def test_redis_flush_invalidates_all_sessions( + client: AsyncClient, redis_client: Redis +): + alice = await create_user(client) + bob = await create_user(client) + + await redis_client.flushdb() + + for session in (alice, bob): + assert (await client.get(ME, headers=bearer(session))).status_code == 401 + + +@pytest.mark.skipif( + os.environ.get("DOCKER_TESTS") != "1", + reason="перезапускает контейнер redis из compose; включается DOCKER_TESTS=1", +) +async def test_sessions_survive_redis_restart(client: AsyncClient, redis_client: Redis): + active = await create_user(client) + revoked = await create_user(client) + assert (await client.delete(LOGOUT, headers=bearer(revoked))).status_code == 204 + + process = await asyncio.create_subprocess_exec( + "docker", + "compose", + "restart", + "redis", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + assert await process.wait() == 0 + + deadline = time.monotonic() + 30 + while True: + try: + await redis_client.ping() + break + except Exception: + if time.monotonic() > deadline: + raise + await asyncio.sleep(0.5) + + response = await client.get(ME, headers=bearer(active)) + assert response.status_code == 200 + assert response.json()["id"] == active["id"] + assert (await client.get(ME, headers=bearer(revoked))).status_code == 401 diff --git a/student-tests/test_users.py b/student-tests/test_users.py new file mode 100644 index 00000000..554cf30d --- /dev/null +++ b/student-tests/test_users.py @@ -0,0 +1,128 @@ +import pytest +from httpx import AsyncClient + +from helpers import PASSWORD, assert_request_id, bearer, create_user, register + + +async def test_register_does_not_leak_password(client: AsyncClient): + response = await client.post( + "/api/v1/users", + json={"username": "alice", "display_name": "Алиса 👋", "password": PASSWORD}, + ) + + assert response.status_code == 201 + body = response.json() + assert set(body) == {"id", "username", "display_name", "created_at"} + assert PASSWORD not in response.text + assert_request_id(response) + + +async def test_get_current_user(client: AsyncClient): + user = await create_user(client, display_name="Алиса 👋") + + response = await client.get("/api/v1/users/me", headers=bearer(user)) + + assert response.status_code == 200 + assert response.json()["id"] == user["id"] + assert response.json()["username"] == user["username"] + assert_request_id(response) + + +@pytest.mark.parametrize( + "body", + [ + {"username": "alice", "display_name": "", "password": PASSWORD}, + {"username": "alice", "display_name": " ", "password": PASSWORD}, + {"username": "alice", "display_name": "a" * 101, "password": PASSWORD}, + {"username": "ab", "display_name": "Alice", "password": PASSWORD}, + {"username": "alice-123", "display_name": "Alice", "password": PASSWORD}, + {"username": "AlicE", "display_name": "Alice", "password": PASSWORD}, + {"username": "a" * 33, "display_name": "Alice", "password": PASSWORD}, + {"username": "alice", "display_name": "Alice"}, + {"username": "alice", "display_name": "Alice", "password": "x" * 11}, + {"username": "alice", "display_name": "Alice", "password": "x" * 129}, + ], +) +async def test_invalid_user_payloads(client: AsyncClient, body: dict): + response = await client.post("/api/v1/users", json=body) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +async def test_password_length_boundaries_are_accepted(client: AsyncClient): + for index, password in enumerate( + ["x" * 12, "y" * 128, "п" * 12, " spaces inside "] + ): + response = await client.post( + "/api/v1/users", + json={ + "username": f"pw_user_{index}", + "display_name": "Alice", + "password": password, + }, + ) + assert response.status_code == 201, (index, response.text) + + +async def test_extra_user_field_is_rejected(client: AsyncClient): + response = await client.post( + "/api/v1/users", + json={ + "username": "alice", + "display_name": "Alice", + "password": PASSWORD, + "extra": True, + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + + +async def test_duplicate_username_returns_conflict(client: AsyncClient): + user = await register(client, username="duplicate_user") + + response = await client.post( + "/api/v1/users", + json={ + "username": user["username"], + "display_name": "Другой", + "password": PASSWORD, + }, + ) + + assert response.status_code == 409 + assert response.json()["error"]["code"] == "conflict" + assert_request_id(response) + + +async def test_user_exact_length_boundaries_are_accepted(client: AsyncClient): + for length in (3, 32): + user = await register(client, username=("a" * length), display_name="d" * 100) + assert len(user["username"]) == length + assert len(user["display_name"]) == 100 + + +@pytest.mark.parametrize("password", [" " * 12, "\t" * 12, " \n" * 6]) +async def test_whitespace_only_password_is_rejected(client: AsyncClient, password: str): + response = await client.post( + "/api/v1/users", + json={"username": "blank_pw", "display_name": "Blank", "password": password}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" + assert_request_id(response) + + +@pytest.mark.parametrize("field", ["display_name", "password"]) +async def test_nul_in_user_fields_is_rejected(client: AsyncClient, field: str): + body = {"username": "nul_user", "display_name": "Nul", "password": PASSWORD} + body[field] = body[field] + "\u0000tail" + + response = await client.post("/api/v1/users", json=body) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_request" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b -- 2.54.0 From 2b9fb3f375d35320ea80ff9db4e852a278c88551 Mon Sep 17 00:00:00 2001 From: gytall Date: Tue, 10 Nov 2026 07:01:17 +0300 Subject: [PATCH 2/6] included rabbitmq as broker --- .env.example | 26 +- Makefile | 34 +- REPORT.md | 256 +++++++++--- app/core/background.py | 33 ++ app/core/cleanup.py | 10 +- app/core/image_processing.py | 14 + app/core/job_admin.py | 65 +++ app/core/job_worker.py | 146 ++++++- app/core/jobs.py | 35 ++ app/core/outbox_publisher.py | 77 +++- app/core/thumbnail_worker.py | 28 ++ app/domain/attachment_jobs.py | 219 ++++++++-- app/domain/attachments.py | 55 ++- app/domain/image_pipeline.py | 95 +++-- app/domain/job_admin.py | 93 +++++ app/domain/job_files.py | 61 +++ app/domain/job_retry.py | 32 ++ app/endpoints/attachments.py | 2 +- app/errors/exceptions.py | 27 ++ app/main.py | 2 +- app/models/__init__.py | 4 +- app/models/attachment.py | 15 +- app/models/job.py | 47 ++- app/models/job_replay.py | 22 + app/schemas/attachment.py | 38 +- app/schemas/job.py | 28 +- app/storage/broker.py | 91 ++++- app/storage/job_admin.py | 46 +++ app/storage/jobs.py | 142 +++++-- app/storage/outbox.py | 38 +- compose.yaml | 121 ++++++ evidence/lab5-jobs.json | 378 ++++++++++++++++++ evidence/lab5-verification.json | 271 +++++++++++++ infra/README.md | 32 +- infra/broker/start-broker.sh | 8 + infra/minio/bootstrap.py | 2 +- .../versions/a62c950e18b3_rabbitmq_cutover.py | 21 + .../c35e91a2f706_async_attachment_jobs.py | 62 +++ .../e7048b6da913_attachment_crop_webp.py | 49 +++ .../versions/f9183d27a640_job_replay.py | 34 ++ requirements.txt | 1 + scripts/check_lab5_jobs.py | 210 ++++++++++ scripts/check_outbox_recovery.py | 83 ++++ scripts/init_local.py | 2 +- scripts/restart.sh | 13 +- student-tests/test_attachments.py | 40 +- student-tests/test_image_pipeline.py | 71 ++++ student-tests/test_jobs.py | 178 +++++++++ student-tests/test_migrations.py | 2 +- 49 files changed, 3072 insertions(+), 287 deletions(-) create mode 100644 app/core/background.py create mode 100644 app/core/image_processing.py create mode 100644 app/core/job_admin.py create mode 100644 app/core/jobs.py create mode 100644 app/core/thumbnail_worker.py create mode 100644 app/domain/job_admin.py create mode 100644 app/domain/job_files.py create mode 100644 app/domain/job_retry.py create mode 100644 app/models/job_replay.py create mode 100644 app/storage/job_admin.py create mode 100644 evidence/lab5-jobs.json create mode 100644 evidence/lab5-verification.json create mode 100755 infra/broker/start-broker.sh create mode 100644 migrations/versions/a62c950e18b3_rabbitmq_cutover.py create mode 100644 migrations/versions/c35e91a2f706_async_attachment_jobs.py create mode 100644 migrations/versions/e7048b6da913_attachment_crop_webp.py create mode 100644 migrations/versions/f9183d27a640_job_replay.py create mode 100644 scripts/check_lab5_jobs.py create mode 100644 scripts/check_outbox_recovery.py create mode 100644 student-tests/test_image_pipeline.py create mode 100644 student-tests/test_jobs.py diff --git a/.env.example b/.env.example index ac8273f4..65196d01 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Скопируйте в .env для локального окружения; .env не коммитится. -# Настройки приложения и Compose; параметры будущей очереди пока не используются. +# Настройки приложения, Compose и очереди lab5. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab @@ -72,10 +72,28 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password MINIO_BROWSER=off -# Lab5: будущие параметры очереди/worker, пока без реализации. +# Lab5: отдельный RabbitMQ broker и ограниченный worker. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 -# Брокер пока не выбран; файл ещё не создаётся и не монтируется. -BROKER_URL_FILE=/run/secrets/broker_url +BROKER_HOST=broker +BROKER_PORT=5672 +BROKER_PASSWORD_FILE=/run/secrets/broker_password +BROKER_QUEUE=attachment-jobs +BROKER_USER=messenger +BROKER_VHOST=/ +JOB_LEASE_SECONDS=90 +JOB_RETRY_SECONDS=5 +PUBLISHER_POLL_SECONDS=1 + +# Качество вариантов и лимиты одного процесса обработки +IMAGE_JPEG_QUALITY=85 +IMAGE_WEBP_QUALITY=80 +IMAGE_WEBP_METHOD=4 +IMAGE_MEMORY_MIB=768 +IMAGE_CPU_SECONDS=20 + +# Retry cap and retained failed sources for explicit replay +JOB_RETRY_MAX_SECONDS=120 +FAILED_STAGING_RETENTION_SECONDS=604800 diff --git a/Makefile b/Makefile index 112af63c..a900c25b 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,8 @@ endef rolling-update failover-all test-lab3-grade4 help: - @echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply" + @echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply" + @echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads" @echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh" @echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh" @echo "Основной: up | down | restart | restore | ps | logs | migrate | instances" @@ -65,10 +66,13 @@ init: test-grade4: $(MAKE) check validate $(MAKE) test-stand-up - $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover - $(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults - $(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10 - $(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)" + $(MAKE) test-docker + $(MAKE) test-stand-smoke + $(MAKE) test-stand-idempotency + $(MAKE) test-stand-persistence + $(MAKE) test-stand-failover + $(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)" + $(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)" test-s3: $(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults @@ -226,3 +230,23 @@ expiry-demo: test-stand-up failure-demo: test-stand-up $(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh + +# --- Очередь lab5 --- +.PHONY: worker-stop worker-start jobs-logs +worker-stop: + $(COMPOSE) stop worker + +worker-start: + $(COMPOSE) start worker + +jobs-logs: + $(COMPOSE) logs -f publisher worker broker + +# Уже работающий изолированный стенд; сам цель его не поднимает. +.PHONY: test-outbox-recovery +test-outbox-recovery: + $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-jobs +test-jobs: + $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 037d0fea..94e40ab9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,75 +1,225 @@ # Отчёт — лабораторная 5 -- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4). -- Целевая оценка: 5; исследовательский трек пока не выбран. -- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5. -- Дата переноса: 07.11.2026. -- Commit/PR lab5: заполнить после публикации. -- Использованные источники: README-файлы и contracts/ репозитория lab5, - исходники lab4. Использована помощь Codex. Новых зависимостей нет. +Автор: Григорян Илья Мурадович, БПИ2303. +Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026. +Дальнейшая цель — 5; исследовательский трек ещё не выбран. +Использована помощь Codex при реализации, проверках и исправлениях. +Commit/PR: заполнить после публикации. -## Запуск из чистого clone +## Запуск -Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск: +Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt. ```sh python3 -m venv .venv .venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt make init make up +make test ``` -HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных. -Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`. -Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда, -`docker compose --env-file .env down --volumes` для основного. -Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4. -Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена. +`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие. +Все environment-параметры Compose подставляются из .env; образец — .env.example. +Пароли и S3 credentials читаются из файлов secrets, не включаются в задания. +API не получает credentials брокера. По умолчанию HTTP доступен на +http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты. -## Архитектура и решения +При обновлении ранее запущенного стенда сначала остановить старые процессы +через `make down` без удаления volumes, затем выполнить `make up`. +Migrate применяет ревизии до a62c950e18b3 перед стартом приложения. +Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые +вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP. +При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs +из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы +обоих брокеров во время перехода должны быть остановлены. -Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии, -идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные -тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO -bootstrap из исходного рабочего дерева. Исходный lab4 не изменён. +## Инфраструктура и устойчивый приём -Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки — -.env.example; там же указаны будущие параметры очереди, пока не используемые. -Compose получает настройки environment из .env, включая пути S3, PostgreSQL -и MinIO к файлам секретов; секретные значения остаются в secrets/. -Пути *_FILE в Compose также задают точки монтирования соответствующих secrets; -MinIO bootstrap читает эти пути из окружения. -Публичные tests/, contracts/, lab.json и README.md lab5 сохранены. +| Компонент | Реализация | Данные | +| --- | --- | --- | +| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий | +| Балансировка | nginx 1.28.0 | единая HTTP-точка входа | +| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit | +| Сессии | Redis 7.4.2, AOF always | redisdata | +| Объекты | приватный MinIO | staging, originals, variants | +| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata | +| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ | +| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL | +| Cleanup | отдельный процесс | незавершённые upload и terminal staging | -Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher → -broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL. -Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready. -Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит. +Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP. +Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером +приёма сообщения (publisher confirm) и только затем фиксирует published_attempt. +При неизвестном результате публикации возможен дубль; намерение остаётся в БД. +Подтверждение worker (consumer ack) следует после устойчивого результата или +устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ +возвращает неподтверждённые доставки в очередь. -## Доказательства критериев +До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет +его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL. +Job одновременно служит transactional outbox — устойчивым намерением публикации. +Worker и broker не входят в синхронный путь upload и readiness API. Остановка +worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT +API проверяет БД, чтобы не удалить оригинал уже принятого задания. -| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство | -| --- | --- | --- | --- | -| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет | -| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 | -| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет | -| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет | -| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет | -| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет | -| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет | -| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет | -| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет | +Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id, +pipeline_version=1, attempt, created_at, schema_version, type и необязательный +trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются. +Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409. +Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены. -## Эксперименты и отказы +## Изображения и ограничения -Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты, -версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность, -прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready, -RSS и другие метрики выбранного трека. +Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без +перекодирования. Все варианты создаются в отдельном ограниченном процессе: -## Ограничения и следующие шаги +| Вариант | Формат и размеры | Параметры | +| --- | --- | --- | +| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha | +| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается | +| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется | -Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и -асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся. -Сначала просмотр пользователем и выбор трека/брокера, затем реализация; -тестовый прогон и новые evidence — по отдельному указанию после реализации. +Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся +в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA. +Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256. + +Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60; +IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс. +Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс; +S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией. + +## Повторы, идемпотентность и очистка + +Состояния: queued → processing → ready; временный отказ возвращает queued, +постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code. +JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit. +Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS, +JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling]. +База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД; +дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при +публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные; +timeout, временные ошибки S3/сети и файловой системы повторяются. + +Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает +истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job, +проверяет актуальность token и lease, держит блокировку до записи original, +трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки. +Устаревшая попытка не может начать запись. Фиксированные ключи по +(attachment_id, pipeline_version, variant) исключают второй набор объектов. +Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один +раз с идемпотентным эффектом, не exactly-once выполнение. + +WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал. +Publisher и worker используют init:true: Python не является PID 1, поэтому +fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу +без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload. + +Queued/processing staging не удаляется. Для ready retention — 86400 секунд; +для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток). +Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым +original, объект сохраняется. Частичные конечные объекты failed автоматически +не удаляются; после replay используются те же ключи. + +## Реестр failed и операторский replay + +Failed registry находится в PostgreSQL. Команды требуют операторского доступа +к Docker/credentials стенда; публичного endpoint для replay нет: + +```sh +docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed +docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID +docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored +docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID +``` + +JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored, +configuration_fixed, manual_review. Actor — аудиторская метка, не система входа. +Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256, +атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не +сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id +с теми же параметрами возвращает already_applied даже после ready; конфликт +параметров отклоняется. После неоднозначного ответа повторяют тот же request-id. +Готовую job с новым request-id повторить нельзя. После удаления staging replay +отклоняется с source_expired; отсутствующий объект даёт source_missing. + +## Проверки уровня 3 и 4 + +Полный воспроизводимый прогон на отдельном стенде: + +```sh +make test-grade4 PYTHON=.venv/bin/python +``` + +Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные +тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две +реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД +для pytest выполняется только в выделенной базе *_test, Redis — DB15. +Публичные tests/ и обязательные contracts/ не изменены. + +Отдельные сценарии на уже работающем изолированном стенде: + +```sh +make test-outbox-recovery BASE_URL=http://localhost:28080 +make test-jobs BASE_URL=http://localhost:28080 +``` + +При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты +в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта, +создают синтетические данные и возвращают штатные worker/publisher в finally. +Перезапуски сохраняют volumes. Сброса пользовательских данных нет. + +Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json; +подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json. +Секреты, tokens и исходные broker payload в evidence не записываются. + +## Фактический результат проверок + +Все перечисленные проверки завершились успешно на отдельном проекте +`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes: + +| Проверка | Результат | +| --- | --- | +| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков | +| Дополнительные timeout и failed retention/SHA-256 | 2 passed | +| Контракт, OpenAPI и Compose config | PASS | +| Публичный smoke | PASS | +| 24 конкурентных повтора сообщения через две API-реплики | PASS | +| Публичные persistence и failover | PASS | +| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload | +| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume | +| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов | +| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified | +| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен | +| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён | +| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 | +| Неизменность публичных tests/contracts/README и whitespace | PASS | + +Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены +в последующие полные запуски make test-grade4. Тесты намеренно меняют только +выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях +используют собственную БД тестового Compose-проекта. + +Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885, +42.633 секунды; все попали в соответствующие интервалы jitter. Задержка +последней failed-попытки сохраняется для диагностики, автоматический retry +после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно. + +Во время проверки исправлена проблема fault hooks с PID 1 через init:true. +Старые собственные проверки lab4 адаптированы к обязательному переходу +202 queued → ready; публичные проверки не изменены. Цель make test-grade4 +переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры +не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий. + +## Границы результата + +Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes. +PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта. +Ожидание в очереди не ограничено; admission control и исследование перегрузки +относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение +алгоритмов требует решения о новой версии/миграции результатов. + +Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), +[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms), +[aio-pika](https://docs.aio-pika.com/quick-start.html), +[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), +[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp). diff --git a/app/core/background.py b/app/core/background.py new file mode 100644 index 00000000..06a4d9a4 --- /dev/null +++ b/app/core/background.py @@ -0,0 +1,33 @@ +"""Остановка фоновых процессов и безопасные сообщения об отказах.""" + +import asyncio +import logging +import signal + +from aio_pika.exceptions import AMQPException, ChannelInvalidStateError +from sqlalchemy.exc import InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError + +TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError) +logger = logging.getLogger(__name__) + + +def shutdown_event() -> asyncio.Event: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + return stop + + +async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None: + try: + async with asyncio.timeout(seconds): + await stop.wait() + except TimeoutError: + pass + + +def log_background_error(operation: str, error: Exception) -> None: + # SDK/DB exceptions могут содержать параметры и credentials; выводим только тип. + logger.warning("%s: %s", operation, type(error).__name__) diff --git a/app/core/cleanup.py b/app/core/cleanup.py index 7e755075..d5201397 100644 --- a/app/core/cleanup.py +++ b/app/core/cleanup.py @@ -6,19 +6,22 @@ from dataclasses import asdict import json from app.core.config import Settings +from app.core.jobs import JobSettings from app.core.database import create_db_engine, create_session_factory from app.core.s3 import S3Settings from app.domain.attachments import reconcile_attachments +from app.domain.attachment_jobs import cleanup_staging from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error from app.storage.s3 import create_s3_storage -async def cleanup_once(factory, storage, apply: bool) -> bool: +async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool: completed = True try: async with factory() as session: result = await reconcile_attachments(session, storage, dry_run=not apply) - print(json.dumps(asdict(result)), flush=True) + staging = await cleanup_staging(session, storage, jobs, apply=apply) + print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True) except RECOVERABLE_STORAGE_ERRORS as error: log_recoverable_error("Reconciliation", error) completed = False @@ -28,12 +31,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool: async def run(once: bool, apply: bool) -> None: settings = Settings() # type: ignore s3_settings = S3Settings() + jobs = JobSettings() engine = create_db_engine(settings) factory = create_session_factory(engine) storage = create_s3_storage(s3_settings) try: while True: - completed = await cleanup_once(factory, storage, apply) + completed = await cleanup_once(factory, storage, apply, jobs) if once: if not completed: raise SystemExit(1) diff --git a/app/core/image_processing.py b/app/core/image_processing.py new file mode 100644 index 00000000..9eb46def --- /dev/null +++ b/app/core/image_processing.py @@ -0,0 +1,14 @@ +"""Качество вариантов и пределы отдельного процесса обработки.""" + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class ImageSettings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore") + + jpeg_quality: int = Field(default=85, ge=1, le=95) + webp_quality: int = Field(default=80, ge=1, le=100) + webp_method: int = Field(default=4, ge=0, le=6) + memory_mib: int = Field(default=768, ge=256, le=1024) + cpu_seconds: int = Field(default=20, ge=1, le=120) diff --git a/app/core/job_admin.py b/app/core/job_admin.py new file mode 100644 index 00000000..369b9f21 --- /dev/null +++ b/app/core/job_admin.py @@ -0,0 +1,65 @@ +"""Операторский интерфейс failed registry. Доступ требует credentials стенда.""" + +import argparse +import asyncio +import json +from uuid import UUID + +from app.core.background import log_background_error +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job +from app.errors.exceptions import JobOperationError +from app.storage.s3 import create_s3_storage + + +async def execute(args): + settings = Settings() + engine = create_db_engine(settings) + storage = None + try: + factory = create_session_factory(engine) + async with factory() as session: + if args.command == "list-failed": + result = await list_failed(session, args.limit) + elif args.command == "show": + result = await show_job(session, args.job_id) + elif args.command == "history": + result = await history(session, args.job_id) + else: + storage = create_s3_storage(settings.s3) + result = await replay_failed_job(session, storage, JobSettings(), args.job_id, + request_id=args.request_id, actor=args.actor, reason=args.reason) + print(json.dumps(result, default=str), flush=True) + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом") + commands = parser.add_subparsers(dest="command", required=True) + listing = commands.add_parser("list-failed") + listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500") + for name in ("show", "history", "replay"): + command = commands.add_parser(name) + command.add_argument("job_id", type=UUID) + if name == "replay": + command.add_argument("--request-id", type=UUID, required=True) + command.add_argument("--actor", required=True) + command.add_argument("--reason", choices=REPLAY_REASONS, required=True) + args = parser.parse_args() + try: + asyncio.run(execute(args)) + except JobOperationError as error: + print(json.dumps({"error_code": error.code}), flush=True) + raise SystemExit(2) from None + except Exception as error: + log_background_error("Job administration failed", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/job_worker.py b/app/core/job_worker.py index 14f36298..4c36d0ff 100644 --- a/app/core/job_worker.py +++ b/app/core/job_worker.py @@ -1,17 +1,137 @@ -"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker. +"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize.""" -Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная -concurrency/prefetch, восстановление pending и остановка с сохранением работы. -Каждая задача получает отдельную AsyncSession. Обработка изображений идёт -в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется. -В Compose и API модуль пока не подключён. -""" +import argparse +import asyncio +from dataclasses import dataclass +import json +import logging +import os +import signal +from uuid import UUID, uuid4 + +from PIL import features +from pydantic import ValidationError + +from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.errors.exceptions import ProcessingError +from app.schemas.job import JobMessage +from app.storage.broker import JobBroker +from app.storage.s3 import create_s3_storage -async def run_worker() -> None: - """Запустить consumer с lease/retry, ограничением ресурсов и корректным ack. +@dataclass(frozen=True) +class Faults: + """Только явные CLI-флаги; обычный worker не включает инъекции сбоев.""" - SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка - worker не заставляет API выполнять resize. Инициализация ресурсов — здесь. - """ - raise NotImplementedError("Lab5: отдельный worker ещё не реализован") + crash_after_object: UUID | None = None + crash_before_ack: UUID | None = None + fail_transient: UUID | None = None + fail_permanent: UUID | None = None + object_name: str = "optimized" + + def kill(self, attachment_id: UUID, phase: str) -> None: + print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True) + os.kill(os.getpid(), signal.SIGKILL) + + def after_object(self, attachment_id: UUID, name: str) -> None: + if attachment_id == self.crash_after_object and name == self.object_name: + self.kill(attachment_id, "after_object:" + name) + + def before_ack(self, attachment_id: UUID) -> None: + if attachment_id == self.crash_before_ack: + self.kill(attachment_id, "before_ack") + + def before_processing(self, attachment_id: UUID) -> None: + if attachment_id == self.fail_transient: + raise TimeoutError("injected transient failure") + if attachment_id == self.fail_permanent: + raise ProcessingError("injected_permanent") + + +async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None: + consumer = uuid4().hex + while not stop.is_set(): + try: + await broker.ensure_ready() + delivery = await broker.receive(consumer) + if delivery is None: + continue + try: + message = JobMessage.model_validate_json(delivery.payload) + except ValidationError: + await broker.reject_invalid(delivery.id) + continue + try: + async with factory() as session: + acknowledge = await process_attachment_job( + session, storage, message, settings, + after_object=faults.after_object, before_processing=faults.before_processing, + ) + except ProcessingError: + await broker.reject_invalid(delivery.id) + else: + if acknowledge: + faults.before_ack(message.attachment_id) + await broker.acknowledge_job(delivery.id) + else: + await broker.defer_job(delivery.id) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Worker delivery deferred", error) + await broker.aclose() + await wait_or_stop(stop, settings.publisher_poll_seconds) + + +async def consume(factory, storage, settings, stop, faults) -> None: + broker = JobBroker(settings) + try: + await consume_connected(factory, storage, broker, settings, stop, faults) + finally: + await broker.aclose() + + +async def run_worker(faults: Faults) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + if not features.check("webp"): + logging.error("Pillow requires WebP support for the image worker") + raise RuntimeError("WebP codec is unavailable") + stop = shutdown_event() + engine = create_db_engine(settings) + storage = None + try: + storage = create_s3_storage(settings.s3) + factory = create_session_factory(engine) + async with asyncio.TaskGroup() as group: + tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)] + await stop.wait() + # Отмена убивает resize-процесс, дожидается SDK и не отправляет ack. + for task in tasks: + task.cancel() + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks") + modes = parser.add_mutually_exclusive_group() + for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"): + modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID") + parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized") + args = parser.parse_args() + faults = Faults(**vars(args)) + try: + asyncio.run(run_worker(faults)) + except Exception as error: + log_background_error("Worker stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/jobs.py b/app/core/jobs.py new file mode 100644 index 00000000..e7badd3e --- /dev/null +++ b/app/core/jobs.py @@ -0,0 +1,35 @@ +"""Настройки очереди из .env; credentials брокера читаются отдельно из файла.""" + +from pathlib import Path +from typing import Self + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class JobSettings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + broker_host: str = "broker" + broker_port: int = Field(default=5672, ge=1, le=65535) + broker_password_file: Path = Path("/run/secrets/broker_password") + broker_queue: str = "attachment-jobs" + broker_user: str = "messenger" + broker_vhost: str = "/" + worker_concurrency: int = Field(default=2, ge=1, le=4) + job_max_attempts: int = Field(default=5, ge=1, le=100) + job_timeout_seconds: int = Field(default=60, ge=10, le=300) + job_lease_seconds: int = Field(default=90, ge=20, le=600) + job_retry_seconds: int = Field(default=5, ge=1, le=300) + job_retry_max_seconds: int = Field(default=120, ge=1, le=3600) + failed_staging_retention_seconds: int = Field(default=604800, ge=300) + publisher_poll_seconds: float = Field(default=1, gt=0, le=10) + staging_retention_seconds: int = Field(default=86400, ge=300) + + @model_validator(mode="after") + def validate_lease(self) -> Self: + if self.job_lease_seconds < self.job_timeout_seconds + 30: + raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд") + if self.job_retry_max_seconds < self.job_retry_seconds: + raise ValueError("Retry maximum must be at least the base delay") + return self diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py index 04f2cf58..8d23e461 100644 --- a/app/core/outbox_publisher.py +++ b/app/core/outbox_publisher.py @@ -1,15 +1,70 @@ -"""Заготовка процесса доставки PostgreSQL outbox в durable broker. +"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI.""" -Не держать транзакцию/блокировки во время бесконечного сетевого ожидания. -Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish. -Процесс пока не подключён к Compose или жизненному циклу API. -""" +import argparse +import asyncio +import json +import logging +import os +import signal +from uuid import UUID + +from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs +from app.storage.broker import JobBroker -async def run_publisher() -> None: - """Публиковать сохранённые намерения и фиксировать подтверждение брокера. +def crash_before_publish(attachment_id: UUID) -> None: + """Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish.""" + print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True) + os.kill(os.getpid(), signal.SIGKILL) - После crash между publish и отметкой допустим дубль того же job_id. - Недоступность брокера не должна терять уже принятые задания. - """ - raise NotImplementedError("Lab5: outbox publisher ещё не реализован") + +async def run_publisher(crash_attachment_id: UUID | None = None) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + stop = shutdown_event() + engine = create_db_engine(settings) + broker = None + try: + broker = JobBroker(jobs) + factory = create_session_factory(engine) + while not stop.is_set(): + published = False + try: + await broker.ensure_ready() + async with factory() as session: + await recover_expired_jobs(session, jobs) + published = await publish_next_job( + session, broker, attachment_id=crash_attachment_id, + before_publish=crash_before_publish if crash_attachment_id is not None else None, + ) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Publisher deferred", error) + if not published: + await wait_or_stop(stop, jobs.publisher_poll_seconds) + finally: + if broker is not None: + await broker.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ") + parser.add_argument( + "--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID", + help="аварийно завершить этот процесс перед публикацией только указанного вложения", + ) + args = parser.parse_args() + try: + asyncio.run(run_publisher(args.crash_before_publish)) + except Exception as error: + log_background_error("Publisher stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py new file mode 100644 index 00000000..cc3e2be1 --- /dev/null +++ b/app/core/thumbnail_worker.py @@ -0,0 +1,28 @@ +"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU.""" + +from pathlib import Path +import resource +import sys + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code + + +def main() -> int: + settings = ImageSettings() + memory = settings.memory_mib * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)) + source, directory, metadata, content_type, maximum = sys.argv[1:] + code = 0 + try: + result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings) + Path(metadata).write_text(result.model_dump_json()) + except IMAGE_WORKER_ERRORS as error: + code = pipeline_worker_error_code(error) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py index 01deb9bb..5f6d7dcf 100644 --- a/app/domain/attachment_jobs.py +++ b/app/domain/attachment_jobs.py @@ -1,45 +1,198 @@ -"""Заготовки сценариев lab5; не подключены к upload или worker. +"""Приём, durable публикация, попытки и фиксация результата обработки.""" -Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются -в storage/. Одна AsyncSession принадлежит одной последовательной задаче. -""" - -from uuid import UUID +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 +from minio.error import MinioException from sqlalchemy.ext.asyncio import AsyncSession +from urllib3.exceptions import HTTPError as StorageHTTPError + +from app.core.jobs import JobSettings +from app.domain.job_retry import classify_failure, retry_delay +from app.domain.job_files import fetch_original, render_variants +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import VARIANT_FILENAMES +from app.schemas.job import JobMessage +from app.storage import jobs, outbox +from app.storage.broker import JobBroker +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage -async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID: - """Создать job и outbox в транзакции queued metadata после durable staging. - - Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload; - 202 допустим только после устойчивого оригинала и успешного commit. - """ - raise NotImplementedError("Lab5: постановка задания ещё не реализована") +@dataclass(frozen=True) +class Claim: + work: JobWork | None = None + acknowledge: bool = False -async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None: - """Захватить job с lease и опубликовать original и обязательные варианты. - - Ready-дубль не меняет результат. Lease и версия защищают от устаревшей - попытки. S3/metadata фиксируются до ack; отмена не подавляется. - """ - raise NotImplementedError("Lab5: обработка задания ещё не реализована") +async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: + """Включить намерение обработки в транзакцию queued metadata; без commit.""" + return await jobs.create_job(session, attachment_id, staging_key) -async def recover_expired_jobs(session: AsyncSession) -> int: - """Вернуть зависший processing в очередь после истечения lease. - - Учитывать лимит попыток и защиту от конкурентного завершения старым worker. - Точное правило heartbeat/deadline определяется до реализации. - """ - raise NotImplementedError("Lab5: восстановление lease ещё не реализовано") +async def publish_next_job( + session: AsyncSession, broker: JobBroker, *, + attachment_id: UUID | None = None, + before_publish: Callable[[UUID], None] | None = None, +) -> bool: + """Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль.""" + job = await outbox.next_unpublished_job(session, attachment_id=attachment_id) + published = job is not None + if job is not None: + if before_publish is not None: + before_publish(job.attachment_id) + await broker.publish_job(JobMessage( + job_id=job.id, attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, attempt=job.attempt + 1, + created_at=job.created_at, + )) + outbox.mark_published(job) + await session.commit() + return published -async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None: - """Явно повторить только failed job, сохранив аудит и ключ логического эффекта. +async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim: + """Дубликат не захватывает живую аренду и не откатывает terminal-состояние.""" + row = await jobs.lock_job(session, message.job_id) + result = Claim() + if row is None: + raise ProcessingError("unknown_job") + job, attachment = row + if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version: + raise ProcessingError("invalid_job") + now = await jobs.database_time(session) + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + if attachment.status in {"ready", "failed"}: + result = Claim(acknowledge=True) + elif message.attempt <= job.attempt and attachment.status == "queued": + # Предыдущая попытка уже устойчиво заменена новой в PostgreSQL. + result = Claim(acknowledge=True) + elif message.attempt < job.attempt: + result = Claim(acknowledge=True) + elif message.attempt > job.attempt + 1: + raise ProcessingError("invalid_attempt") + elif attachment.status == "queued" and job.next_attempt_at <= now: + if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None: + jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0) + result = Claim(acknowledge=True) + else: + result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds)) + await session.commit() + return result - Не переисполнять ready. Проверка допустимости, лимиты повторов и способ - операторского вызова будут определены вместе с реестром failed/DLQ. - """ - raise NotImplementedError("Lab5: replay ещё не реализован") + +async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None: + # Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется. + await session.rollback() + row = await jobs.lock_job(session, work.job_id) + if row is not None: + job, attachment = row + if attachment.status == "processing" and job.lease_token == work.lease_token: + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, attachment, await jobs.database_time(session), + "retry_exhausted" if exhausted and not permanent else code, + terminal=permanent or exhausted, + retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + ) + job.last_error_code = code + job.last_error_permanent = permanent + await session.commit() + + +async def build_result( + session: AsyncSession, storage: MinioStorage, work: JobWork, + after_object: Callable[[UUID, str], None] | None = None, +) -> bool: + """Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки.""" + with TemporaryDirectory(prefix="messenger-job-") as directory: + source = Path(directory) / "original" + await fetch_original(storage, work, source) + result = await render_variants(storage, work, source, Path(directory)) + row = await jobs.lock_job(session, work.job_id) + if row is None: + await session.rollback() + return True + job, attachment = row + now = await jobs.database_time(session) + if attachment.status != "processing" or job.lease_token != work.lease_token: + await session.rollback() + return True + if job.lease_until is None or job.lease_until <= now: + await session.rollback() + return False + # Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED. + # При отмене S3 adapter дожидается SDK-потока до освобождения блокировки. + await storage.put_file( + work.original_key, source, size_bytes=work.size_bytes, + content_type=work.content_type, sha256=work.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, "original") + for variant in result.variants: + await storage.put_file( + work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name], + size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, variant.name) + jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session)) + await session.commit() + return True + + +async def process_attachment_job( + session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings, + *, after_object: Callable[[UUID, str], None] | None = None, + before_processing: Callable[[UUID], None] | None = None, +) -> bool: + claim = await claim_job(session, message, settings) + acknowledge = claim.acknowledge + if claim.work is not None: + work = claim.work + try: + async with asyncio.timeout(settings.job_timeout_seconds): + if before_processing is not None: + before_processing(work.attachment_id) + acknowledge = await build_result(session, storage, work, after_object) + except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error: + code, permanent = classify_failure(error) + await save_failure(session, work, settings, code, permanent=permanent) + acknowledge = True + # Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку. + return acknowledge + + +async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int: + rows = await jobs.expired_jobs(session) + now = await jobs.database_time(session) + for job, attachment in rows: + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout", + terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + ) + job.last_error_code = "worker_timeout" + job.last_error_permanent = False + await session.commit() + return len(rows) + + +async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int: + rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds) + if apply: + for job, attachment in rows: + if job.staging_key != attachment.object_key: + await storage.delete_object(job.staging_key) + jobs.clear_staging(job) + await session.commit() + else: + await session.rollback() + return len(rows) diff --git a/app/domain/attachments.py b/app/domain/attachments.py index 3216c636..80caf109 100644 --- a/app/domain/attachments.py +++ b/app/domain/attachments.py @@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.request_capacity import RequestCapacity from app.domain.chats import ensure_chat_member +from app.domain.attachment_jobs import enqueue_attachment_job from app.domain.images import validate_filename, validate_image from app.errors.exceptions import ( conflict, @@ -28,7 +29,7 @@ from app.errors.handlers import ( log_recoverable_error, operation_timeout, ) -from app.schemas.attachment import Attachment +from app.schemas.attachment import Attachment, VARIANT_FILENAMES from app.schemas.user import User from app.storage import attachments as repository from app.storage.s3 import MinioStorage @@ -151,7 +152,7 @@ async def recover_upload( await session.rollback() -async def publish_original( +async def stage_attachment( session: AsyncSession, storage: MinioStorage, user: User, @@ -161,8 +162,9 @@ async def publish_original( filename: str, content_type: str, ) -> Attachment: + """Сохранить исходник в staging и атомарно принять metadata/job.""" id = uuid4() - key = f"originals/{uuid4().hex}" + key = f"staging/{uuid4().hex}" # Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT. await repository.reserve_upload(session, id, key) await session.commit() @@ -181,13 +183,16 @@ async def publish_original( id=id, chat_id=chat_id, owner_id=user.id, - object_key=key, + object_key=f"originals/{id.hex}/v1", + status="queued", filename=filename, content_type=content_type, size_bytes=original.size_bytes, sha256=original.sha256, ) + await enqueue_attachment_job(session, id, key) await repository.finish_upload(session, id) + # Снимок queued строится до COMMIT, даже если worker очень быстрый. result = Attachment.model_validate(record) await session.commit() except BaseException: @@ -223,7 +228,7 @@ async def upload_attachment( settings.chunk_bytes, ) await validate_image(original.path, content_type, settings) - result = await publish_original( + result = await stage_attachment( session, storage, user, @@ -245,6 +250,13 @@ async def get_attachment( return Attachment.model_validate(record) +@dataclass(frozen=True) +class DownloadInfo: + filename: str + content_type: str + size_bytes: int + + async def download_attachment( session: AsyncSession, storage: MinioStorage | None, @@ -252,25 +264,30 @@ async def download_attachment( attachment_id: UUID, *, variant: str = "original", -) -> tuple[Attachment, AsyncIterator[bytes]]: +) -> tuple[DownloadInfo, AsyncIterator[bytes]]: result = await get_attachment(session, user, attachment_id) - if variant != "original": - raise not_found() + if result.status != "ready": + raise conflict("Вложение пока недоступно для скачивания") storage = require_storage(storage) record = await repository.get_attachment(session, attachment_id) if record is None: raise not_found() - key = record.object_key + if variant == "original": + key = record.object_key + info = DownloadInfo(result.filename, result.content_type, result.size_bytes) + else: + selected = next((item for item in result.variants if item.name == variant), None) + keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key} + key = keys.get(variant) + if selected is None or key is None: + raise not_found() + info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes) await session.rollback() - info = await storage.stat_object(key) - if ( - info is None - or info.size_bytes != result.size_bytes - or info.content_type != result.content_type - ): - raise unavailable("Оригинал временно недоступен") + stored = await storage.stat_object(key) + if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type: + raise unavailable("Файл временно недоступен") stream = await storage.open_object(key) - return result, stream + return info, stream async def validate_message_attachments( @@ -283,8 +300,8 @@ async def validate_message_attachments( records = await repository.get_for_chat(session, chat_id, attachment_ids) if len(records) != len(attachment_ids): raise not_found() - if any(record.status != "ready" for record in records): - raise conflict("Вложение ещё не готово") + if any(record.status == "failed" for record in records): + raise conflict("Обработка вложения завершилась ошибкой") async def reconcile_attachments( diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py index 6dffbc03..823ade9d 100644 --- a/app/domain/image_pipeline.py +++ b/app/domain/image_pipeline.py @@ -1,34 +1,79 @@ -"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP. - -Pipeline запускается внутри ограниченного процесса worker. Исходные bytes -не перекодируются; варианты удаляют метаданные по документированной политике. -Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md. -""" +"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка.""" +import hashlib from pathlib import Path +from PIL import Image, ImageOps -def create_thumbnail(source: Path, destination: Path) -> None: - """Создать JPEG, вписанный в 256×256 без увеличения. - - Применить EXIF orientation, alpha положить на белый; округлять размеры - вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160. - """ - raise NotImplementedError("Lab5: thumbnail ещё не реализован") +from app.core.image_processing import ImageSettings +from app.domain.image_validation import decode_image +from app.schemas.attachment import PipelineResult, Variant -def create_crop(source: Path, destination: Path) -> None: - """Создать центрированный квадрат JPEG 128×128; увеличение допустимо. - - Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail. - """ - raise NotImplementedError("Lab5: crop ещё не реализован") +def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]: + """Вписать без увеличения, округляя вниз, минимум один пиксель.""" + longest = max(size) + result = size + if longest > maximum: + result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest)) + return result -def create_optimized(source: Path, destination: Path) -> None: - """Создать WebP, вписанный в 1280×1280 без увеличения. +def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant: + checksum = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + checksum.update(chunk) + return Variant( + name=name, content_type=content_type, size_bytes=path.stat().st_size, + width=size[0], height=size[1], sha256=checksum.hexdigest(), + ) - Удалить метаданные и применить описанную политику качества. Уменьшение - размера любого маленького исходника не гарантируется контрактом. - """ - raise NotImplementedError("Lab5: optimized ещё не реализован") + +def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None: + """Белая подложка и новый image без EXIF/ICC/XMP/текстовых метаданных.""" + with Image.new("RGB", image.size, "white") as clean: + with image.getchannel("A") as alpha: + clean.paste(image, mask=alpha) + clean.save(destination, format="JPEG", quality=quality, exif=b"") + + +def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant: + size = fitted_size(image.size, 256) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + save_jpeg(resized, destination, quality) + return describe_variant("thumbnail", "image/jpeg", destination, size) + + +def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant: + """Центрированный JPEG 128×128; маленький исходник допускает увеличение.""" + with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop: + save_jpeg(crop, destination, quality) + return describe_variant("crop", "image/jpeg", destination, (128, 128)) + + +def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant: + """WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются.""" + size = fitted_size(image.size, 1280) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + with Image.new("RGBA", size) as clean: + clean.paste(resized) + clean.save( + destination, format="WEBP", lossless=False, + quality=settings.webp_quality, method=settings.webp_method, + exif=b"", icc_profile=b"", xmp=b"", + ) + return describe_variant("optimized", "image/webp", destination, size) + + +def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult: + decode_image(source, content_type, max_pixels) + with Image.open(source, formats=["PNG", "JPEG"]) as image: + ImageOps.exif_transpose(image, in_place=True) + with image.convert("RGBA") as pixels: + variants = [ + create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality), + create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality), + create_optimized(pixels, directory / "optimized.webp", settings), + ] + return PipelineResult(variants=variants) diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py new file mode 100644 index 00000000..a1a3494e --- /dev/null +++ b/app/domain/job_admin.py @@ -0,0 +1,93 @@ +"""Replay только failed с проверенным исходником и ограниченным новым бюджетом.""" + +import asyncio +import re +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from minio.error import S3Error + +from app.domain.job_files import fetch_original +from app.errors.exceptions import JobOperationError, ProcessingError +from app.storage import job_admin, jobs +from app.storage.jobs import JobWork + +REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review") + + +def snapshot(job, attachment): + return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status, + attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt, + error_code=attachment.error_code, last_error_code=job.last_error_code, + last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None, + next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at) + + +async def list_failed(session, limit): + return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)] + + +async def show_job(session, job_id): + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + result = snapshot(*row) + await session.rollback() + return result + + +async def history(session, job_id): + return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason, + previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit, + previous_error_code=row.previous_error_code, created_at=row.created_at) + for row in await job_admin.replay_history(session, job_id)] + + +def same_request(record, job_id, actor, reason): + if record.job_id != job_id or record.actor != actor or record.reason != reason: + raise JobOperationError("request_conflict") + return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True} + + +async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason): + if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS: + raise JobOperationError("invalid_audit_fields") + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + record = await job_admin.replay_record(session, request_id) + if record is not None: + result = same_request(record, job_id, actor, reason) + await session.rollback() + return result + job, attachment = row + if attachment.status != "failed": + raise JobOperationError("job_not_failed") + if job.staging_key is None: + raise JobOperationError("source_expired") + # Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT. + work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key, + attachment.content_type, attachment.size_bytes, attachment.sha256) + try: + async with asyncio.timeout(settings.job_timeout_seconds): + with TemporaryDirectory(prefix="replay-source-") as directory: + await fetch_original(storage, work, Path(directory) / "original") + except ProcessingError as error: + raise JobOperationError(error.code) from None + except S3Error as error: + if error.code in {"NoSuchKey", "NoSuchObject"}: + raise JobOperationError("source_missing") from None + raise + limit = job.attempt + settings.job_max_attempts + inserted = await job_admin.insert_replay( + session, id=request_id, job_id=job_id, actor=actor, reason=reason, + previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code, + ) + if not inserted: + result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason) + await session.rollback() + return result + job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit) + await session.commit() + return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit} diff --git a/app/domain/job_files.py b/app/domain/job_files.py new file mode 100644 index 00000000..b48950d2 --- /dev/null +++ b/app/domain/job_files.py @@ -0,0 +1,61 @@ +"""Получение исходника и запуск изолированного процесса обработки вариантов.""" + +import asyncio +import errno +import hashlib +from pathlib import Path +import sys + +from anyio import CancelScope + +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import PipelineResult +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage + + +async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None: + if work.size_bytes > storage.settings.max_upload_bytes: + raise ProcessingError("image_limits") + checksum = hashlib.sha256() + size = 0 + stream = await storage.open_object(work.staging_key) + try: + with destination.open("wb") as target: + async for chunk in stream: + size += len(chunk) + if size > work.size_bytes: + raise ProcessingError("source_mismatch") + target.write(chunk) + checksum.update(chunk) + finally: + await stream.aclose() + if size != work.size_bytes or checksum.hexdigest() != work.sha256: + raise ProcessingError("source_mismatch") + + +async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult: + metadata = directory / "variants.json" + process = await asyncio.create_subprocess_exec( + sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory), + str(metadata), work.content_type, str(storage.settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, + ) + try: + code = await process.wait() + finally: + if process.returncode is None: + process.kill() + with CancelScope(shield=True): + await process.wait() + if code == 2: + raise ProcessingError("invalid_image") + if code == 3 or code < 0: + raise ProcessingError("image_limits") + if code == 4: + raise OSError(errno.EIO, "Pipeline filesystem unavailable") + if code == 5: + raise ProcessingError("processing_configuration") + if code != 0: + raise RuntimeError("Image pipeline subprocess failed") + return PipelineResult.model_validate_json(metadata.read_text()) diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py new file mode 100644 index 00000000..2d9620df --- /dev/null +++ b/app/domain/job_retry.py @@ -0,0 +1,32 @@ +"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой.""" + +import errno +import random + +from minio.error import S3Error + +from app.core.jobs import JobSettings +from app.errors.exceptions import ProcessingError + + +def retry_delay(settings: JobSettings, cycle_attempt: int) -> float: + ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1))) + return random.uniform(ceiling / 2, ceiling) + + +def classify_failure(error: Exception) -> tuple[str, bool]: + if isinstance(error, ProcessingError): + return error.code, True + if isinstance(error, S3Error): + if error.code in {"NoSuchKey", "NoSuchObject"}: + return "source_missing", True + if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}: + return "storage_configuration", True + if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}: + return "storage_unavailable", False + return "storage_rejected", True + if isinstance(error, TimeoutError): + return "processing_timeout", False + if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + return "processing_configuration", True + return "processing_unavailable", False diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py index 94255c0c..8387b3e2 100644 --- a/app/endpoints/attachments.py +++ b/app/endpoints/attachments.py @@ -15,7 +15,7 @@ from app.schemas.attachment import Attachment router = APIRouter(prefix='/api/v1', tags=['attachments']) -@router.post('/chats/{chat_id}/attachments', status_code=201) +@router.post('/chats/{chat_id}/attachments', status_code=202) async def upload( request: Request, chat_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage, capacity: UploadCapacity, diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py index b434529e..59d06a96 100644 --- a/app/errors/exceptions.py +++ b/app/errors/exceptions.py @@ -1,3 +1,5 @@ +import errno + from collections.abc import Mapping from PIL import Image, UnidentifiedImageError @@ -107,3 +109,28 @@ def image_validation_error(code: int) -> ApiError: if code == 3 or code < 0: error = payload_too_large("Превышен лимит пикселей или ресурсов изображения") return error + + +class ProcessingError(Exception): + """Постоянная ошибка pipeline с безопасным машинным кодом.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class JobOperationError(Exception): + """Безопасный код отказа операторской команды.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +def pipeline_worker_error_code(error: Exception) -> int: + """Отделить ошибки файловой системы от ошибки декодирования изображения.""" + if isinstance(error, OSError) and error.errno is not None: + if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + return 5 + return 4 + return image_worker_error_code(error) diff --git a/app/main.py b/app/main.py index 7bf806c9..de683883 100644 --- a/app/main.py +++ b/app/main.py @@ -45,7 +45,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: await redis.aclose() await engine.dispose() - application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan) + application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan) application.state.settings = settings application.state.engine = engine diff --git a/app/models/__init__.py b/app/models/__init__.py index 6679c3ef..43097851 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,4 +1,6 @@ +from app.models.job_replay import JobReplay from app.models.base import Base +from app.models.job import AttachmentJob from app.models.attachment import Attachment, MessageAttachment, UploadIntent from app.models.chat import Chat from app.models.chat_member import ChatMember @@ -7,4 +9,4 @@ from app.models.idempotency_key import IdempotencyKey from app.models.message import Message from app.models.user import User -__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] +__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] diff --git a/app/models/attachment.py b/app/models/attachment.py index 3b7c678d..24d075a2 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -3,7 +3,8 @@ import uuid from datetime import datetime -from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column from app.models.base import Base @@ -17,7 +18,8 @@ class Attachment(Base): "content_type IN ('image/png', 'image/jpeg')", name="content_type" ), CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), - CheckConstraint("status = 'ready'", name="status"), + CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"), + CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"), Index("ix_attachments_chat_id", "chat_id"), ) @@ -29,7 +31,12 @@ class Attachment(Base): content_type: Mapped[str] = mapped_column(String(32)) size_bytes: Mapped[int] sha256: Mapped[str] = mapped_column(String(64)) - status: Mapped[str] = mapped_column(String(16), server_default="ready") + status: Mapped[str] = mapped_column(String(16), server_default="queued") + error_code: Mapped[str | None] = mapped_column(String(64)) + thumbnail_key: Mapped[str | None] = mapped_column(String(128)) + crop_key: Mapped[str | None] = mapped_column(String(128)) + optimized_key: Mapped[str | None] = mapped_column(String(128)) + variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb")) created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) @@ -47,7 +54,7 @@ class MessageAttachment(Base): class UploadIntent(Base): - """Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных.""" + """Фиксируется до PUT; удаляется атомарно с принятием queued metadata/job.""" __tablename__ = "upload_intents" __table_args__ = (Index("ix_upload_intents_created_at", "created_at"),) diff --git a/app/models/job.py b/app/models/job.py index 95a1af01..00ae6a20 100644 --- a/app/models/job.py +++ b/app/models/job.py @@ -1,16 +1,37 @@ -"""Место для будущих ORM-моделей job, outbox и вариантов lab5. +"""Устойчивая работа и намерение публикации; один pipeline на вложение.""" -Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4 -до реализации миграции. Планируемые данные: +from datetime import datetime +from uuid import UUID, uuid4 -* job: job_id, attachment_id, pipeline_version, attempt/max_attempts, - status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps; -* outbox: event_id, job_id, payload/version, состояние публикации и её lease; -* variant: attachment_id, pipeline_version, variant, приватный object_key, - content_type, size_bytes, sha256, размеры; -* replay audit: операторское действие и связь с прежней логической job. +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column -Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker -не публикует metadata поверх новой версии; точная схема зависит от трека 5. -Изменение ready-only constraints attachments — отдельная новая миграция. -""" +from app.models.base import Base + + +class AttachmentJob(Base): + __tablename__ = "attachment_jobs" + __table_args__ = ( + CheckConstraint("pipeline_version = 1", name="pipeline_version"), + CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"), + Index("ix_attachment_jobs_due", "next_attempt_at"), + Index("ix_attachment_jobs_lease", "lease_until"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey("attachments.id"), unique=True, + ) + staging_key: Mapped[str | None] = mapped_column(String(128)) + pipeline_version: Mapped[int] = mapped_column(server_default="1") + attempt: Mapped[int] = mapped_column(server_default="0") + cycle_start_attempt: Mapped[int] = mapped_column(server_default="0") + attempt_limit: Mapped[int | None] + last_error_permanent: Mapped[bool | None] + published_attempt: Mapped[int] = mapped_column(server_default="0") + next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + lease_token: Mapped[UUID | None] + lease_until: Mapped[datetime | None] + last_error_code: Mapped[str | None] = mapped_column(String(64)) + finished_at: Mapped[datetime | None] + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job_replay.py b/app/models/job_replay.py new file mode 100644 index 00000000..7f2b27af --- /dev/null +++ b/app/models/job_replay.py @@ -0,0 +1,22 @@ +"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность.""" + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ForeignKey, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class JobReplay(Base): + __tablename__ = "job_replays" + + id: Mapped[UUID] = mapped_column(primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True) + actor: Mapped[str] = mapped_column(String(64)) + reason: Mapped[str] = mapped_column(String(32)) + previous_attempt: Mapped[int] + attempt_limit: Mapped[int] + previous_error_code: Mapped[str | None] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py index 74574dd9..93911c14 100644 --- a/app/schemas/attachment.py +++ b/app/schemas/attachment.py @@ -1,14 +1,25 @@ -"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту.""" +"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются.""" from datetime import datetime from typing import Literal from uuid import UUID -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from app.schemas.base import Base +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid", from_attributes=True) + + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int = Field(ge=1) + width: int = Field(ge=1) + height: int = Field(ge=1) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + class Attachment(Base): chat_id: UUID owner_id: UUID @@ -16,8 +27,25 @@ class Attachment(Base): content_type: Literal["image/png", "image/jpeg"] size_bytes: int = Field(ge=1, le=10_485_760) sha256: str = Field(pattern=r"^[a-f0-9]{64}$") - status: Literal["ready"] = "ready" - variants: list[object] = Field(default_factory=list, max_length=0) - error_code: None = None + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] = Field(default_factory=list) + error_code: str | None = None created_at: datetime updated_at: datetime + + +VARIANT_FILENAMES = { + "thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp", +} + + +class PipelineResult(BaseModel): + model_config = ConfigDict(extra="forbid") + variants: list[Variant] = Field(min_length=3, max_length=3) + + @field_validator("variants") + @classmethod + def require_all_variants(cls, variants: list[Variant]) -> list[Variant]: + if {item.name for item in variants} != set(VARIANT_FILENAMES): + raise ValueError("Нужны thumbnail, crop и optimized без повторов") + return variants diff --git a/app/schemas/job.py b/app/schemas/job.py index d9c71ca2..7fb7f341 100644 --- a/app/schemas/job.py +++ b/app/schemas/job.py @@ -1,10 +1,20 @@ -"""Место для будущей схемы сообщения contracts/jobs.schema.json. +"""Сообщение брокера: только идентификаторы и служебные поля контракта v1.""" -Пока классы не реализованы и сообщения не принимаются/не публикуются. -Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id, -pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339). -Допустимы traceparent и tracestate; дополнительные поля запрещены. -job_id сохраняется при redelivery; очередная управляемая попытка отражается -в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются. -Pydantic-схема должна соответствовать публичному JSON Schema, не менять его. -""" +from typing import Literal +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + type: Literal["attachment.process.v1"] = "attachment.process.v1" + job_id: UUID + attachment_id: UUID + pipeline_version: int = Field(default=1, ge=1) + attempt: int = Field(ge=1) + created_at: AwareDatetime + traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + tracestate: str | None = Field(default=None, max_length=512) diff --git a/app/storage/broker.py b/app/storage/broker.py index 3145d2f8..7358a523 100644 --- a/app/storage/broker.py +++ b/app/storage/broker.py @@ -1,27 +1,78 @@ -"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран. +"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack.""" -Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные -поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity, -формат delivery handle и интерфейсы уточнить после выбора брокера/версии. -Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/. -""" +import asyncio +import logging +from dataclasses import dataclass -from collections.abc import Mapping +import aio_pika +from aio_pika.abc import AbstractIncomingMessage + +from app.core.jobs import JobSettings +from app.schemas.job import JobMessage -async def publish_job(payload: Mapping[str, object]) -> str: - """Опубликовать устойчивое сообщение и вернуть идентификатор доставки. - - Дождаться подтверждения сохранения брокером; неопределённый исход разрешает - повтор с тем же job_id. Не считать публикацию обработкой изображения. - """ - raise NotImplementedError("Lab5: публикация в брокер ещё не реализована") +@dataclass(frozen=True) +class Delivery: + id: AbstractIncomingMessage + payload: bytes -async def acknowledge_job(delivery_id: str) -> None: - """Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля. +class JobBroker: + """Один consumer/канал на экземпляр; publisher использует отдельный экземпляр.""" - Retry сначала устойчиво планируется, затем закрывается старая доставка. - Конкретный delivery handle зависит от брокера; сигнатура предварительная. - """ - raise NotImplementedError("Lab5: ack ещё не реализован") + def __init__(self, settings: JobSettings) -> None: + self.settings = settings + self.connection = self.channel = self.queue = self.iterator = None + # Transport exceptions могут содержать URL с credentials. + logging.getLogger("aio_pika").setLevel(logging.CRITICAL) + logging.getLogger("aiormq").setLevel(logging.CRITICAL) + + async def ensure_ready(self) -> None: + if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed: + await self.aclose() + self.connection = await aio_pika.connect( + host=self.settings.broker_host, port=self.settings.broker_port, + login=self.settings.broker_user, + password=self.settings.broker_password_file.read_text().strip(), + virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30, + ) + self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) + await self.channel.set_qos(prefetch_count=1) + self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True) + + async def publish_job(self, message: JobMessage) -> None: + await self.ensure_ready() + await self.channel.default_exchange.publish( + aio_pika.Message( + body=message.model_dump_json(exclude_none=True).encode(), + content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=f"{message.job_id}:{message.attempt}", + ), routing_key=self.settings.broker_queue, mandatory=True, timeout=10, + ) + + async def receive(self, consumer: str) -> Delivery: + await self.ensure_ready() + if self.iterator is None: + self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer) + await self.iterator.__aenter__() + message = await self.iterator.__anext__() + return Delivery(message, message.body) + + async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None: + await delivery_id.ack() + + async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None: + # Живая чужая lease: не удерживать единственный prefetch slot навсегда. + await asyncio.sleep(self.settings.publisher_poll_seconds) + await delivery_id.nack(requeue=True) + + async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None: + # Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ. + logging.getLogger(__name__).warning("Rejected invalid job envelope") + await delivery_id.reject(requeue=False) + + async def aclose(self) -> None: + connection = self.connection + self.connection = self.channel = self.queue = self.iterator = None + if connection is not None and not connection.is_closed: + await connection.close() diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py new file mode 100644 index 00000000..87782c24 --- /dev/null +++ b/app/storage/job_admin.py @@ -0,0 +1,46 @@ +"""Операторские выборки и журнал replay; транзакция принадлежит domain.""" + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert + +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay + + +async def failed_jobs(session, limit: int): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == "failed") + .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit) + ) + return result.all() + + +async def replay_record(session, request_id): + return await session.get(JobReplay, request_id) + + +async def insert_replay(session, **values) -> bool: + result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id)) + return result is not None + + +async def replay_history(session, job_id): + return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all() + + +def reset_failed(job, attachment, now, attempt_limit): + job.cycle_start_attempt = job.attempt + job.attempt_limit = attempt_limit + job.published_attempt = job.attempt + job.next_attempt_at = now + job.lease_token = None + job.lease_until = None + job.finished_at = None + job.last_error_code = None + job.last_error_permanent = None + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now diff --git a/app/storage/jobs.py b/app/storage/jobs.py index 2f35056e..d67545b7 100644 --- a/app/storage/jobs.py +++ b/app/storage/jobs.py @@ -1,37 +1,127 @@ -"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет. - -Будущие ORM-операции захвата, lease, retry и результата должны защищаться -от конкурирующих попыток. Границы транзакций находятся в domain/. -Тип результата захвата уточняется после определения модели job. -""" +"""SQL и изменения jobs/attachments; commit остаётся в domain.""" +from dataclasses import dataclass +from datetime import datetime, timedelta from uuid import UUID +from sqlalchemy import and_, or_, func, select from sqlalchemy.ext.asyncio import AsyncSession - -async def claim_job(session: AsyncSession, job_id: UUID) -> None: - """Условно захватить допустимое задание и сохранить lease/номер попытки. - - Ready и актуальная чужая аренда не переходят обратно в processing. - Будущий результат должен различать захват, завершённый дубль и занятость. - """ - raise NotImplementedError("Lab5: захват job ещё не реализован") +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.schemas.attachment import VARIANT_FILENAMES -async def complete_job(session: AsyncSession, job_id: UUID) -> None: - """Атомарно записать metadata вариантов и ready после сохранения объектов. +@dataclass(frozen=True) +class JobWork: + job_id: UUID + attachment_id: UUID + lease_token: UUID + pipeline_version: int + staging_key: str + content_type: str + size_bytes: int + sha256: str - Предусловия lease/attempt/pipeline_version и структура результата будут - добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage. - """ - raise NotImplementedError("Lab5: фиксация результата ещё не реализована") + @property + def original_key(self) -> str: + return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}" + + def variant_key(self, name: str) -> str: + filename = VARIANT_FILENAMES[name] + return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}" -async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None: - """Записать безопасный код сбоя без stack trace и secrets. +async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: + job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key) + session.add(job) + await session.flush() + return job.id - Domain выбирает queued с next_attempt_at либо конечный failed согласно - классификации ошибки и лимиту попыток; storage лишь сохраняет решение. - """ - raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано") + +async def lock_job(session: AsyncSession, job_id: UUID): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(AttachmentJob.id == job_id) + .with_for_update(of=AttachmentJob) + .execution_options(populate_existing=True) + ) + return result.one_or_none() + + +async def database_time(session: AsyncSession) -> datetime: + return await session.scalar(select(func.clock_timestamp())) + + +async def expired_jobs(session: AsyncSession, limit: int = 20): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp()) + .order_by(AttachmentJob.lease_until) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork: + job.attempt += 1 + job.lease_token = token + job.lease_until = now + timedelta(seconds=lease_seconds) + attachment.status = "processing" + attachment.updated_at = now + return JobWork( + job.id, attachment.id, token, job.pipeline_version, job.staging_key, + attachment.content_type, attachment.size_bytes, attachment.sha256, + ) + + +def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None: + job.lease_token = None + job.lease_until = None + job.last_error_code = error_code + job.next_attempt_at = now + timedelta(seconds=retry_seconds) + job.finished_at = now if terminal else None + attachment.status = "failed" if terminal else "queued" + attachment.error_code = error_code if terminal else None + attachment.updated_at = now + + +def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None: + attachment.object_key = work.original_key + attachment.thumbnail_key = work.variant_key("thumbnail") + attachment.crop_key = work.variant_key("crop") + attachment.optimized_key = work.variant_key("optimized") + attachment.variants = variants + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = now + job.lease_token = None + job.lease_until = None + job.last_error_code = None + job.last_error_permanent = None + job.finished_at = now + + +async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + AttachmentJob.staging_key.is_not(None), + or_( + and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)), + and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)), + ), + ) + .order_by(AttachmentJob.finished_at) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def clear_staging(job) -> None: + job.staging_key = None diff --git a/app/storage/outbox.py b/app/storage/outbox.py index 7010f4e4..e791c841 100644 --- a/app/storage/outbox.py +++ b/app/storage/outbox.py @@ -1,23 +1,31 @@ -"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker. - -Outbox — таблица намерений публикации в той же транзакции, что и job. -Повторная публикация после неопределённого confirm допустима; job_id стабилен. -Форма записей, захват пачки и lease publisher уточняются вместе с моделью. -""" +"""Jobs одновременно хранят намерение публикации очередной попытки.""" from uuid import UUID +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession - -async def add_job_event(session: AsyncSession, job_id: UUID) -> None: - """Сохранить намерение публикации job без commit и обращения к брокеру.""" - raise NotImplementedError("Lab5: запись outbox ещё не реализована") +from app.models.attachment import Attachment +from app.models.job import AttachmentJob -async def mark_event_published(session: AsyncSession, event_id: UUID) -> None: - """Отметить событие опубликованным после устойчивого подтверждения брокера. +async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None: + statement = ( + select(AttachmentJob) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + Attachment.status == "queued", + AttachmentJob.next_attempt_at <= func.clock_timestamp(), + AttachmentJob.published_attempt <= AttachmentJob.attempt, + ) + .order_by(AttachmentJob.next_attempt_at, AttachmentJob.id) + .limit(1) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + if attachment_id is not None: + statement = statement.where(AttachmentJob.attachment_id == attachment_id) + return await session.scalar(statement) - Не помечать job завершённой: publisher confirm отличается от consumer ack. - """ - raise NotImplementedError("Lab5: отметка outbox ещё не реализована") + +def mark_published(job: AttachmentJob) -> None: + job.published_attempt = job.attempt + 1 diff --git a/compose.yaml b/compose.yaml index ac342c52..e17a0985 100644 --- a/compose.yaml +++ b/compose.yaml @@ -45,6 +45,28 @@ x-s3-env: &s3-env S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env} S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env} +x-jobs-env: &jobs-env + BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env} + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env} + JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env} + PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env} + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} + IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env} + IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env} + IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env} + IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env} + IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env} + services: minio: build: @@ -154,6 +176,100 @@ services: cpus: "0.5" memory: 512M + broker: + image: rabbitmq:4.4.0 + hostname: broker + restart: unless-stopped + entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"] + environment: + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + secrets: + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + volumes: + - ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro + - rabbitmqdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + + publisher: + <<: *api-image + init: true + command: ["python", "-m", "app.core.outbox_publisher"] + restart: unless-stopped + environment: + <<: [*base-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_grace_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + worker: + <<: *api-image + init: true + command: ["python", "-m", "app.core.job_worker"] + restart: unless-stopped + environment: + <<: [*base-env, *s3-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m,mode=1777 + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + minio: + condition: service_healthy + minio-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_signal: SIGTERM + stop_grace_period: 20s + deploy: + resources: + limits: + cpus: "2.0" + memory: 2048M + migrate: <<: *api-image command: ["alembic", "upgrade", "head"] @@ -229,6 +345,8 @@ services: restart: unless-stopped environment: <<: [*base-env, *s3-env] + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} secrets: - db_password - source: s3_access_key_id @@ -274,6 +392,7 @@ volumes: pgdata: redisdata: miniodata: + rabbitmqdata: networks: default: @@ -281,6 +400,8 @@ networks: internal: true secrets: + broker_password: + file: ./secrets/broker_password db_password: file: ./secrets/db_password redis_password: diff --git a/evidence/lab5-jobs.json b/evidence/lab5-jobs.json new file mode 100644 index 00000000..639dc0fb --- /dev/null +++ b/evidence/lab5-jobs.json @@ -0,0 +1,378 @@ +{ + "status": "passed", + "results": [ + { + "scenario": "worker_stopped_restart_all", + "attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707", + "queue_before": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "queue_after": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "after": { + "job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:27:44.556734+00:00", + "next_attempt_at": "2026-11-10 03:26:56.662360+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:27:44.548000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:27:44.552000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:27:44.543000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:27:44.536000+00:00" + } + ] + } + }, + { + "scenario": "crash-after-object", + "attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc", + "exit_code": 137, + "before": { + "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "status": "processing", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:27:49.144648+00:00", + "next_attempt_at": "2026-11-10 03:27:47.218612+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:27:49.912000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:27:49.917000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:27:49.907000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:27:49.894000+00:00" + } + ] + }, + "after": { + "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-10 03:29:24.765539+00:00", + "next_attempt_at": "2026-11-10 03:29:23.142176+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:24.750000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:24.758000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:24.742000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:24.725000+00:00" + } + ] + } + }, + { + "scenario": "crash-before-ack", + "attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1", + "exit_code": 137, + "before": { + "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:29.833142+00:00", + "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:29.824000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:29.829000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:29.818000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:29.805000+00:00" + } + ] + }, + "after": { + "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:29.833142+00:00", + "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:29.824000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:29.829000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:29.818000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:29.805000+00:00" + } + ] + } + }, + { + "scenario": "fail-transient", + "attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea", + "samples": [ + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:45.863494+00:00", + "next_attempt_at": "2026-11-10 03:29:50.476742+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-10 03:29:51.216904+00:00", + "next_attempt_at": "2026-11-10 03:29:58.053724+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 3, + "published_attempt": 3, + "limit": 5, + "updated_at": "2026-11-10 03:29:58.279470+00:00", + "next_attempt_at": "2026-11-10 03:30:15.602067+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 4, + "published_attempt": 4, + "limit": 5, + "updated_at": "2026-11-10 03:30:16.386336+00:00", + "next_attempt_at": "2026-11-10 03:30:47.271081+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "failed", + "attempt": 5, + "published_attempt": 5, + "limit": 5, + "updated_at": "2026-11-10 03:30:47.622716+00:00", + "next_attempt_at": "2026-11-10 03:31:30.255216+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + } + ], + "after": { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "ready", + "attempt": 6, + "published_attempt": 6, + "limit": 10, + "updated_at": "2026-11-10 03:30:59.696883+00:00", + "next_attempt_at": "2026-11-10 03:30:53.135190+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:30:59.687000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:30:59.692000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:30:59.683000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:30:59.670000+00:00" + } + ] + }, + "replay_idempotent": true + }, + { + "scenario": "fail-permanent", + "attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9", + "samples": [ + { + "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "status": "failed", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:31:12.333162+00:00", + "next_attempt_at": "2026-11-10 03:31:16.827024+00:00", + "last_error_code": "injected_permanent", + "permanent": true, + "objects": [] + } + ], + "after": { + "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 6, + "updated_at": "2026-11-10 03:31:24.282794+00:00", + "next_attempt_at": "2026-11-10 03:31:17.137221+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:31:24.272000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:31:24.278000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:31:24.267000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:31:24.254000+00:00" + } + ] + }, + "replay_idempotent": true + } + ] +} diff --git a/evidence/lab5-verification.json b/evidence/lab5-verification.json new file mode 100644 index 00000000..2beb4658 --- /dev/null +++ b/evidence/lab5-verification.json @@ -0,0 +1,271 @@ +{ + "date": "2026-11-10", + "status": "passed", + "project": "mtusi-lab5-verification", + "base_url": "http://localhost:28080", + "checks": [ + { + "name": "pytest_regression", + "exit_code": 0, + "stdout": "199 passed in 82.29s (0:01:22)", + "scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled" + }, + { + "name": "pytest_timeout_retention", + "exit_code": 0, + "stdout": "2 passed, 5 deselected in 1.91s", + "scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled" + }, + { + "name": "compose_configuration", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "config", + "--quiet" + ], + "exit_code": 0, + "stdout": "" + }, + { + "name": "contract", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "scripts/check_contract.py" + ], + "exit_code": 0, + "stdout": "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested." + }, + { + "name": "openapi", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "scripts/validate_openapi.py" + ], + "exit_code": 0, + "stdout": "PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested." + }, + { + "name": "public_smoke", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/smoke.py", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate." + }, + { + "name": "message_idempotency", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "-m", + "scripts.check_idempotency", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "PASS: 24 concurrent replays, 2 replicas, one message" + }, + { + "name": "outbox_crash", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "-m", + "scripts.check_outbox_recovery", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "{\"scenario\": \"commit_before_publish\", \"attachment_id\": \"0fff1dce-c479-40d8-8627-f28cc85cb6cf\", \"message_id\": \"e79bfb7e-c2af-4011-a87a-52528d7a7840\", \"publisher_exit_code\": 137, \"status_after_crash\": \"queued\", \"status_after_restart\": \"ready\", \"variants\": [\"thumbnail\", \"crop\", \"optimized\"], \"reuploaded\": false}" + }, + { + "name": "public_persistence", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/resilience.py", + "--mode", + "persistence", + "--action-script", + "scripts/restart.sh", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." + }, + { + "name": "public_failover", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/resilience.py", + "--mode", + "failover", + "--action-script", + "scripts/stop-one.sh", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 104, \"failed_attempts\": 5, \"first_recovery_seconds\": 1.907}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence." + }, + { + "name": "rabbitmq_prefetch", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "broker", + "rabbitmqctl", + "list_channels", + "--formatter=json", + "consumer_count", + "prefetch_count", + "messages_unacknowledged" + ], + "exit_code": 0, + "stdout": "[\n{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":0,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n]" + }, + { + "name": "rabbitmq_durability_ack", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "broker", + "rabbitmqctl", + "list_queues", + "--formatter=json", + "name", + "durable", + "messages_ready", + "messages_unacknowledged", + "consumers" + ], + "exit_code": 0, + "stdout": "[\n{\"name\":\"attachment-jobs\",\"messages_unacknowledged\":0,\"consumers\":2,\"durable\":true,\"messages_ready\":0}\n]" + }, + { + "name": "worker_fault_retry_replay", + "exit_code": 0, + "evidence": "lab5-jobs.json", + "scenarios": [ + "worker_stopped_restart_all", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent" + ] + }, + { + "name": "whitespace", + "command": [ + "git", + "diff", + "--check" + ], + "exit_code": 0, + "stdout": "" + }, + { + "name": "public_contract_tests_unchanged", + "exit_code": 0 + }, + { + "name": "final_services", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "ps", + "--format", + "{{.Service}} {{.State}} {{.Health}}" + ], + "exit_code": 0, + "stdout": "api running healthy\napi running healthy\nbroker running healthy\ncleanup running \nminio running healthy\nnginx running healthy\npostgres running healthy\npublisher running \nredis running healthy\nworker running" + } + ], + "git_base_commit": "0a703588290e26d0f148b45d0e6bee2e8eefce7d", + "uncommitted_changes": true, + "environment": { + "test_runner_python": "3.12.14", + "runtime_python": "3.12.15", + "broker": "RabbitMQ 4.4.0", + "sessions": "Redis 7.4.2", + "worker_memory_bytes": 2147483648, + "worker_cpu_limit": 2.0, + "worker_init": true + }, + "profile": { + "data": "Synthetic users/chats only; tests/fixtures/sample.png for fault scenarios; no credentials in evidence", + "worker_concurrency": 2, + "prefetch_per_consumer": 1, + "max_attempts_per_cycle": 5, + "timeout_seconds": 60, + "lease_seconds": 90, + "retry_base_seconds": 5, + "retry_cap_seconds": 120, + "image_memory_mib": 768, + "image_cpu_seconds": 20, + "isolated_limit_tests": "256 MiB / 1 CPU second in child process; timeout cancellation checked separately" + }, + "criteria": { + "3.1_lab4_and_durable_202": [ + "pytest_regression", + "public_smoke", + "public_persistence", + "worker_fault_retry_replay" + ], + "3.2_durable_broker_worker_late_ack": [ + "rabbitmq_durability_ack", + "rabbitmq_prefetch", + "worker_fault_retry_replay" + ], + "3.3_states_original_thumbnail": [ + "public_smoke", + "pytest_regression", + "worker_fault_retry_replay" + ], + "3.4_bounded_retry_idempotency": [ + "pytest_regression", + "worker_fault_retry_replay" + ], + "3.5_stopped_worker_api_broker_restart": [ + "worker_fault_retry_replay", + "public_persistence" + ], + "4.1_crop_webp_and_worker_limits": [ + "pytest_regression", + "worker_fault_retry_replay" + ], + "4.2_commit_publish_recovery": [ + "outbox_crash" + ], + "4.3_backoff_jitter_classification_registry_replay": [ + "pytest_regression", + "pytest_timeout_retention", + "worker_fault_retry_replay" + ], + "4.4_crash_after_put_lease_prefetch_cleanup": [ + "worker_fault_retry_replay", + "rabbitmq_prefetch", + "pytest_regression", + "pytest_timeout_retention" + ] + }, + "limitations": [ + "Single Docker host, no disk-loss or cluster failover claim", + "Grade 5 experiments not performed", + "No production-scale performance/security claim" + ] +} diff --git a/infra/README.md b/infra/README.md index fda1b846..a6b520db 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,17 +1,27 @@ # Инфраструктура студента -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis +только на localhost. Nginx направляет запросы на две API-реплики. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2 +с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь, +persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata. +Broker и MinIO не публикуют внешние порты. API не получает credentials брокера. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет +original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack. +Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого +consumer. Publisher/worker используют init:true для корректных сигналов и +воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает. -## Перенесённая база lab4 +Все environment-настройки Compose берутся из .env; образец — .env.example. +Credentials передаются файлами secrets; make init создаёт только отсутствующие. +Migrate и minio-bootstrap завершаются до старта зависимых сервисов. +Перед обновлением остановить старые процессы без удаления volumes. Миграция +a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ. -В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap, -migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления. -Роли broker, outbox publisher и worker пока не реализованы. -core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди. -Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md. -Настройки environment в Compose задаются через .env; образец — .env.example. -Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам. +Restart hook сохраняет volumes и состояние заранее остановленного worker. +Failover hook останавливает выбранную API-реплику. Retry, retention, операторские +команды, ограничения и результаты проверок описаны в REPORT.md. +Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии — +make test-outbox-recovery и make test-jobs на выбранном работающем стенде. diff --git a/infra/broker/start-broker.sh b/infra/broker/start-broker.sh new file mode 100755 index 00000000..8efdf065 --- /dev/null +++ b/infra/broker/start-broker.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Secret читается внутри контейнера, не попадает в compose config/аргументы. +export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")" +export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}" +export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}" +export RABBITMQ_NODE_PORT="${BROKER_PORT:?}" +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py index 3b8c4b29..f1c76d70 100644 --- a/infra/minio/bootstrap.py +++ b/infra/minio/bootstrap.py @@ -81,7 +81,7 @@ def main(): { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": [f"arn:aws:s3:::{bucket}/originals/*"], + "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")], }, ], } diff --git a/migrations/versions/a62c950e18b3_rabbitmq_cutover.py b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py new file mode 100644 index 00000000..3eb0cd89 --- /dev/null +++ b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py @@ -0,0 +1,21 @@ +"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ.""" + +from alembic import op + +revision = "a62c950e18b3" +down_revision = "f9183d27a640" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Старые publisher/worker остановлены. Processing восстановится по lease. + op.execute(""" + UPDATE attachment_jobs j SET published_attempt = attempt + FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued' + """) + + +def downgrade() -> None: + # При обратном переключении требуется такая же повторная публикация. + upgrade() diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py new file mode 100644 index 00000000..280c7dca --- /dev/null +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -0,0 +1,62 @@ +"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "c35e91a2f706" +down_revision = "8f31b94a2d10" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.alter_column("attachments", "status", server_default="queued") + op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)) + op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb"))) + op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)") + op.create_table( + "attachment_jobs", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("staging_key", sa.String(128), nullable=True), + sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False), + sa.Column("attempt", sa.Integer(), server_default="0", nullable=False), + sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False), + sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column("lease_token", sa.UUID(), nullable=True), + sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error_code", sa.String(64), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")), + sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")), + sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")), + sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")), + sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")), + ) + op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"]) + op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"]) + # Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail. + op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments") + op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()") + + +def downgrade() -> None: + connection = op.get_bind() + unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")) + if unfinished: + raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения") + op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs") + op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs") + op.drop_table("attachment_jobs") + op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check") + op.drop_column("attachments", "variants") + op.drop_column("attachments", "thumbnail_key") + op.drop_column("attachments", "error_code") + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'") + op.alter_column("attachments", "status", server_default="ready") diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py new file mode 100644 index 00000000..767fbe7e --- /dev/null +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -0,0 +1,49 @@ +"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений.""" + +from alembic import op +import sqlalchemy as sa + +revision = "e7048b6da913" +down_revision = "c35e91a2f706" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True)) + op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True)) + # Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup. + op.execute(""" + INSERT INTO upload_intents (id, object_key) + SELECT gen_random_uuid(), j.staging_key + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE a.status = 'ready' AND j.staging_key IS NOT NULL + AND j.staging_key <> a.object_key + ON CONFLICT (object_key) DO NOTHING + """) + op.execute(""" + UPDATE attachment_jobs j + SET staging_key = a.object_key, attempt = 0, published_attempt = 0, + next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL, + last_error_code = NULL, finished_at = NULL + FROM attachments a + WHERE a.id = j.attachment_id AND a.status = 'ready' + """) + op.execute(""" + UPDATE attachments SET status = 'queued', variants = '[]'::jsonb, + updated_at = clock_timestamp() + WHERE status = 'ready' + """) + + +def downgrade() -> None: + active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")) + if active: + raise RuntimeError("Перед downgrade необходимо завершить активные задания") + op.execute(""" + UPDATE attachments a SET variants = COALESCE( + (SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v + WHERE v->>'name' = 'thumbnail'), '[]'::jsonb) + """) + op.drop_column("attachments", "optimized_key") + op.drop_column("attachments", "crop_key") diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py new file mode 100644 index 00000000..7e6ed24c --- /dev/null +++ b/migrations/versions/f9183d27a640_job_replay.py @@ -0,0 +1,34 @@ +"""Бюджет повторов и аудит операторского replay.""" + +from alembic import op +import sqlalchemy as sa + +revision = "f9183d27a640" +down_revision = "e7048b6da913" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0")) + op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)) + op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True)) + op.create_table( + "job_replays", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column("actor", sa.String(64), nullable=False), + sa.Column("reason", sa.String(32), nullable=False), + sa.Column("previous_attempt", sa.Integer(), nullable=False), + sa.Column("attempt_limit", sa.Integer(), nullable=False), + sa.Column("previous_error_code", sa.String(64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()), + ) + op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"]) + + +def downgrade() -> None: + op.drop_table("job_replays") + op.drop_column("attachment_jobs", "last_error_permanent") + op.drop_column("attachment_jobs", "attempt_limit") + op.drop_column("attachment_jobs", "cycle_start_attempt") diff --git a/requirements.txt b/requirements.txt index f171cf56..a3f67b5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,3 +50,4 @@ pytest-asyncio==1.4.1 httpx==0.28.1 minio==7.2.20 Pillow==13.0.0 +aio-pika==10.2.2 diff --git a/scripts/check_lab5_jobs.py b/scripts/check_lab5_jobs.py new file mode 100644 index 00000000..ebba45fa --- /dev/null +++ b/scripts/check_lab5_jobs.py @@ -0,0 +1,210 @@ +"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные.""" + +import argparse +from datetime import datetime +import hashlib +from io import BytesIO +import json +from pathlib import Path +import subprocess +import time +from uuid import UUID, uuid4 + +from PIL import Image + +from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose +from scripts.deployment import ComposeProject +from tests.client import Client, check + +PROBE = ''' +import asyncio, hashlib, json +from uuid import UUID +from sqlalchemy import select +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.models.job import AttachmentJob +from app.models.attachment import Attachment +from app.storage.s3 import create_s3_storage +async def main(): + settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3) + try: + async with create_session_factory(engine)() as session: + job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one() + result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent) + objects=[] + for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'): + entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True))) + for obj in entries: + stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256() + try: + async for chunk in stream: checksum.update(chunk) + finally: await stream.aclose() + objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified)) + result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) + finally: await storage.aclose();await engine.dispose() +asyncio.run(main()) +''' + + +def probe(attachment_id): + code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE + result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30) + check(result.returncode == 0, 'Probe failed; diagnostic output hidden') + return json.loads(result.stdout) + + +def admin(*arguments, expected=0): + result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments) + check(result.returncode == expected, 'Unexpected operator command exit status') + return json.loads(result.stdout) + + +def verify_result(client, actor, attachment, original): + ready = client.wait_ready(actor, attachment, wait_seconds=180) + check(ready['id'] == attachment['id'], 'Attachment identity changed') + path = '/api/v1/attachments/' + ready['id'] + '/content' + check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed') + check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline') + for variant in ready['variants']: + data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes'] + check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch') + with Image.open(BytesIO(data)) as image: + check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch') + check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format') + check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked') + if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size') + snapshot=probe(ready['id']) + check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants') + return snapshot + + +def crash(client, actor, chat, mode): + name='lab5-worker-fault-'+uuid4().hex + try: + require_compose('stop','worker') + attachment,original=client.attachment(actor,chat) + result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) + phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack' + marker=json.dumps({'fault':phase,'attachment_id':attachment['id']}) + check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed') + before=probe(attachment['id']) + check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state') + check(len(before['objects'])==4,'Crash must occur after all four PUTs') + finally: + subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30) + require_compose('start','worker') + after=verify_result(client,actor,attachment,original) + if mode=='crash-before-ack': + # Wait for RabbitMQ to deliver and acknowledge the ready duplicate. + wait_queue_empty() + after=probe(attachment['id']) + check(after==before,'Ready duplicate rewrote result') + else: + check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt') + check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content') + return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after) + + +def retry_replay(client,actor,chat,permanent): + name='lab5-retry-fault-'+uuid4().hex + samples=[] + configuration=ComposeProject().config['services']['worker']['environment'] + retry_base=float(configuration['JOB_RETRY_SECONDS']) + retry_max=float(configuration['JOB_RETRY_MAX_SECONDS']) + try: + require_compose('stop','worker') + attachment,original=client.attachment(actor,chat) + mode='fail-permanent' if permanent else 'fail-transient' + require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) + deadline=time.monotonic()+180 + while time.monotonic()=1 and queued_before['durable'], 'No durable queued message before restart') + result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180) + check(result.returncode==0,'Restart hook failed') + check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker') + queued_after=queue_state() + check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart') + client.message(actor,chat,'session and chat survived broker/API restart') + finally:require_compose('start','worker') + after=verify_result(client,actor,attachment,original) + return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after) + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--base-url',required=True) + parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all') + parser.add_argument('--output',default='evidence/lab5-jobs.json') + args=parser.parse_args() + client=Client(args.base_url,timeout=40) + aliases={r.instance_id for r in ComposeProject().replicas() if r.running} + check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL') + actor,_,_,chat=client.bootstrap() + results=[] + operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)} + selected=operations.values() if args.scenario=='all' else [operations[args.scenario]] + for operation in selected: + result=operation(client,actor,chat);results.append(result) + Path(args.output).parent.mkdir(parents=True,exist_ok=True) + Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n') + print('PASS: '+result['scenario'],flush=True) + Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n') + + +if __name__=='__main__':main() diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py new file mode 100644 index 00000000..e5379afb --- /dev/null +++ b/scripts/check_outbox_recovery.py @@ -0,0 +1,83 @@ +"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload. + +Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API. +Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +from uuid import uuid4 + +from scripts.deployment import ComposeProject +from tests.client import Client, check + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess: + return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout) + + +def require_compose(*arguments: str) -> str: + result = compose(*arguments) + check(result.returncode == 0, "Compose command failed; output hidden") + return result.stdout + + +def main() -> None: + parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish") + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + args = parser.parse_args() + running = set(require_compose("ps", "--status", "running", "--services").splitlines()) + check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario") + client = Client(args.base_url, args.ca_file, timeout=40) + project = ComposeProject() + aliases = {replica.instance_id for replica in project.replicas() if replica.running} + live = client.call("GET", "/health/live", schema="Health") + check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project") + alice, _, _, chat = client.bootstrap() + attachment = None + fault_container = "lab5-outbox-fault-" + uuid4().hex + try: + require_compose("stop", "publisher") + attachment, original = client.attachment(alice, chat) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice) + killed = compose( + "run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m", + "app.core.outbox_publisher", "--crash-before-publish", attachment["id"], + ) + marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]}) + check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed") + pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"] + check(pending["status"] == "queued", "Crash must leave accepted job queued") + finally: + # Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется. + try: + subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30) + finally: + require_compose("start", "publisher") + ready = client.wait_ready(alice, attachment, wait_seconds=60) + check(ready["id"] == attachment["id"], "Recovered a different attachment") + check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants") + path = f"/api/v1/attachments/{attachment['id']}/content" + check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery") + for variant in ready["variants"]: + downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"] + check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch") + print(json.dumps({ + "scenario": "commit_before_publish", "attachment_id": ready["id"], + "message_id": message["id"], "publisher_exit_code": killed.returncode, + "status_after_crash": pending["status"], "status_after_restart": ready["status"], + "variants": [variant["name"] for variant in ready["variants"]], + "reuploaded": False, + })) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_local.py b/scripts/init_local.py index baa14022..a3fb687a 100644 --- a/scripts/init_local.py +++ b/scripts/init_local.py @@ -14,7 +14,7 @@ def main() -> None: destination.chmod(0o600) directory = root / "secrets" directory.mkdir(mode=0o700, exist_ok=True) - for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): + for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): path = directory / name try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) diff --git a/scripts/restart.sh b/scripts/restart.sh index 1d313c98..493ee343 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -22,11 +22,16 @@ wait_healthy() { done } -compose stop cleanup -compose restart postgres redis minio -wait_healthy postgres redis minio +worker_running=$(compose ps --status running --quiet worker) +compose stop worker publisher cleanup +compose restart postgres redis minio broker +wait_healthy postgres redis minio broker compose restart api wait_healthy api compose restart nginx wait_healthy nginx -compose start cleanup +compose start publisher cleanup +# Сценарий с заранее остановленным worker сохраняет его остановленным. +if [ -n "$worker_running" ]; then + compose start worker +fi diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 5661d148..0deb88f5 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -1,4 +1,4 @@ -"""Lab4: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" +"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" import asyncio import hashlib @@ -17,6 +17,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from starlette.requests import ClientDisconnect from app.core.s3 import S3Settings +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.models.job import AttachmentJob +from app.schemas.job import JobMessage from app.domain import attachments as domain from app.errors.exceptions import unavailable from app.models.attachment import Attachment, UploadIntent @@ -79,14 +83,31 @@ async def upload(client, actor, chat, data=PNG, filename='фото.png', content ) +async def finish_attachment(client, storage, actor, attachment_id): + engine = client._transport.app.state.engine + async with AsyncSession(engine) as session: + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) + message = JobMessage(job_id=job.id, attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at) + await session.rollback() + assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None)) + result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor)) + assert result.json()['status'] == 'ready' + return result.json() + + async def test_upload_download_and_attachment_only_message(client, context): storage, alice, bob, eve, chat = context response = await upload(client, alice, chat) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text attachment = response.json() - assert attachment['status'] == 'ready' + assert attachment['status'] == 'queued' + before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)) + assert before.status_code == 409 + attachment = await finish_attachment(client, storage, alice, attachment['id']) assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest() - assert attachment['variants'] == [] and attachment['error_code'] is None + assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'} + assert attachment['error_code'] is None assert 'object_key' not in attachment id = attachment['id'] for actor in (alice, bob): @@ -120,13 +141,14 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ async def test_jpeg_is_stored_without_reencoding(client, context): - _, alice, _, _, chat = context + storage, alice, _, _, chat = context output = BytesIO() Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG') original = output.getvalue() response = await upload(client, alice, chat, original, content_type='image/jpeg') - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text id = response.json()['id'] + await finish_attachment(client, storage, alice, id) downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) assert downloaded.content == original @@ -167,7 +189,7 @@ async def test_exact_byte_limit_accepts_valid_png(client, context): padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload)) image = PNG[:-12] + padding + PNG[-12:] response = await upload(client, alice, chat, image) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()['size_bytes'] == maximum assert response.json()['sha256'] == hashlib.sha256(image).hexdigest() @@ -178,7 +200,7 @@ async def test_exact_pixel_limit_accepts_valid_png(client, context): with Image.new('RGB', (8000, 5000), 'blue') as image: image.save(output, format='PNG') response = await upload(client, alice, chat, output.getvalue()) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text async def test_pixel_limit_before_decoding(client, context): @@ -226,6 +248,7 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): async def test_s3_outage_is_503(client, context): storage, alice, _, _, chat = context id = (await upload(client, alice, chat)).json()['id'] + await finish_attachment(client, storage, alice, id) storage.down = True response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) assert response.status_code == 503 @@ -353,6 +376,7 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): storage, alice, _, _, chat = context id = (await upload(client, alice, chat)).json()['id'] + await finish_attachment(client, storage, alice, id) async with AsyncSession(db_engine) as session: attachment = await session.get(Attachment, UUID(id)) key = attachment.object_key diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 00000000..bbed62f2 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,71 @@ +"""Реальные преобразования и OS-лимиты дочернего процесса worker.""" + +import asyncio +from io import BytesIO +import os +from pathlib import Path +import subprocess +import sys +from uuid import uuid4 + +from PIL import Image +import pytest + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.domain.job_files import render_variants +from app.storage.jobs import JobWork +from test_attachments import MemoryStorage + + +@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)]) +async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent): + source=tmp_path/'source.png' + with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image: + exif=Image.Exif();exif[270]='private description' + image.save(source,format='PNG',exif=exif) + result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None)) + variants={v.name:v for v in result.variants} + assert (variants['crop'].width,variants['crop'].height)==(128,128) + assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size) + assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size) + for name in ('thumbnail.jpg','crop.jpg','optimized.webp'): + with Image.open(tmp_path/name) as image: + assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp') + if transparent: + if name.endswith('webp'):assert image.getpixel((0,0))[3]==0 + else:assert min(image.getpixel((0,0)))>=250 + + +async def test_exif_orientation_applied(tmp_path): + source=tmp_path/'source.jpg' + with Image.new('RGB',(400,200),'blue') as image: + exif=Image.Exif();exif[274]=6 + image.save(source,exif=exif) + result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None)) + assert (result.variants[0].width,result.variants[0].height)==(128,256) + + +@pytest.mark.parametrize('failure', ['memory','cpu']) +async def test_worker_os_limits(failure,tmp_path): + body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass' + code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())" + result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'], + env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15) + assert result.returncode==(3 if failure=='memory' else -9) + assert not (tmp_path/'metadata').exists() + + +async def test_render_timeout_kills_child(monkeypatch,tmp_path): + create=asyncio.create_subprocess_exec + children=[] + async def slow_child(*args,**kwargs): + child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs) + children.append(child) + return child + monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child) + work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64) + with pytest.raises(TimeoutError): + async with asyncio.timeout(.2): + await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path) + assert len(children)==1 and children[0].returncode is not None diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 00000000..aeab6499 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,178 @@ +"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL.""" + +import asyncio +from datetime import timedelta +import errno +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.jobs import JobSettings +from app.domain import attachment_jobs +from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result +from app.domain.job_admin import replay_failed_job +from app.domain.job_retry import classify_failure, retry_delay +from app.errors.exceptions import JobOperationError, ProcessingError +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay +from app.schemas.job import JobMessage +from app.storage import jobs +from test_attachments import context, upload, finish_attachment + + +async def envelope(session, attachment_id): + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) + return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1, + attempt=job.attempt + 1, created_at=job.created_at) + + +async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None, job_max_attempts=3) + def fail(_): + raise TimeoutError + async with AsyncSession(db_engine, expire_on_commit=False) as session: + first = await envelope(session, attachment['id']) + for attempt in range(1, 4): + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == attempt + delay = (job.next_attempt_at - item.updated_at).total_seconds() + assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1) + assert item.status == ('failed' if attempt == 3 else 'queued') + assert job.last_error_permanent is False + job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1) + await session.commit() + assert item.error_code == 'retry_exhausted' + request_id = uuid4() + result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, + actor='test', reason='dependency_restored') + assert result['attempt_limit'] == 6 + # Old envelope cannot consume the fresh replay budget. + assert await process_attachment_job(session, storage, first, settings) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == 3 and item.status == 'queued' + await session.commit() + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings) + before = dict(storage.objects) + assert await process_attachment_job(session, storage, message, settings) + assert storage.objects == before + repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, + actor='test', reason='dependency_restored') + assert repeat['already_applied'] + with pytest.raises(JobOperationError, match='job_not_failed'): + await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review') + await session.rollback() + assert len((await session.scalars(select(JobReplay))).all()) == 1 + + +async def test_permanent_failure_and_missing_source_replay(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + def fail(_): + raise ProcessingError('image_limits') + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent + job.staging_key = None + await session.commit() + with pytest.raises(JobOperationError, match='source_expired'): + await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review') + + +async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + claim = await claim_job(session, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + job.lease_token = uuid4() + job.finished_at = await jobs.database_time(session) - timedelta(days=10) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + # Rendering must not be allowed to PUT on a stale lease. + before = dict(storage.objects) + assert await build_result(session, storage, claim.work) + assert storage.objects == before + + +async def test_cleanup_ready_preserves_final_objects(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + await finish_attachment(client, storage, alice, attachment['id']) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + job, item = await jobs.lock_job(session, message.job_id) + staging = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1 + assert staging in storage.objects + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1 + assert staging not in storage.objects and len(storage.objects) == 4 + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0 + + +async def test_retry_classification_and_cap(): + settings = JobSettings(_env_file=None) + for attempt in (1, 2, 6, 100): + cap = min(120, 5 * 2 ** min(30, attempt - 1)) + assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100)) + assert classify_failure(TimeoutError()) == ('processing_timeout', False) + assert classify_failure(OSError(errno.EACCES, 'denied'))[1] + assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1] + + +async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + async def stalled(*args, **kwargs): + await asyncio.sleep(60) + monkeypatch.setattr(attachment_jobs, 'build_result', stalled) + settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05}) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == 'queued' and job.last_error_code == 'processing_timeout' + assert job.lease_token is None and job.lease_until is None + + +async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + def fail(_): + raise ProcessingError('invalid_image') + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, message.job_id) + key = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + data, metadata = storage.objects[key] + storage.objects[key] = (b'x' * len(data), metadata) + with pytest.raises(JobOperationError, match='source_mismatch'): + await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review') + await session.rollback() + assert not (await session.scalars(select(JobReplay))).all() + job, item = await jobs.lock_job(session, message.job_id) + job.finished_at = await jobs.database_time(session) - timedelta(days=8) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 1 + assert key not in storage.objects diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py index f13139d8..f8efed1b 100644 --- a/student-tests/test_migrations.py +++ b/student-tests/test_migrations.py @@ -8,7 +8,7 @@ from sqlalchemy import inspect, text from app.models import Base from conftest import reset_schema, upgrade_head -EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents"} +EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"} @pytest.fixture -- 2.54.0 From 7cd02da51d3fac08e5172eb48db77f221c7dec0b Mon Sep 17 00:00:00 2001 From: gytall Date: Tue, 10 Nov 2026 07:44:19 +0300 Subject: [PATCH 3/6] pure version of lab5 --- .env.example | 10 +- Dockerfile | 4 +- Makefile | 269 ++++--------- REPORT.md | 257 ++++--------- app/domain/idempotency.py | 2 +- compose.yaml | 6 +- evidence/{lab5-jobs.json => jobs.json} | 144 +++---- ...b5-verification.json => verification.json} | 41 +- infra/README.md | 37 +- .../c35e91a2f706_async_attachment_jobs.py | 2 +- .../e7048b6da913_attachment_crop_webp.py | 2 +- scripts/check_grade4.py | 355 ------------------ scripts/check_idempotency.py | 39 -- scripts/{check_lab5_jobs.py => check_jobs.py} | 11 +- scripts/check_nginx.py | 6 +- scripts/check_outbox_recovery.py | 4 +- scripts/check_s3.py | 24 +- scripts/collect_evidence.py | 204 ---------- scripts/deployment.py | 47 --- scripts/expiry_demo.sh | 28 -- scripts/explain_history.py | 132 ------- scripts/failover_all.py | 148 -------- scripts/failure_demo.sh | 64 ---- scripts/restart.example.sh | 2 +- scripts/rolling_update.py | 92 ----- scripts/s3_faults.py | 108 ------ scripts/upload_load.py | 113 ------ scripts/validate_openapi.py | 4 +- student-tests/conftest.py | 2 +- student-tests/test_attachments.py | 2 +- student-tests/test_jobs.py | 2 +- 31 files changed, 295 insertions(+), 1866 deletions(-) rename evidence/{lab5-jobs.json => jobs.json} (66%) rename evidence/{lab5-verification.json => verification.json} (87%) delete mode 100644 scripts/check_grade4.py delete mode 100644 scripts/check_idempotency.py rename scripts/{check_lab5_jobs.py => check_jobs.py} (96%) delete mode 100644 scripts/collect_evidence.py delete mode 100755 scripts/expiry_demo.sh delete mode 100644 scripts/explain_history.py delete mode 100644 scripts/failover_all.py delete mode 100755 scripts/failure_demo.sh delete mode 100644 scripts/rolling_update.py delete mode 100644 scripts/s3_faults.py delete mode 100644 scripts/upload_load.py diff --git a/.env.example b/.env.example index 65196d01..8b2de339 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,8 @@ # Скопируйте в .env для локального окружения; .env не коммитится. -# Настройки приложения, Compose и очереди lab5. +# Настройки приложения, Compose и очереди. APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab +APP_ENV=development LOG_LEVEL=INFO MAX_MESSAGES=100000 @@ -35,7 +35,7 @@ DB_STATEMENT_TIMEOUT_MS=5000 # --- Балансировка и идемпотентность --- API_REPLICAS=2 -API_IMAGE=mtusi-messenger:lab5 +API_IMAGE=messenger:local API_BUILD_REVISION=development IDEMPOTENCY_TTL_SECONDS=86400 SHUTDOWN_DRAIN_SECONDS=1 @@ -48,7 +48,7 @@ LB_HOST_PORT=18081 DB_HOST_PORT=25432 REDIS_HOST_PORT=26379 -# --- Lab4: MinIO, ограниченный upload и очистка журнала --- +# --- MinIO, ограниченный upload и очистка журнала --- S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments @@ -72,7 +72,7 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password MINIO_BROWSER=off -# Lab5: отдельный RabbitMQ broker и ограниченный worker. +# Отдельный RabbitMQ broker и ограниченный worker. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 diff --git a/Dockerfile b/Dockerfile index 17f476a1..24102d5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,5 @@ FROM python:3.12.15-slim-bookworm -LABEL io.mtusi.messenger.secrets="file-v1" - ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ @@ -19,7 +17,7 @@ COPY migrations ./migrations COPY app ./app ARG API_BUILD_REVISION -LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION +LABEL org.opencontainers.image.revision=$API_BUILD_REVISION USER appuser diff --git a/Makefile b/Makefile index a900c25b..4bad55df 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,12 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard ven PYTEST_ARGS ?= BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080) CA_FILE ?= +SERVICE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) - COMPOSE = docker compose --env-file .env -# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis -# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh. -TEST_PROJECT ?= mtusi-messenger-lab5-test +TEST_PROJECT ?= messenger-test TEST_LB_PORT ?= 28080 TEST_DB_PORT ?= $(DB_HOST_PORT) TEST_REDIS_PORT ?= $(REDIS_HOST_PORT) @@ -27,58 +25,49 @@ TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file /dev/null && exit 0; sleep 1; \ - done; echo "API $(1) не готов за 90 секунд" >&2; exit 1 -endef - -.PHONY: help check validate test load test-persistence test-failover \ - init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \ - up down restart restore ps logs migrate migration instances \ - test-stand-up test-stand-down test-stand-destroy test-stand-ps \ - test-stand-smoke test-stand-persistence test-stand-failover \ - test-own test-docker test-limits test-concurrency test-migrations test-all \ - explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \ - rolling-update failover-all test-lab3-grade4 +.PHONY: help init up down restart ps logs migrate migration worker-stop worker-start \ + cleanup-dry-run cleanup-apply check validate test test-unit test-pipeline \ + test-nginx test-storage test-outbox test-jobs test-persistence test-failover \ + test-suite test-stand-up test-stand-destroy help: - @echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply" - @echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads" - @echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh" - @echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh" - @echo "Основной: up | down | restart | restore | ps | logs | migrate | instances" - @echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps" - @echo " test-stand-smoke | test-stand-persistence | test-stand-failover" - @echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all" - @echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)" - @echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)" - @echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)" - @echo "База lab3: rolling-update | failover-all" - @echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)" - @echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)" - -# --- Проверки API и сценариев отказа (tests/) --- + @echo "Запуск: init | up | down | restart | ps | logs SERVICE=worker" + @echo "Обслуживание: migrate | migration m='описание' | worker-stop | worker-start" + @echo "Очистка объектов: cleanup-dry-run | cleanup-apply" + @echo "Проверки: check | validate | test | test-pipeline | test-unit PYTEST_ARGS=..." + @echo "Сервисы: test-nginx | test-storage | test-outbox | test-jobs" + @echo "Отказы API (по запросу): test-persistence | test-failover" + @echo "Автономный прогон с удалением тестового стенда: test-suite" init: $(PYTHON) scripts/init_local.py -test-grade4: - $(MAKE) check validate - $(MAKE) test-stand-up - $(MAKE) test-docker - $(MAKE) test-stand-smoke - $(MAKE) test-stand-idempotency - $(MAKE) test-stand-persistence - $(MAKE) test-stand-failover - $(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)" - $(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)" +up: + $(COMPOSE) up -d --build --wait --wait-timeout 120 -test-s3: - $(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults +down: + $(COMPOSE) down -load-uploads: - $(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10 +restart: + scripts/restart.sh + +ps: + $(COMPOSE) ps $(SERVICE) + +logs: + $(COMPOSE) logs -f $(SERVICE) + +migrate: + $(COMPOSE) run --rm --build migrate + +migration: + DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)" + +worker-stop: + $(COMPOSE) stop worker + +worker-start: + $(COMPOSE) start worker cleanup-dry-run: $(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once @@ -92,161 +81,49 @@ check: validate: $(PYTHON) scripts/validate_openapi.py -test-nginx: - $(PYTHON) scripts/check_nginx.py - -test-lab3-grade3: - @set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \ - $(MAKE) check validate; \ - $(MAKE) test-stand-up; \ - $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover - -test-lab3-grade4: - @set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \ - $(MAKE) check validate; \ - $(MAKE) test-stand-up; \ - $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \ - $(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)" - test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) -load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +# Все сохранённые регрессионные тесты запускаются явно; PYTEST_ARGS выбирает подмножество. +test-unit: + $(TEST_ENV) $(PYTHON) -m pytest -q $(PYTEST_ARGS) -test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-pipeline: + $(TEST_ENV) $(PYTHON) -m pytest -q student-tests/test_attachments.py student-tests/test_jobs.py student-tests/test_image_pipeline.py $(PYTEST_ARGS) -test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-nginx: + $(PYTHON) scripts/check_nginx.py -# --- Основной стенд --- +test-storage: + $(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" $(TLS_ARGS) -rolling-update: - $(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS) - -failover-all: - $(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS) - -up: - $(COMPOSE) up -d --build - $(call wait_ready,$(BASE_URL)) - -down: - $(COMPOSE) down - -restart: - scripts/restart.sh - $(call wait_ready,$(BASE_URL)) - -# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры). -restore: - $(COMPOSE) up -d --wait api - -ps: - $(COMPOSE) ps - -logs: - $(COMPOSE) logs -f api nginx - -migrate: - $(COMPOSE) run --rm --build migrate - -# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id). -instances: - @for i in $$(seq 1 10); do \ - curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \ - done - -# Генерация миграции против БД тестового стенда (основной стенд БД не публикует). -migration: - DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)" - -# --- Тестовый стенд --- - -test-stand-up: - $(TEST_COMPOSE) up -d --build - $(call wait_ready,$(TEST_BASE_URL)) - $(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true' - -test-stand-down: - $(TEST_COMPOSE) down - -test-stand-destroy: - $(TEST_COMPOSE) down -v - -test-stand-ps: - $(TEST_COMPOSE) ps - -test-stand-smoke: - $(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)" - -test-stand-idempotency: - $(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)" - -test-stand-persistence: - $(TEST_COMPOSE) ps - $(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)" - $(TEST_COMPOSE) ps - -test-stand-failover: - $(TEST_COMPOSE) ps - $(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)" - $(TEST_COMPOSE) ps -a api - $(TEST_COMPOSE) up -d --wait api - -# --- Собственные тесты (student-tests/) --- - -test-own: - $(PYTHON) -m pytest -q $(PYTEST_ARGS) - -test-docker: - $(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS) - -test-limits: - $(PYTHON) -m pytest student-tests/test_limits.py -q - -test-concurrency: - $(PYTHON) -m pytest student-tests/test_concurrency.py -q - -test-migrations: - $(PYTHON) -m pytest student-tests/test_migrations.py -q - -test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence - -# --- Демо (на тестовом стенде) --- - -explain: - $(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py - -expiry-demo: test-stand-up - $(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api - $(call wait_ready,$(TEST_BASE_URL)) - BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh - $(TEST_COMPOSE) up -d api - $(call wait_ready,$(TEST_BASE_URL)) - -failure-demo: test-stand-up - $(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh - -# --- Очередь lab5 --- -.PHONY: worker-stop worker-start jobs-logs -worker-stop: - $(COMPOSE) stop worker - -worker-start: - $(COMPOSE) start worker - -jobs-logs: - $(COMPOSE) logs -f publisher worker broker - -# Уже работающий изолированный стенд; сам цель его не поднимает. -.PHONY: test-outbox-recovery -test-outbox-recovery: +test-outbox: $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS) -.PHONY: test-jobs test-jobs: - $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)" + $(PYTHON) -m scripts.check_jobs --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-persistence: + $(PYTHON) tests/resilience.py --mode persistence --action-script "$(or $(ACTION_SCRIPT),scripts/restart.sh)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-failover: + $(PYTHON) tests/resilience.py --mode failover --action-script "$(or $(ACTION_SCRIPT),scripts/stop-one.sh)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# Отдельные PostgreSQL/Redis нужны для локальных тестов; тестовая БД создаётся явно. +test-stand-up: + $(TEST_COMPOSE) up -d --build --wait --wait-timeout 120 + $(TEST_COMPOSE) exec -T postgres sh -c 'psql -U "$$POSTGRES_USER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '\''$${POSTGRES_DB}_test'\''" | grep -q 1 || createdb -U "$$POSTGRES_USER" "$${POSTGRES_DB}_test"' + +test-stand-destroy: + $(TEST_COMPOSE) down --volumes --remove-orphans + +# Проверки текущего pipeline; полный регрессионный набор и отказы API — отдельные цели. +test-suite: + @set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \ + $(MAKE) check validate; \ + $(MAKE) test-stand-up; \ + $(TEST_ENV) $(MAKE) test-pipeline; \ + $(TEST_ENV) $(MAKE) test BASE_URL="$(TEST_BASE_URL)"; \ + $(TEST_ENV) $(MAKE) test-storage BASE_URL="$(TEST_BASE_URL)"; \ + $(TEST_ENV) $(MAKE) test-outbox BASE_URL="$(TEST_BASE_URL)"; \ + $(TEST_ENV) $(MAKE) test-jobs BASE_URL="$(TEST_BASE_URL)"; diff --git a/REPORT.md b/REPORT.md index 94e40ab9..afac7ebc 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,14 +1,14 @@ # Отчёт — лабораторная 5 -Автор: Григорян Илья Мурадович, БПИ2303. -Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026. -Дальнейшая цель — 5; исследовательский трек ещё не выбран. -Использована помощь Codex при реализации, проверках и исправлениях. -Commit/PR: заполнить после публикации. +- Автор, группа: Григорян Илья Мурадович, БПИ2303. +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек — **TODO**. Все пункты уровней **3 и 4** реализованы и проверены. +- Commit/PR: TODO после публикации. +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI, SQLAlchemy, Pillow, aio-pika; [RabbitMQ](https://www.rabbitmq.com/docs/reliability), [Pillow](https://pillow.readthedocs.io/en/stable/); помощь Codex при реализации и проверках. -## Запуск +## Запуск из чистого clone -Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt. +Нужны Python 3.12, pip и Docker Compose. Версии зависимостей закреплены в requirements.txt; +основные образы — PostgreSQL 16.10, Redis 7.4.2, RabbitMQ 4.4.0 и nginx 1.28.0. ```sh python3 -m venv .venv @@ -16,210 +16,83 @@ python3 -m venv .venv make init make up make test +make down ``` -`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие. -Все environment-параметры Compose подставляются из .env; образец — .env.example. -Пароли и S3 credentials читаются из файлов secrets, не включаются в задания. -API не получает credentials брокера. По умолчанию HTTP доступен на -http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты. +`make init` создаёт отсутствующие .env и secrets/, миграции и bootstrap выполняются +при запуске. Все настройки берутся из .env; образец — .env.example. +HTTP: http://localhost:18081. Внутренние хранилища и брокер наружу не открыты. +`make down` сохраняет данные; `docker compose down --volumes` намеренно удаляет их. +При обновлении схемы сначала остановить старые процессы, сохранив volumes. -При обновлении ранее запущенного стенда сначала остановить старые процессы -через `make down` без удаления volumes, затем выполнить `make up`. -Migrate применяет ревизии до a62c950e18b3 перед стартом приложения. -Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые -вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP. -При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs -из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы -обоих брокеров во время перехода должны быть остановлены. +## Архитектура и решения -## Инфраструктура и устойчивый приём +API → приватный MinIO staging + PostgreSQL job/outbox → RabbitMQ → worker → +конечные объекты MinIO + PostgreSQL metadata. Redis хранит сессии. +Сохранены аутентификация, чаты, сообщения, пагинация и идемпотентность предыдущих работ. +Загрузка возвращает **202 queued** после устойчивого приёма. Состояния: +queued / processing / ready / failed; скачивание до ready возвращает 409. -| Компонент | Реализация | Данные | -| --- | --- | --- | -| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий | -| Балансировка | nginx 1.28.0 | единая HTTP-точка входа | -| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit | -| Сессии | Redis 7.4.2, AOF always | redisdata | -| Объекты | приватный MinIO | staging, originals, variants | -| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata | -| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ | -| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL | -| Cleanup | отдельный процесс | незавершённые upload и terminal staging | +RabbitMQ использует устойчивую очередь, сохранение сообщений на диск и подтверждения. +Publisher отмечает публикацию после подтверждения брокера; worker подтверждает доставку +после фиксации результата. Outbox восстанавливает разрыв COMMIT → publish. +Фиксированные ключи объектов, блокировка job и lease защищают от повторов и устаревшего worker. -Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP. -Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером -приёма сообщения (publisher confirm) и только затем фиксирует published_attempt. -При неизвестном результате публикации возможен дубль; намерение остаётся в БД. -Подтверждение worker (consumer ack) следует после устойчивого результата или -устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ -возвращает неподтверждённые доставки в очередь. +Original не перекодируется. Варианты: JPEG thumbnail ≤256×256, центральный JPEG crop +128×128 и WebP ≤1280×1280. JPEG quality=85, WebP quality=80/method=4; EXIF orientation +учитывается, EXIF/ICC/XMP удаляются. Для JPEG alpha заменяется белой подложкой, +WebP сохраняет прозрачность. Лимиты: 10 MiB, 40 млн пикселей, timeout 60 секунд, +768 MiB и 20 секунд CPU на дочерний процесс; concurrency=2, prefetch=1, lease=90 секунд. -До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет -его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL. -Job одновременно служит transactional outbox — устойчивым намерением публикации. -Worker и broker не входят в синхронный путь upload и readiness API. Остановка -worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT -API проверяет БД, чтобы не удалить оригинал уже принятого задания. +Повторы ограничены пятью попытками. Задержка со случайным разбросом: +c=min(120, 5×2^(n−1)), delay∈[c/2,c]. Постоянные ошибки завершаются сразу. +Реестр failed и аудит находятся в PostgreSQL; явный replay проверяет исходник, +выделяет новый бюджет и не запускает уже готовую job. Staging активных jobs сохраняется; +ready очищается через сутки, failed — через семь суток. -Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id, -pipeline_version=1, attempt, created_at, schema_version, type и необязательный -trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются. -Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409. -Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены. - -## Изображения и ограничения - -Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без -перекодирования. Все варианты создаются в отдельном ограниченном процессе: - -| Вариант | Формат и размеры | Параметры | -| --- | --- | --- | -| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha | -| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается | -| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется | - -Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся -в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA. -Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256. - -Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60; -IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс. -Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс; -S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией. - -## Повторы, идемпотентность и очистка - -Состояния: queued → processing → ready; временный отказ возвращает queued, -постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code. -JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit. -Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS, -JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling]. -База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД; -дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при -публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные; -timeout, временные ошибки S3/сети и файловой системы повторяются. - -Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает -истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job, -проверяет актуальность token и lease, держит блокировку до записи original, -трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки. -Устаревшая попытка не может начать запись. Фиксированные ключи по -(attachment_id, pipeline_version, variant) исключают второй набор объектов. -Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один -раз с идемпотентным эффектом, не exactly-once выполнение. - -WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал. -Publisher и worker используют init:true: Python не является PID 1, поэтому -fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу -без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload. - -Queued/processing staging не удаляется. Для ready retention — 86400 секунд; -для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток). -Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым -original, объект сохраняется. Частичные конечные объекты failed автоматически -не удаляются; после replay используются те же ключи. - -## Реестр failed и операторский replay - -Failed registry находится в PostgreSQL. Команды требуют операторского доступа -к Docker/credentials стенда; публичного endpoint для replay нет: +Операторские команды: ```sh docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed -docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored -docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID ``` -JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored, -configuration_fixed, manual_review. Actor — аудиторская метка, не система входа. -Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256, -атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не -сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id -с теми же параметрами возвращает already_applied даже после ready; конфликт -параметров отклоняется. После неоднозначного ответа повторяют тот же request-id. -Готовую job с новым request-id повторить нельзя. После удаления staging replay -отклоняется с source_expired; отсутствующий объект даёт source_missing. +JOB_ID и REQUEST_UUID заменить UUID. Повтор команды с тем же request-id безопасен. -## Проверки уровня 3 и 4 +## Доказательства критериев -Полный воспроизводимый прогон на отдельном стенде: +`make test-suite` поднимает отдельный стенд, проверяет pipeline и удаляет контейнеры, +сети и volumes при завершении, включая ошибку. Полная сохранённая регрессия запускается +отдельно через `make test-unit`; отдельные сервисы — `test-nginx` и `test-storage`. +Проверки pipeline: 42 теста. Полная сохранённая регрессия: 201 тест. +Публичные smoke/persistence/failover прошли. Для отдельных pytest-прогонов нужен +`make test-stand-up`; после проверки — `make test-stand-destroy`. -```sh -make test-grade4 PYTHON=.venv/bin/python -``` +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | make test | 202 → ready, неизменный original, JPEG thumbnail | evidence/verification.json | +| Устойчивый приём и сохранение возможностей lab4 | DOCKER_TESTS=1 make test-unit; make test-persistence | Данные и сессия сохранены, загрузка асинхронная | evidence/verification.json | +| Отдельные broker/worker, поздний ack | make test-jobs | Сообщения сохранены, очередь подтверждена | evidence/jobs.json | +| Состояния, ограниченные повторы, идемпотентность | make test-pipeline; make test-jobs | failed после бюджета, дубли не создают новые объекты | evidence/verification.json | +| Остановка worker и рестарт API/брокера | make test-jobs | То же вложение стало ready, сообщение пережило рестарт | evidence/jobs.json | +| Crop/WebP и ограничения worker | make test-pipeline; make test-jobs | Форматы/размеры, metadata и лимиты проверены | evidence/verification.json | +| Разрыв COMMIT → publish | make test-outbox | SIGKILL publisher, затем ready без нового upload | evidence/verification.json | +| Backoff/jitter, failed registry, replay | make test-jobs | Постоянные/временные ошибки различаются, replay безопасен | evidence/jobs.json | +| Crash после PUT до ack, lease и cleanup | make test-jobs; make test-pipeline | Тот же результат, prefetch=1; активный staging сохранён | evidence/jobs.json; evidence/verification.json | +| Выбранный трек 5 | TODO | TODO | TODO | -Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные -тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две -реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД -для pytest выполняется только в выделенной базе *_test, Redis — DB15. -Публичные tests/ и обязательные contracts/ не изменены. +## Эксперименты и отказы -Отдельные сценарии на уже работающем изолированном стенде: +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| Crash после PUT до COMMIT | Повтор той же job | ready на второй попытке, те же четыре объекта и SHA-256 | После lease/retry; потерь нет | +| Crash после COMMIT до ack | Готовый дубль без обработки | Metadata и Last-Modified не изменились | После доставки; потерь нет | +| Временная / постоянная ошибка | Ограниченный retry / немедленный failed | Пять / одна попытка; явный replay завершился ready | В пределах бюджета; потерь нет | -```sh -make test-outbox-recovery BASE_URL=http://localhost:28080 -make test-jobs BASE_URL=http://localhost:28080 -``` +Трек 5 — **TODO**: гипотеза, методика, измерения и выводы. -При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты -в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта, -создают синтетические данные и возвращают штатные worker/publisher в finally. -Перезапуски сохраняют volumes. Сброса пользовательских данных нет. +## Ограничения и следующие шаги -Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json; -подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json. -Секреты, tokens и исходные broker payload в evidence не записываются. - -## Фактический результат проверок - -Все перечисленные проверки завершились успешно на отдельном проекте -`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes: - -| Проверка | Результат | -| --- | --- | -| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков | -| Дополнительные timeout и failed retention/SHA-256 | 2 passed | -| Контракт, OpenAPI и Compose config | PASS | -| Публичный smoke | PASS | -| 24 конкурентных повтора сообщения через две API-реплики | PASS | -| Публичные persistence и failover | PASS | -| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload | -| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume | -| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов | -| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified | -| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен | -| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён | -| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 | -| Неизменность публичных tests/contracts/README и whitespace | PASS | - -Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены -в последующие полные запуски make test-grade4. Тесты намеренно меняют только -выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях -используют собственную БД тестового Compose-проекта. - -Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885, -42.633 секунды; все попали в соответствующие интервалы jitter. Задержка -последней failed-попытки сохраняется для диагностики, автоматический retry -после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно. - -Во время проверки исправлена проблема fault hooks с PID 1 через init:true. -Старые собственные проверки lab4 адаптированы к обязательному переходу -202 queued → ready; публичные проверки не изменены. Цель make test-grade4 -переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры -не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий. - -## Границы результата - -Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes. -PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта. -Ожидание в очереди не ограничено; admission control и исследование перегрузки -относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение -алгоритмов требует решения о новой версии/миграции результатов. - -Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), -[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms), -[aio-pika](https://docs.aio-pika.com/quick-start.html), -[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), -[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp). +Один Docker-хост, без репликации хранилищ и защиты от потери диска. Очередь не имеет +admission control; pipeline фиксирован на версии 1. Доставка как минимум один раз, +с идемпотентным результатом. Исследование для уровня 5 — **TODO**. diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py index b30318ec..83b13acc 100644 --- a/app/domain/idempotency.py +++ b/app/domain/idempotency.py @@ -29,7 +29,7 @@ def validate_idempotency_key(key: str | None) -> str: def request_fingerprint(new_message: CreateMessage) -> str: """Хеширует нормализованное содержимое запроса.""" body = new_message.model_dump(mode="json") - # Сохраняет отпечатки текстовых запросов lab3 и равенство absent / []. + # Сохраняет прежние отпечатки текстовых запросов и равенство absent / []. if not body["attachment_ids"]: body.pop("attachment_ids") canonical = json.dumps(body, sort_keys=True, ensure_ascii=False) diff --git a/compose.yaml b/compose.yaml index e17a0985..af2748ea 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -name: mtusi-messenger-lab5 +name: messenger x-api-image: &api-image build: @@ -73,7 +73,7 @@ services: context: . dockerfile: infra/minio/Dockerfile target: server - image: mtusi-messenger-minio:lab5 + image: messenger-minio:local command: ["server", "/data", "--console-address", "127.0.0.1:9001"] restart: unless-stopped environment: @@ -104,7 +104,7 @@ services: context: . dockerfile: infra/minio/Dockerfile target: bootstrap - image: mtusi-messenger-minio-bootstrap:lab5 + image: messenger-minio-bootstrap:local environment: S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env} S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env} diff --git a/evidence/lab5-jobs.json b/evidence/jobs.json similarity index 66% rename from evidence/lab5-jobs.json rename to evidence/jobs.json index 639dc0fb..57fd54d7 100644 --- a/evidence/lab5-jobs.json +++ b/evidence/jobs.json @@ -3,7 +3,7 @@ "results": [ { "scenario": "worker_stopped_restart_all", - "attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707", + "attachment_id": "f5cc75ba-d2c7-43ff-8cf4-123b723118b8", "queue_before": { "name": "attachment-jobs", "messages_unacknowledged": 0, @@ -19,13 +19,13 @@ "messages_ready": 1 }, "after": { - "job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf", + "job_id": "9adcea8e-9046-4736-9763-c24bb6daf417", "status": "ready", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:27:44.556734+00:00", - "next_attempt_at": "2026-11-10 03:26:56.662360+00:00", + "updated_at": "2026-11-10 04:36:45.543612+00:00", + "next_attempt_at": "2026-11-10 04:35:56.727228+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -33,41 +33,41 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:27:44.548000+00:00" + "last_modified": "2026-11-10 04:36:45.525000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:27:44.552000+00:00" + "last_modified": "2026-11-10 04:36:45.533000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:27:44.543000+00:00" + "last_modified": "2026-11-10 04:36:45.521000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:27:44.536000+00:00" + "last_modified": "2026-11-10 04:36:45.508000+00:00" } ] } }, { "scenario": "crash-after-object", - "attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc", + "attachment_id": "1927c264-c721-4de1-8914-32d9beab48e5", "exit_code": 137, "before": { - "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "job_id": "bda1d00d-41c7-4d28-b2e5-07759be55985", "status": "processing", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:27:49.144648+00:00", - "next_attempt_at": "2026-11-10 03:27:47.218612+00:00", + "updated_at": "2026-11-10 04:36:50.666631+00:00", + "next_attempt_at": "2026-11-10 04:36:48.673346+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -75,36 +75,36 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:27:49.912000+00:00" + "last_modified": "2026-11-10 04:36:51.443000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:27:49.917000+00:00" + "last_modified": "2026-11-10 04:36:51.448000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:27:49.907000+00:00" + "last_modified": "2026-11-10 04:36:51.434000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:27:49.894000+00:00" + "last_modified": "2026-11-10 04:36:51.420000+00:00" } ] }, "after": { - "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "job_id": "bda1d00d-41c7-4d28-b2e5-07759be55985", "status": "ready", "attempt": 2, "published_attempt": 2, "limit": 5, - "updated_at": "2026-11-10 03:29:24.765539+00:00", - "next_attempt_at": "2026-11-10 03:29:23.142176+00:00", + "updated_at": "2026-11-10 04:38:26.694238+00:00", + "next_attempt_at": "2026-11-10 04:38:25.019759+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -112,41 +112,41 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:29:24.750000+00:00" + "last_modified": "2026-11-10 04:38:26.675000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:29:24.758000+00:00" + "last_modified": "2026-11-10 04:38:26.686000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:29:24.742000+00:00" + "last_modified": "2026-11-10 04:38:26.666000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:29:24.725000+00:00" + "last_modified": "2026-11-10 04:38:26.656000+00:00" } ] } }, { "scenario": "crash-before-ack", - "attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1", + "attachment_id": "e02d817d-9615-4058-ae5c-63ab895aa6f5", "exit_code": 137, "before": { - "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "job_id": "9bfea78c-ce05-4a00-bf76-a2213ed3a8df", "status": "ready", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:29:29.833142+00:00", - "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "updated_at": "2026-11-10 04:38:32.063984+00:00", + "next_attempt_at": "2026-11-10 04:38:29.436074+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -154,36 +154,36 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:29:29.824000+00:00" + "last_modified": "2026-11-10 04:38:32.054000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:29:29.829000+00:00" + "last_modified": "2026-11-10 04:38:32.059000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:29:29.818000+00:00" + "last_modified": "2026-11-10 04:38:32.048000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:29:29.805000+00:00" + "last_modified": "2026-11-10 04:38:32.032000+00:00" } ] }, "after": { - "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "job_id": "9bfea78c-ce05-4a00-bf76-a2213ed3a8df", "status": "ready", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:29:29.833142+00:00", - "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "updated_at": "2026-11-10 04:38:32.063984+00:00", + "next_attempt_at": "2026-11-10 04:38:29.436074+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -191,102 +191,102 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:29:29.824000+00:00" + "last_modified": "2026-11-10 04:38:32.054000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:29:29.829000+00:00" + "last_modified": "2026-11-10 04:38:32.059000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:29:29.818000+00:00" + "last_modified": "2026-11-10 04:38:32.048000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:29:29.805000+00:00" + "last_modified": "2026-11-10 04:38:32.032000+00:00" } ] } }, { "scenario": "fail-transient", - "attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea", + "attachment_id": "a9c15ce8-cacf-43c0-8373-8e3ff5fa23aa", "samples": [ { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "queued", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:29:45.863494+00:00", - "next_attempt_at": "2026-11-10 03:29:50.476742+00:00", + "updated_at": "2026-11-10 04:38:46.283315+00:00", + "next_attempt_at": "2026-11-10 04:38:50.994899+00:00", "last_error_code": "processing_timeout", "permanent": false, "objects": [] }, { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "queued", "attempt": 2, "published_attempt": 2, "limit": 5, - "updated_at": "2026-11-10 03:29:51.216904+00:00", - "next_attempt_at": "2026-11-10 03:29:58.053724+00:00", + "updated_at": "2026-11-10 04:38:51.039278+00:00", + "next_attempt_at": "2026-11-10 04:38:59.940067+00:00", "last_error_code": "processing_timeout", "permanent": false, "objects": [] }, { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "queued", "attempt": 3, "published_attempt": 3, "limit": 5, - "updated_at": "2026-11-10 03:29:58.279470+00:00", - "next_attempt_at": "2026-11-10 03:30:15.602067+00:00", + "updated_at": "2026-11-10 04:39:00.122276+00:00", + "next_attempt_at": "2026-11-10 04:39:19.754304+00:00", "last_error_code": "processing_timeout", "permanent": false, "objects": [] }, { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "queued", "attempt": 4, "published_attempt": 4, "limit": 5, - "updated_at": "2026-11-10 03:30:16.386336+00:00", - "next_attempt_at": "2026-11-10 03:30:47.271081+00:00", + "updated_at": "2026-11-10 04:39:20.250087+00:00", + "next_attempt_at": "2026-11-10 04:39:56.121121+00:00", "last_error_code": "processing_timeout", "permanent": false, "objects": [] }, { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "failed", "attempt": 5, "published_attempt": 5, "limit": 5, - "updated_at": "2026-11-10 03:30:47.622716+00:00", - "next_attempt_at": "2026-11-10 03:31:30.255216+00:00", + "updated_at": "2026-11-10 04:39:56.526646+00:00", + "next_attempt_at": "2026-11-10 04:41:15.809038+00:00", "last_error_code": "processing_timeout", "permanent": false, "objects": [] } ], "after": { - "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59", "status": "ready", "attempt": 6, "published_attempt": 6, "limit": 10, - "updated_at": "2026-11-10 03:30:59.696883+00:00", - "next_attempt_at": "2026-11-10 03:30:53.135190+00:00", + "updated_at": "2026-11-10 04:40:09.651133+00:00", + "next_attempt_at": "2026-11-10 04:40:03.031585+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -294,25 +294,25 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:30:59.687000+00:00" + "last_modified": "2026-11-10 04:40:09.642000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:30:59.692000+00:00" + "last_modified": "2026-11-10 04:40:09.647000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:30:59.683000+00:00" + "last_modified": "2026-11-10 04:40:09.637000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:30:59.670000+00:00" + "last_modified": "2026-11-10 04:40:09.624000+00:00" } ] }, @@ -320,29 +320,29 @@ }, { "scenario": "fail-permanent", - "attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9", + "attachment_id": "d8a6e312-e468-4e4b-a7ee-1d84751d5843", "samples": [ { - "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "job_id": "0da2ef52-65a8-4a74-8d43-015666e2975c", "status": "failed", "attempt": 1, "published_attempt": 1, "limit": 5, - "updated_at": "2026-11-10 03:31:12.333162+00:00", - "next_attempt_at": "2026-11-10 03:31:16.827024+00:00", + "updated_at": "2026-11-10 04:40:24.826416+00:00", + "next_attempt_at": "2026-11-10 04:40:29.630307+00:00", "last_error_code": "injected_permanent", "permanent": true, "objects": [] } ], "after": { - "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "job_id": "0da2ef52-65a8-4a74-8d43-015666e2975c", "status": "ready", "attempt": 2, "published_attempt": 2, "limit": 6, - "updated_at": "2026-11-10 03:31:24.282794+00:00", - "next_attempt_at": "2026-11-10 03:31:17.137221+00:00", + "updated_at": "2026-11-10 04:40:40.004814+00:00", + "next_attempt_at": "2026-11-10 04:40:30.084006+00:00", "last_error_code": null, "permanent": null, "objects": [ @@ -350,25 +350,25 @@ "name": "crop.jpg", "size": 2462, "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", - "last_modified": "2026-11-10 03:31:24.272000+00:00" + "last_modified": "2026-11-10 04:40:39.988000+00:00" }, { "name": "optimized.webp", "size": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", - "last_modified": "2026-11-10 03:31:24.278000+00:00" + "last_modified": "2026-11-10 04:40:39.996000+00:00" }, { "name": "thumbnail.jpg", "size": 5813, "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", - "last_modified": "2026-11-10 03:31:24.267000+00:00" + "last_modified": "2026-11-10 04:40:39.982000+00:00" }, { "name": "v1", "size": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", - "last_modified": "2026-11-10 03:31:24.254000+00:00" + "last_modified": "2026-11-10 04:40:39.968000+00:00" } ] }, diff --git a/evidence/lab5-verification.json b/evidence/verification.json similarity index 87% rename from evidence/lab5-verification.json rename to evidence/verification.json index 2beb4658..6c80daea 100644 --- a/evidence/lab5-verification.json +++ b/evidence/verification.json @@ -157,7 +157,7 @@ { "name": "worker_fault_retry_replay", "exit_code": 0, - "evidence": "lab5-jobs.json", + "evidence": "jobs.json", "scenarios": [ "worker_stopped_restart_all", "crash-after-object", @@ -267,5 +267,42 @@ "Single Docker host, no disk-loss or cluster failover claim", "Grade 5 experiments not performed", "No production-scale performance/security claim" - ] + ], + "post_test_cleanup": { + "date": "2026-11-10", + "project_removed": "mtusi-lab5-verification", + "test_volumes_removed": 4, + "conserver_removed": true, + "old_lab3_image_removed": true, + "old_empty_networks_removed": true, + "current_lab5_images_retained": true + }, + "maintenance_checks": { + "status": "passed", + "command": "make test-suite", + "project": "messenger-test", + "pipeline_tests": 42, + "checks": [ + "contract", + "OpenAPI", + "Compose startup", + "public smoke", + "MinIO private access/bootstrap", + "outbox recovery", + "worker_stopped_restart_all", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent" + ], + "standalone_nginx": "passed", + "test_resources_removed": true, + "obsolete_image_tags_removed": [ + "mtusi-messenger-minio-bootstrap:lab5", + "mtusi-messenger-minio:lab5", + "mtusi-messenger:lab5" + ], + "regression_tests_preserved": true, + "note": "Earlier recorded commands retain their original names as historical evidence; operational scripts and resource names are now neutral." + } } diff --git a/infra/README.md b/infra/README.md index a6b520db..3f2954c9 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,27 +1,16 @@ -# Инфраструктура студента +# Инфраструктура -Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis -только на localhost. Nginx направляет запросы на две API-реплики. +compose.yaml запускает две API-реплики за nginx, PostgreSQL, Redis сессий, +RabbitMQ, приватный MinIO, publisher, worker и cleanup. Все environment-настройки +берутся из .env; credentials передаются файлами secrets. Миграции и bootstrap +завершаются до старта зависимых сервисов. -PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2 -с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь, -persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata. -Broker и MinIO не публикуют внешние порты. API не получает credentials брокера. +Publisher читает outbox PostgreSQL и ждёт подтверждения RabbitMQ. Worker фиксирует +результат в MinIO/PostgreSQL до ack; prefetch=1 на consumer. init:true обеспечивает +корректные сигналы. Broker и хранилища не публикуют внешние порты. -Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет -original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack. -Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого -consumer. Publisher/worker используют init:true для корректных сигналов и -воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает. - -Все environment-настройки Compose берутся из .env; образец — .env.example. -Credentials передаются файлами secrets; make init создаёт только отсутствующие. -Migrate и minio-bootstrap завершаются до старта зависимых сервисов. -Перед обновлением остановить старые процессы без удаления volumes. Миграция -a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ. - -Restart hook сохраняет volumes и состояние заранее остановленного worker. -Failover hook останавливает выбранную API-реплику. Retry, retention, операторские -команды, ограничения и результаты проверок описаны в REPORT.md. -Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии — -make test-outbox-recovery и make test-jobs на выбранном работающем стенде. +make up/down управляют основным проектом messenger и сохраняют volumes. +make test-suite использует отдельный messenger-test и удаляет его вместе с +volumes и сетями при завершении. compose.test.yaml открывает тестовые PostgreSQL +и Redis только на localhost. Полная регрессия доступна через make test-unit, +отдельные проверки — make test-nginx, test-storage, test-outbox и test-jobs. diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py index 280c7dca..302ebc6d 100644 --- a/migrations/versions/c35e91a2f706_async_attachment_jobs.py +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -1,4 +1,4 @@ -"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации.""" +"""Состояния, варианты и устойчивые jobs с намерением публикации.""" from alembic import op import sqlalchemy as sa diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py index 767fbe7e..2576cad3 100644 --- a/migrations/versions/e7048b6da913_attachment_crop_webp.py +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -1,4 +1,4 @@ -"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений.""" +"""Ключи crop/WebP и дообработка старых ready-вложений.""" from alembic import op import sqlalchemy as sa diff --git a/scripts/check_grade4.py b/scripts/check_grade4.py deleted file mode 100644 index a2c90bcd..00000000 --- a/scripts/check_grade4.py +++ /dev/null @@ -1,355 +0,0 @@ -"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда.""" - -import asyncio -from collections import Counter -from concurrent.futures import ThreadPoolExecutor -import hashlib -import json -from pathlib import Path -import subprocess -from threading import Event -import time -from uuid import uuid4 - -import asyncpg - -from scripts.deployment import ComposeProject, ROOT, deployment_lock -from scripts.failover_all import exercise, failure_kind, history -from scripts.rolling_update import update -from tests.client import Client, Failure, arguments, check, run - -PROBE = """import json, sys, urllib.request, urllib.error -payload = json.load(sys.stdin) -results = [] -for item in payload: - data = json.dumps(item['body']).encode() if 'body' in item else None - request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET')) - try: - response = urllib.request.urlopen(request, timeout=7) - except urllib.error.HTTPError as error: - response = error - with response: - results.append({'status': response.status, 'body': json.loads(response.read())}) -print(json.dumps(results)) -""" - - -def probe(project, runner, requests): - """Обращается к внутреннему API, передавая токен только через stdin.""" - result = subprocess.run( - ["docker", "exec", "-i", runner, "python", "-c", PROBE], - input=json.dumps(requests), - text=True, - capture_output=True, - timeout=10, - ) - check(result.returncode == 0, "internal HTTP probe failed") - return json.loads(result.stdout) - - -def audit(project): - """Проверяет фактические лимиты, порты и область доступа к секретам.""" - allowed = { - "postgres": {"db_password"}, - "redis": {"redis_password"}, - "migrate": {"db_password"}, - "api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"}, - "minio": {"minio_root_user", "minio_root_password"}, - "minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"}, - "cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"}, - "nginx": set(), - } - ids = project.run("ps", "--all", "--quiet").splitlines() - containers = json.loads(project.docker("inspect", *ids)) - for item in containers: - service = item["Config"]["Labels"]["com.docker.compose.service"] - check(service in allowed, "unexpected service") - host = item["HostConfig"] - check( - host["Memory"] > 0 and host["NanoCpus"] > 0, - service + ": missing CPU/RAM limit", - ) - env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]} - check( - not env_names - & { - "DB_PASSWORD", - "REDIS_PASSWORD", - "POSTGRES_PASSWORD", - "REDISCLI_AUTH", - "DATABASE_URL", - "MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", - }, - service + ": password in environment", - ) - secrets = { - Path(mount["Destination"]).name - for mount in item["Mounts"] - if mount["Destination"].startswith("/run/secrets/") - } - check(secrets == allowed[service], service + ": incorrect secret scope") - if service in {"api", "minio"}: - check(not host["PortBindings"], service + " publishes ports") - for mount in item["Mounts"]: - if mount["Destination"].startswith("/run/secrets/"): - check(not mount["RW"], "writable secret mount") - check( - "container_name" not in project.config["services"][service], - "fixed container name", - ) - check( - len(containers) == len(allowed) - 1 + len(project.replicas()), - "unexpected container count", - ) - print( - "PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords", - flush=True, - ) - - -async def graceful(project, base_url): - """Удерживает реальный POST в БД и завершает его после SIGTERM.""" - replicas = project.replicas() - target, runner = replicas[:2] - client = Client(base_url) - actor, _, _, chat = await asyncio.to_thread(client.bootstrap) - path = f"/api/v1/chats/{chat['id']}/messages" - key = uuid4().hex - raw = json.dumps([actor["id"], "POST", path, key]).encode() - lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True) - database = project.config["services"]["postgres"] - env = database["environment"] - conn = await asyncpg.connect( - host="127.0.0.1", - port=int(database["ports"][0]["published"]), - database=env["POSTGRES_DB"], - user=env["POSTGRES_USER"], - password=(ROOT / "secrets/db_password").read_text().strip(), - timeout=3, - ) - network = project.config['networks']['default']['name'] - address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}' - address = project.docker( - "inspect", - "--format", - address_template, - target.id, - ) - check(bool(address), 'API has no address in the application network') - port = project.config["services"]["api"]["environment"]["APP_PORT"] - direct = f"http://{address}:{port}" - started = time.monotonic() - task = None - try: - async with conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id) - task = asyncio.create_task( - asyncio.to_thread( - probe, - project, - runner.id, - [ - { - "url": direct + path, - "method": "POST", - "body": {"text": "graceful-" + key}, - "headers": { - "Authorization": "Bearer " + actor["token"], - "Idempotency-Key": key, - "Content-Type": "application/json", - }, - } - ], - ) - ) - deadline = time.monotonic() + 3 - while not await conn.fetchval( - "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())" - ): - check(time.monotonic() < deadline, "POST did not reach database lock") - await asyncio.sleep(0.02) - started = time.monotonic() - await asyncio.to_thread( - project.docker, "kill", "--signal", "SIGTERM", target.id - ) - results = await asyncio.to_thread( - probe, - project, - runner.id, - [ - {"url": direct + "/health/ready"}, - {"url": direct + "/health/live"}, - {"url": direct + "/api/v1/users/me"}, - ], - ) - check( - [item["status"] for item in results] == [503, 200, 503], - "incorrect drain readiness/liveness/new work", - ) - await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started))) - result = (await task)[0] - check(result["status"] == 201, "in-flight POST failed during SIGTERM") - await asyncio.to_thread(project.docker, "wait", target.id, timeout=10) - state = json.loads( - project.docker("inspect", "--format", "{{json .State}}", target.id) - ) - check( - not state["OOMKilled"] and state["ExitCode"] != 137, - "forced shutdown instead of graceful completion", - ) - check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget") - items = await asyncio.to_thread(history, client, actor, chat) - check( - result["body"]["id"] in {item["id"] for item in items}, - "acknowledged in-flight message lost", - ) - print( - f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s", - flush=True, - ) - finally: - await conn.close() - if task: - await asyncio.gather(task, return_exceptions=True) - if any(item.id == target.id and item.running for item in project.replicas()): - await asyncio.to_thread(project.docker, "stop", target.id) - await asyncio.to_thread(project.docker, "start", target.id) - await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas}) - await asyncio.to_thread(project.wait_routed, target.instance_id, base_url) - - -class ObservedProject(ComposeProject): - """Подсчитывает миграции и проверяет готовность перед заменой.""" - - def __init__(self): - self.migrations = 0 - self.replacements = 0 - self.replacement_started = [] - super().__init__() - - def run(self, *args, **kwargs): - if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"): - self.migrations += 1 - return super().run(*args, **kwargs) - - def docker(self, *args, **kwargs): - if args[0] == "stop": - healthy = [ - item - for item in self.replicas() - if item.id != args[1] and item.running and item.health == "healthy" - ] - check(len(healthy) >= 2, "rolling update reduces healthy capacity") - self.replacements += 1 - self.replacement_started.append(time.monotonic()) - return super().docker(*args, **kwargs) - - -def rolling(base_url): - """Заменяет образ под постоянной записью с проверкой сохранности.""" - project = ObservedProject() - before = project.replicas() - client = Client(base_url) - actor, _, _, chat = client.bootstrap() - stop = Event() - results, errors = [], [] - - def worker(): - local = Client(base_url, timeout=2) - key = uuid4().hex - while not stop.is_set(): - started = time.monotonic() - try: - result = local.message(actor, chat, "rolling-" + key, key=key) - results.append( - (started, time.monotonic(), result["json"]["id"], result["elapsed"]) - ) - key = uuid4().hex - except Failure as error: - errors.append(failure_kind(error)) - stop.wait(0.08) - - with ThreadPoolExecutor(max_workers=2) as pool: - jobs = [pool.submit(worker) for _ in range(2)] - try: - time.sleep(1) - started = time.monotonic() - update(project, base_url, revision="verification-" + uuid4().hex[:12]) - finished = time.monotonic() - time.sleep(1) - finally: - stop.set() - for job in jobs: - job.result(timeout=5) - after = project.replicas() - check( - project.migrations == 1 and project.replacements == len(before), - "migration/replacement count incorrect", - ) - check( - {item.image for item in before}.isdisjoint({item.image for item in after}), - "new image was not deployed", - ) - check( - {item.id for item in before}.isdisjoint({item.id for item in after}), - "old containers remain", - ) - writes = [row for row in results if started <= row[0] <= finished] - check(bool(writes), "no writes during update") - check( - any(project.replacement_started[0] <= row[0] <= finished for row in writes), - "no writes during container replacement", - ) - items = history(client, actor, chat) - check( - {row[2] for row in results} <= {item["id"] for item in items}, - "rolling update lost confirmed messages", - ) - check( - len(items) == len({item["text"] for item in items}), - "rolling update created duplicate messages", - ) - latencies = sorted(row[3] for row in writes) - print( - json.dumps( - { - "scenario": "rolling update", - "replacements": project.replacements, - "migration_runs": project.migrations, - "successful_writes": len(writes), - "failed_attempts": len(errors), - "errors_by_kind": dict(Counter(errors)), - "elapsed_seconds": round(finished - started, 3), - "successful_rps": round(len(writes) / (finished - started), 3), - "p95_seconds": round( - latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3 - ), - } - ), - flush=True, - ) - - -def main(): - """Выполняет проверки только на отдельном тестовом проекте.""" - - def configure(parser): - parser.add_argument( - "--phase", choices=("all", "update-and-failover"), default="all" - ) - - args = arguments("Grade 4 integration checks", configure) - project = ComposeProject() - check(project.name.endswith("-test"), "use isolated test Compose project") - with deployment_lock(project): - audit(project) - if args.phase == "all": - asyncio.run(graceful(project, args.base_url)) - rolling(args.base_url) - args.duration, args.concurrency = 10, 2 - exercise(project, args) - audit(project) - - -if __name__ == "__main__": - run(main) diff --git a/scripts/check_idempotency.py b/scripts/check_idempotency.py deleted file mode 100644 index ae9dfa20..00000000 --- a/scripts/check_idempotency.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Проверяет конкурентные повторы через балансировщик.""" - -from concurrent.futures import ThreadPoolExecutor -from uuid import uuid4 - -from tests.client import Client, arguments, check, run - - -def main() -> None: - """Отправляет один ключ параллельно через разные реплики.""" - args = arguments("Concurrent idempotency through load balancer") - client = Client(args.base_url, args.ca_file) - alice, _, _, chat = client.bootstrap() - key = uuid4().hex - body = "parallel-" + key - - def send(_): - return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key) - - with ThreadPoolExecutor(max_workers=8) as pool: - results = list(pool.map(send, range(24))) - ids = {result["json"]["id"] for result in results} - replicas = {result["headers"]["X-Instance-Id"] for result in results} - check(len(ids) == 1, "parallel retries created duplicate messages") - check(len(replicas) >= 2, "parallel retries did not reach two API replicas") - history = client.call( - "GET", - f"/api/v1/chats/{chat['id']}/messages?limit=100", - actor=alice, - schema="MessagePage", - )["json"]["items"] - check( - len(history) == 1 and history[0] == results[0]["json"], "history has duplicates" - ) - print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message") - - -if __name__ == "__main__": - run(main) diff --git a/scripts/check_lab5_jobs.py b/scripts/check_jobs.py similarity index 96% rename from scripts/check_lab5_jobs.py rename to scripts/check_jobs.py index ebba45fa..dc724b66 100644 --- a/scripts/check_lab5_jobs.py +++ b/scripts/check_jobs.py @@ -1,4 +1,4 @@ -"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные.""" +"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные.""" import argparse from datetime import datetime @@ -79,7 +79,7 @@ def verify_result(client, actor, attachment, original): def crash(client, actor, chat, mode): - name='lab5-worker-fault-'+uuid4().hex + name='messenger-worker-fault-'+uuid4().hex try: require_compose('stop','worker') attachment,original=client.attachment(actor,chat) @@ -106,7 +106,7 @@ def crash(client, actor, chat, mode): def retry_replay(client,actor,chat,permanent): - name='lab5-retry-fault-'+uuid4().hex + name='messenger-retry-fault-'+uuid4().hex samples=[] configuration=ComposeProject().config['services']['worker']['environment'] retry_base=float(configuration['JOB_RETRY_SECONDS']) @@ -189,10 +189,11 @@ def persistence(client,actor,chat): def main(): parser=argparse.ArgumentParser() parser.add_argument('--base-url',required=True) + parser.add_argument('--ca-file') parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all') - parser.add_argument('--output',default='evidence/lab5-jobs.json') + parser.add_argument('--output',default='evidence/jobs.json') args=parser.parse_args() - client=Client(args.base_url,timeout=40) + client=Client(args.base_url,args.ca_file,timeout=40) aliases={r.instance_id for r in ComposeProject().replicas() if r.running} check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL') actor,_,_,chat=client.bootstrap() diff --git a/scripts/check_nginx.py b/scripts/check_nginx.py index e4881fc5..cd350344 100644 --- a/scripts/check_nginx.py +++ b/scripts/check_nginx.py @@ -25,7 +25,7 @@ def check(condition: bool, message: str) -> None: class Stand: def __init__(self, directory: str) -> None: self.directory = Path(directory) - self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12] + self.project = "messenger-nginx-check-" + uuid4().hex[:12] self.base_url = "" backend = self.directory / "backend.conf" backend.write_text( @@ -246,7 +246,7 @@ def run_checks(stand: Stand) -> None: def main() -> None: - with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory: + with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory: stand = Stand(directory) try: run_checks(stand) @@ -254,7 +254,7 @@ def main() -> None: print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True) raise finally: - stand.compose("down", "--remove-orphans", "--timeout", "1") + stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1") if __name__ == "__main__": diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py index e5379afb..288257e9 100644 --- a/scripts/check_outbox_recovery.py +++ b/scripts/check_outbox_recovery.py @@ -34,7 +34,7 @@ def main() -> None: parser.add_argument("--ca-file") args = parser.parse_args() running = set(require_compose("ps", "--status", "running", "--services").splitlines()) - check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario") + check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario") client = Client(args.base_url, args.ca_file, timeout=40) project = ComposeProject() aliases = {replica.instance_id for replica in project.replicas() if replica.running} @@ -42,7 +42,7 @@ def main() -> None: check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project") alice, _, _, chat = client.bootstrap() attachment = None - fault_container = "lab5-outbox-fault-" + uuid4().hex + fault_container = "messenger-outbox-fault-" + uuid4().hex try: require_compose("stop", "publisher") attachment, original = client.attachment(alice, chat) diff --git a/scripts/check_s3.py b/scripts/check_s3.py index c8422781..f94f6a9e 100644 --- a/scripts/check_s3.py +++ b/scripts/check_s3.py @@ -1,8 +1,6 @@ """Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ.""" import json -import base64 -from pathlib import Path import subprocess import time @@ -70,12 +68,15 @@ asyncio.run(main()) def main(): def configure(parser): parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта') - parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД') args = arguments('MinIO private bucket and failure checks', configure) project = ComposeProject() client = Client(args.base_url, args.ca_file, timeout=40) alice, _, _, chat = client.bootstrap() + aliases = {replica.instance_id for replica in project.replicas() if replica.running} + live = client.call('GET', '/health/live', schema='Health') + check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project') attachment, original = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) # Повторный bootstrap не должен очистить bucket или сменить используемые credentials. for _ in range(2): project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100) @@ -96,21 +97,6 @@ def main(): check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback') path = '/api/v1/attachments/' + attachment['id'] + '/content' check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original') - if args.with_faults: - project.run('stop', 'cleanup') - try: - fault_script = Path(__file__).with_name('s3_faults.py').read_text() - result = subprocess.run( - ['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script], - input=json.dumps({'user': alice['user'], 'chat_id': chat['id'], - 'image': base64.b64encode(original).decode()}), - text=True, capture_output=True, timeout=60, - ) - check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)') - check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed') - finally: - project.run('start', 'cleanup') - check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original') if args.with_outage: try: project.run('stop', 'minio') @@ -129,8 +115,6 @@ def main(): check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart') print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console' + (', MinIO outage/recovery' if args.with_outage else '')) - if args.with_faults: - print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup') if __name__ == '__main__': diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py deleted file mode 100644 index f4ff9ef1..00000000 --- a/scripts/collect_evidence.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence.""" - -from datetime import datetime, timezone -import hashlib -from importlib.metadata import version -import json -import os -from pathlib import Path -import platform -import re -import shlex -import subprocess -from tempfile import TemporaryDirectory -from zoneinfo import ZoneInfo - -ROOT = Path(__file__).resolve().parents[1] -PROJECT = 'mtusi-messenger-lab5-test' -EVIDENCE = ROOT / 'evidence' -ENVIRONMENT = { - **os.environ, - 'COMPOSE_PROJECT_NAME': PROJECT, - 'COMPOSE_FILE': 'compose.yaml:compose.test.yaml', - 'LB_HOST_PORT': '28080', -} -COMPOSE = ['docker', 'compose', '--env-file', '.env'] -NONSECRET_SETTINGS = { - 'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT', - 'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS', - 'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS', - 'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS', - 'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS', - 'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS', - 'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS', - 'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES', - 'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS', - 'S3_CLEANUP_INTERVAL_SECONDS', -} -SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}' -IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}' - - -def timestamp(): - return datetime.now(timezone.utc).isoformat() - - -def invoke(command): - return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - -def read_command(command): - result = invoke(command) - if result.returncode: - raise RuntimeError('Не выполнена команда: ' + shlex.join(command)) - return result.stdout.strip() - - -def source_manifest(): - names = { - '.dockerignore', '.env.example', - 'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml', - 'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json', - } - for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'): - names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*') - if path.is_file() and '__pycache__' not in path.parts - and path.suffix != '.md' and not path.name.startswith('.')) - return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names)) - - -def sanitize(text, secrets, staging): - clean = text.replace(str(ROOT), '.').replace(str(staging), '') - for secret in secrets: - clean = clean.replace(secret, '') - clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 ', clean) - clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1', clean) - return clean - - -def container_snapshot(images): - ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines() - rows = [] - if ids: - output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids]) - rows = [json.loads(line) for line in output.splitlines()] - for row in rows: - image = row['image_id'] - if image not in images: - images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image])) - return rows - - -def remaining_resources(): - return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter', - f'label=com.docker.compose.project={PROJECT}']).splitlines() - for resource in ('container', 'network', 'volume')} - - -def extract_metrics(output, upload_file): - rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')] - pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output) - if pytest is None: - raise RuntimeError('В выводе нет успешного итогового результата pytest') - return { - 'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])}, - 'uploads': json.loads(upload_file.read_text()), - 'public_failover': [row for row in rows if 'observed_instances_before' in row], - 'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'], - 'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row], - } - - -def main(): - if any(remaining_resources().values()): - raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно') - started = timestamp() - manifest = source_manifest() - secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()] - secrets = [value for value in secrets if value] - snapshots, images, commands, combined, nginx_output = {}, {}, [], [], '' - info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}'])) - config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json'])) - services = {name: { - 'image': service.get('image'), - 'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}), - 'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS}, - 'ports': service.get('ports', []), 'networks': list(service.get('networks', {})), - 'secret_names': [entry['source'] for entry in service.get('secrets', [])], - 'tmpfs': service.get('tmpfs', []), - } for name, service in config['services'].items()} - with TemporaryDirectory(prefix='lab4-evidence-') as folder: - staging = Path(folder) - upload_file = staging / 'uploads.json' - - def execute(command): - begin = timestamp() - result = invoke(command) - output = sanitize(result.stdout, secrets, staging) - display = sanitize(shlex.join(command), secrets, staging) - commands.append({'command': display, 'started_at_utc': begin, - 'finished_at_utc': timestamp(), 'exit_code': result.returncode}) - print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True) - if result.returncode: - print(output[-5000:], flush=True) - raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы') - return f'$ {display}\n{output}\n' - - cleanup_output = '' - try: - combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}'])) - snapshots['started'] = container_snapshot(images) - combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short'])) - combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}'])) - snapshots['before_restart'] = container_snapshot(images) - combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}'])) - snapshots['after_restart'] = container_snapshot(images) - combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}'])) - snapshots['after_failover'] = container_snapshot(images) - combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults'])) - combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)])) - combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080'])) - snapshots['after_update_and_failures'] = container_snapshot(images) - nginx_output = execute(['make', 'test-nginx']) - finally: - cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans']) - remaining = remaining_resources() - if any(remaining.values()): - raise RuntimeError('После очистки остались ресурсы тестового проекта') - if source_manifest() != manifest: - raise RuntimeError('Во время прогона изменились проверяемые исходники') - output = ''.join(combined) - metrics = extract_metrics(output, upload_file) - run = { - 'started_at_utc': started, 'finished_at_utc': timestamp(), - 'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(), - 'base_commit': read_command(['git', 'rev-parse', 'HEAD']), - 'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])), - 'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(), - 'project': PROJECT, 'runtime': {'python': platform.python_version(), - 'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')}, - 'compose': read_command(['docker', 'compose', 'version'])}, - 'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')}, - 'services': services, 'container_snapshots': snapshots, 'images': images, - 'commands': commands, 'remaining_test_resources': remaining, - 'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z', - 'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z', - } - artifacts = { - 'test-output.txt': output, 'nginx-output.txt': nginx_output, - 'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest, - 'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n', - 'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n', - } - for name, content in artifacts.items(): - if any(secret in content for secret in secrets): - raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы') - EVIDENCE.mkdir(exist_ok=True) - for name, content in artifacts.items(): - (EVIDENCE / name).write_text(content) - print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True) - - -if __name__ == '__main__': - main() diff --git a/scripts/deployment.py b/scripts/deployment.py index a2bf38d8..fba70f5e 100644 --- a/scripts/deployment.py +++ b/scripts/deployment.py @@ -1,17 +1,10 @@ """Общие операции с репликами текущего Compose-проекта.""" -from contextlib import contextmanager from dataclasses import dataclass -import fcntl import json import os from pathlib import Path -import ssl import subprocess -import tempfile -import time -from urllib.error import HTTPError, URLError -from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] @@ -65,43 +58,3 @@ class ComposeProject: Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"]) for row in map(json.loads, rows) ] - - def wait_healthy(self, ids: set[str], timeout: float = 40) -> None: - """Ожидает готовность конкретных реплик.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - current = {replica.id: replica for replica in self.replicas()} - if not ids <= current.keys(): - raise RuntimeError("Контейнер API исчез во время развёртывания") - if all(current[id].running and current[id].health == "healthy" for id in ids): - return - if any(not current[id].running for id in ids): - raise RuntimeError("Контейнер API завершился до готовности") - time.sleep(0.25) - raise TimeoutError("API не стал healthy за отведённое время") - - def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None: - """Ожидает ответа реплики через балансировщик.""" - context = ssl.create_default_context(cafile=ca_file) - deadline = time.monotonic() + 40 - while time.monotonic() < deadline: - try: - with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response: - if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id: - return - except (HTTPError, URLError, TimeoutError, OSError): - pass - time.sleep(0.1) - raise TimeoutError("Реплика не появилась в ответах балансировщика") - - -@contextmanager -def deployment_lock(project: ComposeProject): - """Исключает параллельные изменения одного стенда.""" - path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock" - with path.open("a") as lock: - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None - yield diff --git a/scripts/expiry_demo.sh b/scripts/expiry_demo.sh deleted file mode 100755 index cc32541d..00000000 --- a/scripts/expiry_demo.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -set -eu - -BASE_URL="${BASE_URL:-http://localhost:8080}" -TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}" -API="$BASE_URL/api/v1" -USERNAME="expiry_$(date +%s)" -PASSWORD="expiry-password-123" - -code() { - curl -s -o /dev/null -w '%{http_code}' "$@" -} - -curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \ - -d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}" -SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD") -TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') -EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])') -AUTH="Authorization: Bearer $TOKEN" - -echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT" -echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")" -sleep $((TTL + 1)) -printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")" -echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")" -NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \ - | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') -echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")" diff --git a/scripts/explain_history.py b/scripts/explain_history.py deleted file mode 100644 index 9d571c42..00000000 --- a/scripts/explain_history.py +++ /dev/null @@ -1,132 +0,0 @@ -"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных. - -Данные создаются в одной транзакции и откатываются в конце: база не меняется. -""" - -import asyncio -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncConnection - -from app.core.config import Settings -from app.core.database import create_db_engine - -TARGET_MESSAGES = 10_000 -NOISE_MESSAGES = 40_000 -PAGE = 51 - -HISTORY_FIRST_PAGE = """ -SELECT * FROM messages -WHERE chat_id = :chat_id -ORDER BY created_at, id -LIMIT :page -""" - -HISTORY_AFTER_CURSOR = """ -SELECT * FROM messages -WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id) -ORDER BY created_at, id -LIMIT :page -""" - - -async def seed(connection: AsyncConnection) -> dict[str, object]: - user_id, target_id, noise_id = uuid4(), uuid4(), uuid4() - await connection.execute( - text( - "INSERT INTO users (id, username, display_name, password_hash) " - "VALUES (:id, :username, 'Explain', 'x')" - ), - {"id": user_id, "username": "explain_" + uuid4().hex[:8]}, - ) - await connection.execute( - text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"), - {"a": target_id, "b": noise_id}, - ) - await connection.execute( - text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"), - {"a": target_id, "b": noise_id, "u": user_id}, - ) - for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)): - await connection.execute( - text( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " - "SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, " - "now() - make_interval(secs => :count - g) " - "FROM generate_series(1, :count) AS g" - ), - {"chat_id": chat_id, "user_id": user_id, "count": count}, - ) - await connection.execute(text("ANALYZE messages")) - - middle = ( - await connection.execute( - text( - "SELECT created_at, id FROM messages WHERE chat_id = :c " - "ORDER BY created_at, id OFFSET :o LIMIT 1" - ), - {"c": target_id, "o": TARGET_MESSAGES // 2}, - ) - ).one() - return { - "chat_id": target_id, - "after_created_at": middle.created_at, - "after_id": middle.id, - "page": PAGE, - } - - -async def explain( - connection: AsyncConnection, title: str, query: str, params: dict[str, object] -) -> None: - rows = await connection.execute( - text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params - ) - print(f"== {title}") - print(query.strip()) - print() - for row in rows: - print(row[0]) - print() - - -async def main() -> None: - engine = create_db_engine(Settings()) # type: ignore[call-arg] - async with engine.connect() as connection: - transaction = await connection.begin() - params = await seed(connection) - total = ( - await connection.execute(text("SELECT count(*) FROM messages")) - ).scalar_one() - print( - f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}" - ) - print() - - first_page = {"chat_id": params["chat_id"], "page": PAGE} - await explain( - connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page - ) - await explain( - connection, - "page after cursor in the middle of history", - HISTORY_AFTER_CURSOR, - params, - ) - - await connection.execute(text("SET LOCAL enable_indexscan = off")) - await connection.execute(text("SET LOCAL enable_bitmapscan = off")) - await explain( - connection, - "same page with index scans disabled (for comparison)", - HISTORY_AFTER_CURSOR, - params, - ) - - await transaction.rollback() - await engine.dispose() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/failover_all.py b/scripts/failover_all.py deleted file mode 100644 index 79745d7e..00000000 --- a/scripts/failover_all.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Сценарий поочерёдного отказа всех API-реплик под записью.""" - -from collections import Counter -from concurrent.futures import ThreadPoolExecutor -import json -import os -import re -import subprocess -from threading import Event, Lock -import time -from urllib.parse import quote -from uuid import uuid4 - -from scripts.deployment import ComposeProject, ROOT, deployment_lock -from tests.client import Client, Failure, arguments, check, run - - -def failure_kind(error: Failure) -> str: - """Различает HTTP-отказы, ошибки соединения и проверки контракта.""" - match = re.search(r"received (\d{3})$", str(error)) - if match: - return "http_" + match.group(1) - return "connection" if "connection/TLS failure" in str(error) else "contract" - - -def snapshot(project: ComposeProject, phase: str, target: str) -> list: - """Печатает состояние API без окружения и секретов.""" - replicas = project.replicas() - print(json.dumps({ - "scenario": "replica_failure", "phase": phase, "target": target, - "replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas], - }), flush=True) - return replicas - - -def history(client: Client, actor: dict, chat: dict) -> list[dict]: - """Читает всю историю без повторов курсора.""" - path = f"/api/v1/chats/{chat['id']}/messages?limit=100" - items = [] - cursors = set() - for _ in range(1000): - page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] - items.extend(page["items"]) - cursor = page["next_cursor"] - if cursor is None: - return items - check(cursor not in cursors, "cursor cycle") - cursors.add(cursor) - path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}" - raise Failure("history exceeded safety bound") - - -def exercise(project: ComposeProject, args) -> None: - """Останавливает каждую реплику и восстанавливает её перед следующей.""" - replicas = project.replicas() - check(len(replicas) >= 2, "at least two API replicas are required") - project.wait_healthy({item.id for item in replicas}) - client = Client(args.base_url, args.ca_file, timeout=2) - for item in replicas: - project.wait_routed(item.instance_id, client.base, args.ca_file) - actor, _, _, chat = client.bootstrap() - confirmed = set() - - for target in replicas: - snapshot(project, "before", target.instance_id) - stop = Event() - lock = Lock() - successes = [] - errors = [] - - def worker(): - local = Client(client.base, args.ca_file, timeout=2) - key = uuid4().hex - while not stop.is_set(): - started = time.monotonic() - try: - result = local.message(actor, chat, "failover-" + key, key=key) - with lock: - successes.append((started, time.monotonic(), result["json"]["id"])) - key = uuid4().hex - except Failure as error: - with lock: - errors.append(failure_kind(error)) - time.sleep(0.1) - - started = time.monotonic() - with ThreadPoolExecutor(max_workers=args.concurrency) as pool: - jobs = [pool.submit(worker) for _ in range(args.concurrency)] - try: - time.sleep(2) - stop_started = time.monotonic() - subprocess.run( - [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, - env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id}, - check=True, timeout=10, capture_output=True, - ) - stop_finished = time.monotonic() - stopped = snapshot(project, "stopped", target.instance_id) - check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped") - check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API") - time.sleep(args.duration) - finally: - stop.set() - try: - for job in jobs: - job.result(timeout=5) - finally: - project.docker("start", target.id) - project.wait_healthy({item.id for item in replicas}) - project.wait_routed(target.instance_id, client.base, args.ca_file) - snapshot(project, "restored", target.instance_id) - - elapsed = max(end for _, end, _ in successes) - started if successes else 0 - recovered = [end for begin, end, _ in successes if begin >= stop_finished] - check(bool(recovered), "no confirmed writes after failure") - recovery = min(recovered) - stop_started - check(recovery <= 10, "writes did not recover within 10 seconds") - confirmed.update(id for _, _, id in successes) - items = history(client, actor, chat) - check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost") - texts = [item["text"] for item in items] - check(len(texts) == len(set(texts)), "uncertain retry created duplicates") - print(json.dumps({ - "instance": target.instance_id, - "successful_writes": len(successes), - "failed_attempts": len(errors), - "errors_by_kind": dict(Counter(errors)), - "elapsed_seconds": round(elapsed, 3), - "successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0, - "first_recovery_seconds": round(recovery, 3), - }), flush=True) - - -def main() -> None: - """Готовит параметры эксперимента отказа реплик.""" - def configure(parser): - parser.add_argument("--duration", type=float, default=15) - parser.add_argument("--concurrency", type=int, default=2) - - args = arguments("Fail each API replica under write load", configure) - check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload") - project = ComposeProject() - with deployment_lock(project): - exercise(project, args) - - -if __name__ == "__main__": - run(main) diff --git a/scripts/failure_demo.sh b/scripts/failure_demo.sh deleted file mode 100755 index b5c7332d..00000000 --- a/scripts/failure_demo.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -set -eu -cd "$(dirname "$0")/.." - -BASE_URL="${BASE_URL:-http://localhost:8080}" -API="$BASE_URL/api/v1" -USERNAME="demo_$(date +%s)" -PASSWORD="demo-password-123" - -compose() { - docker compose --env-file .env "$@" -} - -code() { - curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@" -} - -wait_healthy() { - attempts=0 - until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do - attempts=$((attempts + 1)) - [ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; } - sleep 1 - done -} - -curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \ - -d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}" -TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \ - | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])') -AUTH="Authorization: Bearer $TOKEN" - -echo "== baseline" -echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" - -echo -echo "== redis stopped" -compose stop redis >/dev/null 2>&1 -echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" -echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")" -echo "GET /users/me (no token) -> $(code "$API/users/me")" -echo "GET /health/ready -> $(code "$BASE_URL/health/ready")" -echo "GET /health/live -> $(code "$BASE_URL/health/live")" -compose start redis >/dev/null 2>&1 -wait_healthy redis -echo "-- redis started again (AOF restored)" -echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" - -echo -echo "== postgres stopped" -compose stop postgres >/dev/null 2>&1 -echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")" -echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")" -echo "GET /health/ready -> $(code "$BASE_URL/health/ready")" -echo "GET /health/live -> $(code "$BASE_URL/health/live")" -compose start postgres >/dev/null 2>&1 -wait_healthy postgres -echo "-- postgres started again" -echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" - -echo -echo "== logout" -echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")" -echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")" diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh index 967e9512..ba8d9d2b 100755 --- a/scripts/restart.example.sh +++ b/scripts/restart.example.sh @@ -3,7 +3,7 @@ set -eu # TODO: copy to restart.sh and implement using YOUR compose/stack services. # Restart API and ALL used state stores (DB, sessions, S3, broker as applicable). # Preserve persistent volumes and schema/data; never down -v / flush / re-seed. -# Allow operator unseal if required in lab7; do not print credentials. +# Do not print credentials. # Exit 0 only after the intended action; test waits for API readiness separately. printf '%s ' 'TODO: implement restart.sh for your infrastructure' >&2 diff --git a/scripts/rolling_update.py b/scripts/rolling_update.py deleted file mode 100644 index 20aebda8..00000000 --- a/scripts/rolling_update.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Поочерёдно заменяет API с одной дополнительной репликой.""" - -import argparse -import sys - -from scripts.deployment import ComposeProject, deployment_lock - - -def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None: - """Собирает образ, применяет миграции и заменяет реплики.""" - original = project.replicas() - if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original): - raise RuntimeError("До обновления нужны минимум две здоровые API-реплики") - old_images = {item.image for item in original} - if len(old_images) != 1: - raise RuntimeError("Сначала восстановите единый образ работающих реплик") - old_image = old_images.pop() - profile = project.docker( - "image", "inspect", old_image, - "--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}', - ) - if profile != "file-v1": - raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up") - count = len(original) - print("Сборка нового образа API", flush=True) - build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else () - project.run("build", *build_args, "api", timeout=300) - tag = project.config["services"]["api"]["image"] - new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}") - print("Однократное применение совместимых миграций", flush=True) - project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate") - if old_image == new_image: - print("Образ не изменился; замена контейнеров не требуется") - return - - try: - for old in original: - project.run( - "up", "-d", "--no-deps", "--no-build", "--pull", "never", - "--no-recreate", "--scale", f"api={count + 1}", "api", - ) - current = project.replicas() - additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image] - if not additions: - raise RuntimeError("Новая реплика не создана") - project.wait_healthy({item.id for item in additions}) - for added in additions: - project.wait_routed(added.instance_id, base_url, ca_file) - survivors = {item.id for item in project.replicas() if item.id != old.id} - project.wait_healthy(survivors) - project.docker("stop", old.id) - project.docker("rm", old.id) - print(f"Заменена реплика {old.instance_id}", flush=True) - final = project.replicas() - if len(final) != count or any(item.image != new_image for item in final): - raise RuntimeError("Состав реплик после обновления не соответствует плану") - project.wait_healthy({item.id for item in final}) - except Exception: - print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr) - project.run( - "up", "-d", "--no-deps", "--no-build", "--pull", "never", - "--force-recreate", "--scale", f"api={count}", "api", - env={"API_IMAGE": old_image}, - ) - project.wait_healthy({item.id for item in project.replicas()}) - raise - - dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines() - if old_image in dangling: - references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}") - if not references: - try: - project.docker("image", "rm", old_image) - except RuntimeError: - print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr) - print(f"Готово: {count} здоровых реплик нового образа") - - -def main() -> None: - """Запускает обновление с блокировкой проекта.""" - parser = argparse.ArgumentParser(description="Rolling update of API replicas") - parser.add_argument("--base-url", default="http://localhost:8080") - parser.add_argument("--ca-file", default=None) - parser.add_argument("--revision", default=None) - args = parser.parse_args() - project = ComposeProject() - with deployment_lock(project): - update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision) - - -if __name__ == "__main__": - main() diff --git a/scripts/s3_faults.py b/scripts/s3_faults.py deleted file mode 100644 index 07a4bc25..00000000 --- a/scripts/s3_faults.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы.""" - -import asyncio -import base64 -from datetime import datetime, timedelta, timezone -import json -import sys -from uuid import UUID - -from minio.error import S3Error -from sqlalchemy import select -from sqlalchemy.exc import OperationalError - -from app.core.config import Settings -from app.core.database import create_db_engine, create_session_factory -from app.domain import attachments as domain -from app.errors.exceptions import unavailable -from app.models.attachment import UploadIntent -from app.schemas.user import User -from app.storage import attachments as repository -from app.storage.s3 import create_s3_storage - - -async def object_exists(storage, key): - exists = True - try: - await storage.stat_object(key) - except S3Error as error: - if error.code not in {'NoSuchKey', 'NoSuchObject'}: - raise - exists = False - return exists - - -async def main(): - payload = json.load(sys.stdin) - settings = Settings() - engine = create_db_engine(settings) - factory = create_session_factory(engine) - storage = create_s3_storage(settings.s3) - user = User.model_validate(payload['user']) - chat_id = UUID(payload['chat_id']) - original = base64.b64decode(payload['image']) - keys = [] - create_record = repository.create_attachment - put_file, delete_object = storage.put_file, storage.delete_object - - async def capture_put(key, path, **metadata): - keys.append(key) - await put_file(key, path, **metadata) - - async def fail_metadata(*args, **kwargs): - raise OperationalError('injected metadata failure after real S3 PUT', {}, None) - - async def fail_delete(*args): - raise unavailable('injected compensation failure') - - async def chunks(): - yield original - - async def attempt(): - raised = False - async with factory() as session: - try: - await domain.upload_attachment(session, storage, user, chat_id, - filename='fault.png', content_type='image/png', chunks=chunks()) - except OperationalError: - raised = True - assert raised, 'injected database failure was not raised' - - storage.put_file = capture_put - repository.create_attachment = fail_metadata - try: - await attempt() - assert not await object_exists(storage, keys[-1]), 'compensation left original' - async with factory() as session: - intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1])) - assert intent is None, 'compensated upload left intent' - storage.delete_object = fail_delete - await attempt() - key = keys[-1] - assert await object_exists(storage, key), 'fault did not leave real S3 object' - storage.delete_object = delete_object - async with factory() as session: - result = await domain.reconcile_attachments(session, storage, dry_run=False) - assert result.deleted == 0, 'fresh object deleted before grace period' - intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key)) - assert intent is not None, 'durable intent missing' - intent.created_at = datetime.now(timezone.utc) - timedelta(days=2) - await session.commit() - dry_run = await domain.reconcile_attachments(session, storage) - assert dry_run.candidates == 1 and dry_run.deleted == 0 - assert await object_exists(storage, key), 'dry-run removed original' - applied = await domain.reconcile_attachments(session, storage, dry_run=False) - assert applied.deleted == 1 and not await object_exists(storage, key) - repeated = await domain.reconcile_attachments(session, storage, dry_run=False) - assert repeated.examined == repeated.deleted == 0 - print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True, - 'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True})) - finally: - repository.create_attachment = create_record - storage.put_file, storage.delete_object = put_file, delete_object - await storage.aclose() - await engine.dispose() - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/scripts/upload_load.py b/scripts/upload_load.py deleted file mode 100644 index 91445202..00000000 --- a/scripts/upload_load.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Повторяемые параллельные upload и память cgroup API, без записи токенов.""" - -from collections import Counter -from concurrent.futures import ThreadPoolExecutor -import hashlib -from io import BytesIO -import json -from pathlib import Path -import random -import subprocess -from threading import Barrier -import time - -from PIL import Image - -from scripts.deployment import ComposeProject -from tests.client import Client, Failure, arguments, check, run - -MONITOR = r''' -import json, pathlib, sys, threading -root = pathlib.Path('/sys/fs/cgroup') -def current(): - return int((root / 'memory.current').read_text()) -def peak(): - path = root / 'memory.peak' - return int(path.read_text()) if path.exists() else None -stop = threading.Event() -threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start() -initial, before = current(), peak() -sampled = initial -print('ready', flush=True) -while not stop.wait(0.02): - sampled = max(sampled, current()) -print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled, - 'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True) -''' - - -def main(): - def configure(parser): - parser.add_argument('--concurrency', type=int, default=10) - parser.add_argument('--rounds', type=int, default=3) - parser.add_argument('--output', type=Path) - args = arguments('Concurrent attachment uploads and API memory', configure) - check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10') - project = ComposeProject() - client = Client(args.base_url, args.ca_file, timeout=65) - alice, _, _, chat = client.bootstrap() - random_bytes = random.Random(42).randbytes(1500 * 1500 * 3) - image = Image.frombytes('RGB', (1500, 1500), random_bytes) - buffer = BytesIO() - image.save(buffer, format='PNG') - data = buffer.getvalue() - check(len(data) <= 10485760, 'fixture exceeds upload limit') - monitors = [] - measurements = {} - rows = [] - started = time.monotonic() - try: - for replica in project.replicas(): - process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - monitors.append((replica.instance_id, process)) - check(process.stdout.readline().strip() == 'ready', 'memory monitor failed') - check(bool(monitors), 'API not running') - for _ in range(args.rounds): - barrier = Barrier(args.concurrency) - def upload(index): - local = Client(args.base_url, args.ca_file, timeout=65) - barrier.wait(timeout=15) - request_started = time.monotonic() - try: - return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png", - expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data, - headers={'Content-Type': 'image/png'}, schema='Attachment') - except Failure: - # При раннем отказе сервер может закрыть ещё отправляемое тело. - return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started} - with ThreadPoolExecutor(max_workers=args.concurrency) as pool: - rows.extend(pool.map(upload, range(args.concurrency))) - finally: - elapsed = time.monotonic() - started - for alias, process in monitors: - output, _ = process.communicate(input='', timeout=10) - check(process.returncode == 0, 'memory monitor failed') - measurements[alias] = json.loads(output) - succeeded = [row for row in rows if row['status'] == 201] - check(bool(succeeded), 'all uploads were rejected') - digest = hashlib.sha256(data).hexdigest() - for row in succeeded: - check(row['json']['sha256'] == digest, 'upload checksum differs') - original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes'] - check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs') - latencies = sorted(row['elapsed'] for row in rows) - result = { - 'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows), - 'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest}, - 'statuses': dict(Counter(str(row['status']) for row in rows)), - 'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded), - 'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3), - 'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'], - 'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries', - } - encoded = json.dumps(result, ensure_ascii=False, indent=2) - if args.output: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(encoded + '\n') - print(encoded) - check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts') - - -if __name__ == '__main__': - run(main) diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 1dfe4a98..7e1d1afc 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -11,9 +11,9 @@ validate(spec) from referencing import Registry, Resource # The base URI resolves local #/components references inside examples. resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) -registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource)) +registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource)) for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): - schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name} + schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name} Draft202012Validator(schema, registry=registry).validate(example) for path in (root/'contracts').glob('*.schema.json'): schema = json.loads(path.read_text()) diff --git a/student-tests/conftest.py b/student-tests/conftest.py index ca5063f1..a72d1905 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -132,7 +132,7 @@ async def clean_db(db_engine: AsyncEngine, redis_client: Redis) -> None: def build_settings(database_url: str, **overrides: object) -> Settings: values: dict[str, object] = { - # Регрессия lab3 проверяет PostgreSQL/Redis; MinIO подменяется отдельно. + # Регрессия проверяет PostgreSQL/Redis; MinIO подменяется отдельно. "s3_enabled": False, "app_host": "127.0.0.1", "app_port": 8080, diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 0deb88f5..20237f1f 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -1,4 +1,4 @@ -"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" +"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" import asyncio import hashlib diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py index aeab6499..4a08bc26 100644 --- a/student-tests/test_jobs.py +++ b/student-tests/test_jobs.py @@ -1,4 +1,4 @@ -"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL.""" +"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL.""" import asyncio from datetime import timedelta -- 2.54.0 From 9899ee46769fcdb212ee28d74d657e619a129a72 Mon Sep 17 00:00:00 2001 From: gytall Date: Tue, 10 Nov 2026 09:52:42 +0300 Subject: [PATCH 4/6] formatted project by python-black --- app/core/background.py | 9 +- app/core/cleanup.py | 4 +- app/core/image_processing.py | 4 +- app/core/job_admin.py | 27 +- app/core/job_worker.py | 51 +- app/core/jobs.py | 4 +- app/core/outbox_publisher.py | 30 +- app/core/redis.py | 3 +- app/core/s3.py | 20 +- app/core/thumbnail_worker.py | 8 +- app/domain/attachment_jobs.py | 123 +++-- app/domain/attachments.py | 14 +- app/domain/health.py | 11 +- app/domain/idempotency.py | 12 +- app/domain/image_pipeline.py | 48 +- app/domain/job_admin.py | 90 +++- app/domain/job_files.py | 21 +- app/domain/job_retry.py | 27 +- app/domain/messages.py | 13 +- app/endpoints/attachments.py | 56 ++- app/endpoints/health.py | 11 +- app/endpoints/messages.py | 8 +- app/errors/exceptions.py | 15 +- app/errors/handlers.py | 21 +- app/main.py | 4 +- app/models/__init__.py | 15 +- app/models/attachment.py | 13 +- app/models/job.py | 7 +- app/schemas/attachment.py | 4 +- app/schemas/job.py | 4 +- app/storage/broker.py | 30 +- app/storage/health.py | 2 +- app/storage/idempotency.py | 17 +- app/storage/job_admin.py | 19 +- app/storage/jobs.py | 53 +- app/storage/outbox.py | 4 +- infra/minio/bootstrap.py | 5 +- .../49d2c947a3f1_add_idempotency_keys.py | 13 +- .../versions/8f31b94a2d10_add_attachments.py | 59 ++- .../versions/abd273076068_add_counters.py | 17 +- .../c35e91a2f706_async_attachment_jobs.py | 83 +++- .../e7048b6da913_attachment_crop_webp.py | 10 +- .../versions/f9183d27a640_job_replay.py | 27 +- scripts/check_contract.py | 88 ++-- scripts/check_jobs.py | 462 +++++++++++++----- scripts/check_outbox_recovery.py | 114 ++++- scripts/check_s3.py | 91 ++-- scripts/deployment.py | 21 +- scripts/init_local.py | 10 +- scripts/validate_openapi.py | 20 +- student-tests/test_attachments.py | 234 ++++++--- student-tests/test_concurrency.py | 9 +- student-tests/test_history_plan.py | 6 +- student-tests/test_idempotency.py | 72 ++- student-tests/test_image_pipeline.py | 122 +++-- student-tests/test_instance.py | 8 +- student-tests/test_jobs.py | 185 +++++-- student-tests/test_lifecycle.py | 12 +- student-tests/test_limits.py | 38 +- student-tests/test_migrations.py | 20 +- student-tests/test_s3_stream.py | 11 +- tests/client.py | 372 +++++++++----- tests/image_probe.py | 31 +- tests/load.py | 85 +++- tests/resilience.py | 206 +++++--- tests/smoke.py | 228 ++++++--- 66 files changed, 2517 insertions(+), 944 deletions(-) diff --git a/app/core/background.py b/app/core/background.py index 06a4d9a4..a0cfb7bb 100644 --- a/app/core/background.py +++ b/app/core/background.py @@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError from sqlalchemy.exc import InterfaceError, OperationalError from sqlalchemy.exc import TimeoutError as PoolTimeoutError -TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError) +TRANSIENT_JOB_ERRORS = ( + AMQPException, + ChannelInvalidStateError, + InterfaceError, + OperationalError, + PoolTimeoutError, + OSError, +) logger = logging.getLogger(__name__) diff --git a/app/core/cleanup.py b/app/core/cleanup.py index d5201397..722e1a0d 100644 --- a/app/core/cleanup.py +++ b/app/core/cleanup.py @@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool async with factory() as session: result = await reconcile_attachments(session, storage, dry_run=not apply) staging = await cleanup_staging(session, storage, jobs, apply=apply) - print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True) + print( + json.dumps({**asdict(result), "terminal_staging": staging}), flush=True + ) except RECOVERABLE_STORAGE_ERRORS as error: log_recoverable_error("Reconciliation", error) completed = False diff --git a/app/core/image_processing.py b/app/core/image_processing.py index 9eb46def..107dc42f 100644 --- a/app/core/image_processing.py +++ b/app/core/image_processing.py @@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class ImageSettings(BaseSettings): - model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore") + model_config = SettingsConfigDict( + env_prefix="IMAGE_", env_file=".env", extra="ignore" + ) jpeg_quality: int = Field(default=85, ge=1, le=95) webp_quality: int = Field(default=80, ge=1, le=100) diff --git a/app/core/job_admin.py b/app/core/job_admin.py index 369b9f21..f75efa67 100644 --- a/app/core/job_admin.py +++ b/app/core/job_admin.py @@ -9,7 +9,13 @@ from app.core.background import log_background_error from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings -from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job +from app.domain.job_admin import ( + REPLAY_REASONS, + history, + list_failed, + replay_failed_job, + show_job, +) from app.errors.exceptions import JobOperationError from app.storage.s3 import create_s3_storage @@ -29,8 +35,15 @@ async def execute(args): result = await history(session, args.job_id) else: storage = create_s3_storage(settings.s3) - result = await replay_failed_job(session, storage, JobSettings(), args.job_id, - request_id=args.request_id, actor=args.actor, reason=args.reason) + result = await replay_failed_job( + session, + storage, + JobSettings(), + args.job_id, + request_id=args.request_id, + actor=args.actor, + reason=args.reason, + ) print(json.dumps(result, default=str), flush=True) finally: if storage is not None: @@ -39,10 +52,14 @@ async def execute(args): def main(): - parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом") + parser = argparse.ArgumentParser( + description="Реестр failed и явный replay с аудитом" + ) commands = parser.add_subparsers(dest="command", required=True) listing = commands.add_parser("list-failed") - listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500") + listing.add_argument( + "--limit", type=int, choices=range(1, 501), default=50, metavar="1..500" + ) for name in ("show", "history", "replay"): command = commands.add_parser(name) command.add_argument("job_id", type=UUID) diff --git a/app/core/job_worker.py b/app/core/job_worker.py index 4c36d0ff..168d186b 100644 --- a/app/core/job_worker.py +++ b/app/core/job_worker.py @@ -12,7 +12,12 @@ from uuid import UUID, uuid4 from PIL import features from pydantic import ValidationError -from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings @@ -34,7 +39,10 @@ class Faults: object_name: str = "optimized" def kill(self, attachment_id: UUID, phase: str) -> None: - print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True) + print( + json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), + flush=True, + ) os.kill(os.getpid(), signal.SIGKILL) def after_object(self, attachment_id: UUID, name: str) -> None: @@ -52,7 +60,14 @@ class Faults: raise ProcessingError("injected_permanent") -async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None: +async def consume_connected( + factory, + storage, + broker: JobBroker, + settings: JobSettings, + stop: asyncio.Event, + faults: Faults, +) -> None: consumer = uuid4().hex while not stop.is_set(): try: @@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe try: async with factory() as session: acknowledge = await process_attachment_job( - session, storage, message, settings, - after_object=faults.after_object, before_processing=faults.before_processing, + session, + storage, + message, + settings, + after_object=faults.after_object, + before_processing=faults.before_processing, ) except ProcessingError: await broker.reject_invalid(delivery.id) @@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None: storage = create_s3_storage(settings.s3) factory = create_session_factory(engine) async with asyncio.TaskGroup() as group: - tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)] + tasks = [ + group.create_task(consume(factory, storage, jobs, stop, faults)) + for _ in range(jobs.worker_concurrency) + ] await stop.wait() # Отмена убивает resize-процесс, дожидается SDK и не отправляет ack. for task in tasks: @@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks") + parser = argparse.ArgumentParser( + description="Image worker с явными fault-injection hooks" + ) modes = parser.add_mutually_exclusive_group() - for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"): + for flag in ( + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ): modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID") - parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized") + parser.add_argument( + "--object-name", + choices=("original", "thumbnail", "crop", "optimized"), + default="optimized", + ) args = parser.parse_args() faults = Faults(**vars(args)) try: diff --git a/app/core/jobs.py b/app/core/jobs.py index e7badd3e..a67c86da 100644 --- a/app/core/jobs.py +++ b/app/core/jobs.py @@ -29,7 +29,9 @@ class JobSettings(BaseSettings): @model_validator(mode="after") def validate_lease(self) -> Self: if self.job_lease_seconds < self.job_timeout_seconds + 30: - raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд") + raise ValueError( + "Lease должен превышать timeout задания минимум на 30 секунд" + ) if self.job_retry_max_seconds < self.job_retry_seconds: raise ValueError("Retry maximum must be at least the base delay") return self diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py index 8d23e461..54ef3a2c 100644 --- a/app/core/outbox_publisher.py +++ b/app/core/outbox_publisher.py @@ -8,7 +8,12 @@ import os import signal from uuid import UUID -from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.background import ( + TRANSIENT_JOB_ERRORS, + log_background_error, + shutdown_event, + wait_or_stop, +) from app.core.config import Settings from app.core.database import create_db_engine, create_session_factory from app.core.jobs import JobSettings @@ -18,7 +23,10 @@ from app.storage.broker import JobBroker def crash_before_publish(attachment_id: UUID) -> None: """Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish.""" - print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True) + print( + json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), + flush=True, + ) os.kill(os.getpid(), signal.SIGKILL) @@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None: async with factory() as session: await recover_expired_jobs(session, jobs) published = await publish_next_job( - session, broker, attachment_id=crash_attachment_id, - before_publish=crash_before_publish if crash_attachment_id is not None else None, + session, + broker, + attachment_id=crash_attachment_id, + before_publish=( + crash_before_publish + if crash_attachment_id is not None + else None + ), ) except TRANSIENT_JOB_ERRORS as error: log_background_error("Publisher deferred", error) @@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None: def main() -> None: - parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ") + parser = argparse.ArgumentParser( + description="Публикация устойчивых jobs в RabbitMQ" + ) parser.add_argument( - "--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID", + "--crash-before-publish", + type=UUID, + metavar="ATTACHMENT_ID", help="аварийно завершить этот процесс перед публикацией только указанного вложения", ) args = parser.parse_args() diff --git a/app/core/redis.py b/app/core/redis.py index ea525f8b..fb61304d 100644 --- a/app/core/redis.py +++ b/app/core/redis.py @@ -4,6 +4,7 @@ from redis.backoff import NoBackoff from app.core.config import Settings + def create_redis_client(settings: Settings) -> Redis: password = ( settings.redis_password.get_secret_value() if settings.redis_password else None @@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis: decode_responses=True, ) return Redis.from_pool(pool) - - diff --git a/app/core/s3.py b/app/core/s3.py index c90137f8..f2de1880 100644 --- a/app/core/s3.py +++ b/app/core/s3.py @@ -6,6 +6,7 @@ from urllib.parse import urlsplit from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict + class S3Settings(BaseSettings): endpoint: str = "http://minio:9000" region: str = "us-east-1" @@ -24,12 +25,23 @@ class S3Settings(BaseSettings): cleanup_grace_seconds: int = Field(default=86_400, ge=300) cleanup_interval_seconds: int = Field(default=300, ge=1) - model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True) + model_config = SettingsConfigDict( + env_prefix="S3_", env_file=".env", extra="ignore", frozen=True + ) @model_validator(mode="after") def validate_endpoint(self): url = urlsplit(self.endpoint) - if (url.scheme not in {"http", "https"} or not url.hostname or url.username - or url.password or url.path not in {"", "/"} or url.query or url.fragment): - raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути") + if ( + url.scheme not in {"http", "https"} + or not url.hostname + or url.username + or url.password + or url.path not in {"", "/"} + or url.query + or url.fragment + ): + raise ValueError( + "S3_ENDPOINT должен быть http(s) адресом без credentials/пути" + ) return self diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py index cc3e2be1..2c465107 100644 --- a/app/core/thumbnail_worker.py +++ b/app/core/thumbnail_worker.py @@ -13,11 +13,15 @@ def main() -> int: settings = ImageSettings() memory = settings.memory_mib * 1024 * 1024 resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) - resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)) + resource.setrlimit( + resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds) + ) source, directory, metadata, content_type, maximum = sys.argv[1:] code = 0 try: - result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings) + result = create_variants( + Path(source), Path(directory), content_type, int(maximum), settings + ) Path(metadata).write_text(result.model_dump_json()) except IMAGE_WORKER_ERRORS as error: code = pipeline_worker_error_code(error) diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py index 5f6d7dcf..ccf99346 100644 --- a/app/domain/attachment_jobs.py +++ b/app/domain/attachment_jobs.py @@ -29,13 +29,17 @@ class Claim: acknowledge: bool = False -async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: +async def enqueue_attachment_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: """Включить намерение обработки в транзакцию queued metadata; без commit.""" return await jobs.create_job(session, attachment_id, staging_key) async def publish_next_job( - session: AsyncSession, broker: JobBroker, *, + session: AsyncSession, + broker: JobBroker, + *, attachment_id: UUID | None = None, before_publish: Callable[[UUID], None] | None = None, ) -> bool: @@ -45,24 +49,33 @@ async def publish_next_job( if job is not None: if before_publish is not None: before_publish(job.attachment_id) - await broker.publish_job(JobMessage( - job_id=job.id, attachment_id=job.attachment_id, - pipeline_version=job.pipeline_version, attempt=job.attempt + 1, - created_at=job.created_at, - )) + await broker.publish_job( + JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) + ) outbox.mark_published(job) await session.commit() return published -async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim: +async def claim_job( + session: AsyncSession, message: JobMessage, settings: JobSettings +) -> Claim: """Дубликат не захватывает живую аренду и не откатывает terminal-состояние.""" row = await jobs.lock_job(session, message.job_id) result = Claim() if row is None: raise ProcessingError("unknown_job") job, attachment = row - if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version: + if ( + message.attachment_id != attachment.id + or message.pipeline_version != job.pipeline_version + ): raise ProcessingError("invalid_job") now = await jobs.database_time(session) if job.attempt_limit is None: @@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet elif message.attempt > job.attempt + 1: raise ProcessingError("invalid_attempt") elif attachment.status == "queued" and job.next_attempt_at <= now: - if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None: - jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0) + if ( + job.attempt >= (job.attempt_limit or settings.job_max_attempts) + or job.staging_key is None + ): + jobs.record_failure( + job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0 + ) result = Claim(acknowledge=True) else: - result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds)) + result = Claim( + work=jobs.start_job( + job, attachment, uuid4(), now, settings.job_lease_seconds + ) + ) await session.commit() return result -async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None: +async def save_failure( + session: AsyncSession, + work: JobWork, + settings: JobSettings, + code: str, + *, + permanent: bool, +) -> None: # Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется. await session.rollback() row = await jobs.lock_job(session, work.job_id) @@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin if attachment.status == "processing" and job.lease_token == work.lease_token: exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) jobs.record_failure( - job, attachment, await jobs.database_time(session), + job, + attachment, + await jobs.database_time(session), "retry_exhausted" if exhausted and not permanent else code, terminal=permanent or exhausted, - retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + retry_seconds=retry_delay( + settings, job.attempt - job.cycle_start_attempt + ), ) job.last_error_code = code job.last_error_permanent = permanent @@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin async def build_result( - session: AsyncSession, storage: MinioStorage, work: JobWork, + session: AsyncSession, + storage: MinioStorage, + work: JobWork, after_object: Callable[[UUID, str], None] | None = None, ) -> bool: """Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки.""" @@ -129,26 +164,42 @@ async def build_result( # Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED. # При отмене S3 adapter дожидается SDK-потока до освобождения блокировки. await storage.put_file( - work.original_key, source, size_bytes=work.size_bytes, - content_type=work.content_type, sha256=work.sha256, + work.original_key, + source, + size_bytes=work.size_bytes, + content_type=work.content_type, + sha256=work.sha256, ) if after_object is not None: after_object(work.attachment_id, "original") for variant in result.variants: await storage.put_file( - work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name], - size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256, + work.variant_key(variant.name), + Path(directory) / VARIANT_FILENAMES[variant.name], + size_bytes=variant.size_bytes, + content_type=variant.content_type, + sha256=variant.sha256, ) if after_object is not None: after_object(work.attachment_id, variant.name) - jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session)) + jobs.complete_job( + job, + attachment, + work, + [variant.model_dump() for variant in result.variants], + await jobs.database_time(session), + ) await session.commit() return True async def process_attachment_job( - session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings, - *, after_object: Callable[[UUID, str], None] | None = None, + session: AsyncSession, + storage: MinioStorage, + message: JobMessage, + settings: JobSettings, + *, + after_object: Callable[[UUID, str], None] | None = None, before_processing: Callable[[UUID], None] | None = None, ) -> bool: claim = await claim_job(session, message, settings) @@ -160,7 +211,13 @@ async def process_attachment_job( if before_processing is not None: before_processing(work.attachment_id) acknowledge = await build_result(session, storage, work, after_object) - except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error: + except ( + ProcessingError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, + ) as error: code, permanent = classify_failure(error) await save_failure(session, work, settings, code, permanent=permanent) acknowledge = True @@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> job.attempt_limit = settings.job_max_attempts exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) jobs.record_failure( - job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout", - terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + job, + attachment, + now, + "retry_exhausted" if exhausted else "worker_timeout", + terminal=exhausted, + retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), ) job.last_error_code = "worker_timeout" job.last_error_permanent = False @@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> return len(rows) -async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int: - rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds) +async def cleanup_staging( + session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool +) -> int: + rows = await jobs.terminal_staging( + session, + settings.staging_retention_seconds, + settings.failed_staging_retention_seconds, + ) if apply: for job, attachment in rows: if job.staging_key != attachment.object_key: diff --git a/app/domain/attachments.py b/app/domain/attachments.py index 80caf109..2a33d010 100644 --- a/app/domain/attachments.py +++ b/app/domain/attachments.py @@ -276,12 +276,20 @@ async def download_attachment( key = record.object_key info = DownloadInfo(result.filename, result.content_type, result.size_bytes) else: - selected = next((item for item in result.variants if item.name == variant), None) - keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key} + selected = next( + (item for item in result.variants if item.name == variant), None + ) + keys = { + "thumbnail": record.thumbnail_key, + "crop": record.crop_key, + "optimized": record.optimized_key, + } key = keys.get(variant) if selected is None or key is None: raise not_found() - info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes) + info = DownloadInfo( + VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes + ) await session.rollback() stored = await storage.stat_object(key) if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type: diff --git a/app/domain/health.py b/app/domain/health.py index aabd4d03..c5d6ac31 100644 --- a/app/domain/health.py +++ b/app/domain/health.py @@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage def liveness() -> Health: - return Health(status='ok') + return Health(status="ok") async def readiness( - engine: AsyncEngine, redis: Redis, storage: MinioStorage | None, - lifecycle: Lifecycle, timeout_seconds: float, + engine: AsyncEngine, + redis: Redis, + storage: MinioStorage | None, + lifecycle: Lifecycle, + timeout_seconds: float, ) -> Health: if lifecycle.draining: raise unavailable() @@ -30,4 +33,4 @@ async def readiness( dependencies = await asyncio.gather(*checks, return_exceptions=True) if not all(result is True for result in dependencies): raise unavailable() - return Health(status='ok') + return Health(status="ok") diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py index 83b13acc..5ad93908 100644 --- a/app/domain/idempotency.py +++ b/app/domain/idempotency.py @@ -57,16 +57,22 @@ async def send_message_idempotent( raise conflict("Idempotency-Key уже использован для другого сообщения") message = await message_storage.get_message(session, previous.message_id) else: - await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids) + await validate_message_attachments( + session, sender, chat_id, new_message.attachment_ids + ) if not await message_storage.reserve_message_slot(session, max_messages): raise unavailable("Достигнут лимит количества сообщений") message = await message_storage.create_message( session, chat_id=chat_id, sender_id=sender.id, text=new_message.text ) - await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids) + await attachment_storage.bind_attachments( + session, message.id, new_message.attachment_ids + ) await key_storage.save_result(session, scope, fingerprint, message.id) links = await attachment_storage.message_attachment_ids(session, [message.id]) - result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]}) + result = Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]} + ) await session.commit() return result diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py index 823ade9d..c7f63535 100644 --- a/app/domain/image_pipeline.py +++ b/app/domain/image_pipeline.py @@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]: longest = max(size) result = size if longest > maximum: - result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest)) + result = ( + max(1, size[0] * maximum // longest), + max(1, size[1] * maximum // longest), + ) return result -def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant: +def describe_variant( + name: str, content_type: str, path: Path, size: tuple[int, int] +) -> Variant: checksum = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(65536), b""): checksum.update(chunk) return Variant( - name=name, content_type=content_type, size_bytes=path.stat().st_size, - width=size[0], height=size[1], sha256=checksum.hexdigest(), + name=name, + content_type=content_type, + size_bytes=path.stat().st_size, + width=size[0], + height=size[1], + sha256=checksum.hexdigest(), ) @@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant: """Центрированный JPEG 128×128; маленький исходник допускает увеличение.""" - with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop: + with ImageOps.fit( + image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) as crop: save_jpeg(crop, destination, quality) return describe_variant("crop", "image/jpeg", destination, (128, 128)) -def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant: +def create_optimized( + image: Image.Image, destination: Path, settings: ImageSettings +) -> Variant: """WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются.""" size = fitted_size(image.size, 1280) with image.resize(size, Image.Resampling.LANCZOS) as resized: with Image.new("RGBA", size) as clean: clean.paste(resized) clean.save( - destination, format="WEBP", lossless=False, - quality=settings.webp_quality, method=settings.webp_method, - exif=b"", icc_profile=b"", xmp=b"", + destination, + format="WEBP", + lossless=False, + quality=settings.webp_quality, + method=settings.webp_method, + exif=b"", + icc_profile=b"", + xmp=b"", ) return describe_variant("optimized", "image/webp", destination, size) -def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult: +def create_variants( + source: Path, + directory: Path, + content_type: str, + max_pixels: int, + settings: ImageSettings, +) -> PipelineResult: decode_image(source, content_type, max_pixels) with Image.open(source, formats=["PNG", "JPEG"]) as image: ImageOps.exif_transpose(image, in_place=True) with image.convert("RGBA") as pixels: variants = [ - create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality), + create_thumbnail( + pixels, directory / "thumbnail.jpg", settings.jpeg_quality + ), create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality), create_optimized(pixels, directory / "optimized.webp", settings), ] diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py index a1a3494e..6a6be443 100644 --- a/app/domain/job_admin.py +++ b/app/domain/job_admin.py @@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review") def snapshot(job, attachment): - return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status, - attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt, - error_code=attachment.error_code, last_error_code=job.last_error_code, - last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None, - next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at) + return dict( + job_id=str(job.id), + attachment_id=str(attachment.id), + status=attachment.status, + attempt=job.attempt, + attempt_limit=job.attempt_limit, + cycle_start_attempt=job.cycle_start_attempt, + error_code=attachment.error_code, + last_error_code=job.last_error_code, + last_error_permanent=job.last_error_permanent, + staging_retained=job.staging_key is not None, + next_attempt_at=job.next_attempt_at, + lease_until=job.lease_until, + finished_at=job.finished_at, + ) async def list_failed(session, limit): - return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)] + return [ + snapshot(job, attachment) + for job, attachment in await job_admin.failed_jobs(session, limit) + ] async def show_job(session, job_id): @@ -38,20 +51,37 @@ async def show_job(session, job_id): async def history(session, job_id): - return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason, - previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit, - previous_error_code=row.previous_error_code, created_at=row.created_at) - for row in await job_admin.replay_history(session, job_id)] + return [ + dict( + request_id=str(row.id), + actor=row.actor, + reason=row.reason, + previous_attempt=row.previous_attempt, + attempt_limit=row.attempt_limit, + previous_error_code=row.previous_error_code, + created_at=row.created_at, + ) + for row in await job_admin.replay_history(session, job_id) + ] def same_request(record, job_id, actor, reason): if record.job_id != job_id or record.actor != actor or record.reason != reason: raise JobOperationError("request_conflict") - return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True} + return { + "request_id": str(record.id), + "job_id": str(job_id), + "already_applied": True, + } -async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason): - if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS: +async def replay_failed_job( + session, storage, settings, job_id, *, request_id, actor, reason +): + if ( + re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None + or reason not in REPLAY_REASONS + ): raise JobOperationError("invalid_audit_fields") row = await jobs.lock_job(session, job_id) if row is None: @@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a if job.staging_key is None: raise JobOperationError("source_expired") # Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT. - work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key, - attachment.content_type, attachment.size_bytes, attachment.sha256) + work = JobWork( + job.id, + attachment.id, + uuid4(), + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, + ) try: async with asyncio.timeout(settings.job_timeout_seconds): with TemporaryDirectory(prefix="replay-source-") as directory: @@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a raise limit = job.attempt + settings.job_max_attempts inserted = await job_admin.insert_replay( - session, id=request_id, job_id=job_id, actor=actor, reason=reason, - previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code, + session, + id=request_id, + job_id=job_id, + actor=actor, + reason=reason, + previous_attempt=job.attempt, + attempt_limit=limit, + previous_error_code=job.last_error_code, ) if not inserted: - result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason) + result = same_request( + await job_admin.replay_record(session, request_id), job_id, actor, reason + ) await session.rollback() return result job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit) await session.commit() - return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit} + return { + "request_id": str(request_id), + "job_id": str(job_id), + "already_applied": False, + "status": "queued", + "attempt_limit": limit, + } diff --git a/app/domain/job_files.py b/app/domain/job_files.py index b48950d2..b99bda6c 100644 --- a/app/domain/job_files.py +++ b/app/domain/job_files.py @@ -14,7 +14,9 @@ from app.storage.jobs import JobWork from app.storage.s3 import MinioStorage -async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None: +async def fetch_original( + storage: MinioStorage, work: JobWork, destination: Path +) -> None: if work.size_bytes > storage.settings.max_upload_bytes: raise ProcessingError("image_limits") checksum = hashlib.sha256() @@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path raise ProcessingError("source_mismatch") -async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult: +async def render_variants( + storage: MinioStorage, work: JobWork, source: Path, directory: Path +) -> PipelineResult: metadata = directory / "variants.json" process = await asyncio.create_subprocess_exec( - sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory), - str(metadata), work.content_type, str(storage.settings.max_image_pixels), - stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, + sys.executable, + "-m", + "app.core.thumbnail_worker", + str(source), + str(directory), + str(metadata), + work.content_type, + str(storage.settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, ) try: code = await process.wait() diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py index 2d9620df..f6a67906 100644 --- a/app/domain/job_retry.py +++ b/app/domain/job_retry.py @@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError def retry_delay(settings: JobSettings, cycle_attempt: int) -> float: - ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1))) + ceiling = min( + settings.job_retry_max_seconds, + settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)), + ) return random.uniform(ceiling / 2, ceiling) @@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]: if isinstance(error, S3Error): if error.code in {"NoSuchKey", "NoSuchObject"}: return "source_missing", True - if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}: + if error.code in { + "AccessDenied", + "InvalidAccessKeyId", + "SignatureDoesNotMatch", + "NoSuchBucket", + }: return "storage_configuration", True - if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}: + if error.code in { + "InternalError", + "SlowDown", + "ServiceUnavailable", + "RequestTimeout", + "Throttling", + }: return "storage_unavailable", False return "storage_rejected", True if isinstance(error, TimeoutError): return "processing_timeout", False - if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + if isinstance(error, OSError) and error.errno in { + errno.EACCES, + errno.EPERM, + errno.EROFS, + errno.ENOENT, + }: return "processing_configuration", True return "processing_unavailable", False diff --git a/app/domain/messages.py b/app/domain/messages.py index f35cbfb3..dae112b5 100644 --- a/app/domain/messages.py +++ b/app/domain/messages.py @@ -32,10 +32,15 @@ async def list_messages( rows = await messages_storage.list_messages(session, chat_id, after, limit + 1) page_rows, has_next = split_page(rows, limit) - links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows]) - items = [Message.model_validate(message).model_copy( - update={"attachment_ids": links[message.id]}, - ) for message in page_rows] + links = await attachments_storage.message_attachment_ids( + session, [message.id for message in page_rows] + ) + items = [ + Message.model_validate(message).model_copy( + update={"attachment_ids": links[message.id]}, + ) + for message in page_rows + ] page = MessagePage( items=items, next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next), diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py index 8387b3e2..ddeddfa9 100644 --- a/app/endpoints/attachments.py +++ b/app/endpoints/attachments.py @@ -12,41 +12,63 @@ from app.domain import attachments from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity from app.schemas.attachment import Attachment -router = APIRouter(prefix='/api/v1', tags=['attachments']) +router = APIRouter(prefix="/api/v1", tags=["attachments"]) -@router.post('/chats/{chat_id}/attachments', status_code=202) +@router.post("/chats/{chat_id}/attachments", status_code=202) async def upload( - request: Request, chat_id: UUID, user: CurrentUser, session: DbSession, - storage: S3Storage, capacity: UploadCapacity, + request: Request, + chat_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + capacity: UploadCapacity, filename: Annotated[str, Query(min_length=1, max_length=255)], ) -> Attachment: return await attachments.receive_attachment( - session, storage, user, chat_id, capacity=capacity, filename=filename, - content_type=request.headers.get('content-type', '').lower(), - declared_length=request.headers.get('content-length'), chunks=request.stream(), + session, + storage, + user, + chat_id, + capacity=capacity, + filename=filename, + content_type=request.headers.get("content-type", "").lower(), + declared_length=request.headers.get("content-length"), + chunks=request.stream(), ) -@router.get('/attachments/{attachment_id}') -async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment: +@router.get("/attachments/{attachment_id}") +async def metadata( + attachment_id: UUID, user: CurrentUser, session: DbSession +) -> Attachment: return await attachments.get_attachment(session, user, attachment_id) -@router.get('/attachments/{attachment_id}/content') +@router.get("/attachments/{attachment_id}/content") async def content( - attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage, - variant: Annotated[str, Query()] = 'original', + attachment_id: UUID, + user: CurrentUser, + session: DbSession, + storage: S3Storage, + variant: Annotated[str, Query()] = "original", ) -> StreamingResponse: record, stream = await attachments.download_attachment( - session, storage, user, attachment_id, variant=variant, + session, + storage, + user, + attachment_id, + variant=variant, ) return StreamingResponse( - stream, media_type=record.content_type, + stream, + media_type=record.content_type, headers={ - 'Content-Length': str(record.size_bytes), - 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''), - 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store', + "Content-Length": str(record.size_bytes), + "Content-Disposition": "attachment; filename*=UTF-8''" + + quote(record.filename, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", }, background=BackgroundTask(stream.aclose), ) diff --git a/app/endpoints/health.py b/app/endpoints/health.py index 9c28c7d4..2be984b6 100644 --- a/app/endpoints/health.py +++ b/app/endpoints/health.py @@ -3,18 +3,21 @@ from fastapi import APIRouter, Request from app.domain import health from app.schemas.base import Health -router = APIRouter(prefix='/health', tags=['health']) +router = APIRouter(prefix="/health", tags=["health"]) -@router.get('/live') +@router.get("/live") async def liveness() -> Health: return health.liveness() -@router.get('/ready') +@router.get("/ready") async def readiness(request: Request) -> Health: state = request.app.state return await health.readiness( - state.engine, state.redis, state.s3, state.lifecycle, + state.engine, + state.redis, + state.s3, + state.lifecycle, state.settings.readiness_timeout_seconds, ) diff --git a/app/endpoints/messages.py b/app/endpoints/messages.py index 50dc1a18..4bfbd3f3 100644 --- a/app/endpoints/messages.py +++ b/app/endpoints/messages.py @@ -21,9 +21,13 @@ async def send_new_message( idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, ) -> Message: return await send_message_idempotent( - session, sender, chat_id, new_message, + session, + sender, + chat_id, + new_message, validate_idempotency_key(idempotency_key), - settings.max_messages, settings.idempotency_ttl_seconds, + settings.max_messages, + settings.idempotency_ttl_seconds, ) diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py index 59d06a96..72cc0782 100644 --- a/app/errors/exceptions.py +++ b/app/errors/exceptions.py @@ -90,14 +90,23 @@ def internal_error() -> ApiError: IMAGE_WORKER_ERRORS = ( - ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning, - MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError, + ApiError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + MemoryError, + UnidentifiedImageError, + OSError, + ValueError, + SyntaxError, ) def image_worker_error_code(error: Exception) -> int: code = 2 - resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError)) + resource_error = isinstance( + error, + (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError), + ) declared_limit = isinstance(error, ApiError) and error.status_code == 413 if resource_error or declared_limit: code = 3 diff --git a/app/errors/handlers.py b/app/errors/handlers.py index 9dd48190..faa82677 100644 --- a/app/errors/handlers.py +++ b/app/errors/handlers.py @@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError from app.core.instance import INSTANCE_ID_HEADER from app.core.request_id import REQUEST_ID_HEADER -from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable +from app.errors.exceptions import ( + ApiError, + internal_error, + invalid_request, + not_found, + unavailable, +) logger = logging.getLogger(__name__) TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57") TRANSIENT_SQLSTATES = frozenset({"55P03"}) RECOVERABLE_STORAGE_ERRORS = ( - ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError, + ApiError, + SQLAlchemyError, + MinioException, + StorageHTTPError, + OSError, + TimeoutError, ) @@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError): async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse: - return await api_error_handler(request, invalid_request("Загрузка прервана клиентом")) + return await api_error_handler( + request, invalid_request("Загрузка прервана клиентом") + ) def log_recoverable_error(operation: str, error: Exception) -> None: """Согласованный безопасный лог восстановления без содержимого SDK-ошибки.""" - logger.warning('%s deferred: %s', operation, type(error).__name__) + logger.warning("%s deferred: %s", operation, type(error).__name__) def _get_request_id(request: Request) -> str: diff --git a/app/main.py b/app/main.py index de683883..f68247b8 100644 --- a/app/main.py +++ b/app/main.py @@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI: application.state.session_factory = create_session_factory(engine) application.state.redis = redis application.state.s3 = s3 - application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads) + application.state.upload_capacity = create_request_capacity( + settings.s3.max_concurrent_uploads + ) application.state.request_capacity = create_request_capacity(settings.max_in_flight) application.state.instance_id = resolve_instance_id(settings.instance_id) application.state.lifecycle = Lifecycle() diff --git a/app/models/__init__.py b/app/models/__init__.py index 43097851..d9476c43 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey from app.models.message import Message from app.models.user import User -__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] +__all__ = [ + "JobReplay", + "AttachmentJob", + "Base", + "Attachment", + "MessageAttachment", + "UploadIntent", + "Chat", + "ChatMember", + "Counter", + "IdempotencyKey", + "Message", + "User", +] diff --git a/app/models/attachment.py b/app/models/attachment.py index 24d075a2..3afebe0d 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -18,8 +18,13 @@ class Attachment(Base): "content_type IN ('image/png', 'image/jpeg')", name="content_type" ), CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), - CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"), - CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", name="status" + ), + CheckConstraint( + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + name="error_code", + ), Index("ix_attachments_chat_id", "chat_id"), ) @@ -36,7 +41,9 @@ class Attachment(Base): thumbnail_key: Mapped[str | None] = mapped_column(String(128)) crop_key: Mapped[str | None] = mapped_column(String(128)) optimized_key: Mapped[str | None] = mapped_column(String(128)) - variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb")) + variants: Mapped[list[dict]] = mapped_column( + JSONB, default=list, server_default=text("'[]'::jsonb") + ) created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job.py b/app/models/job.py index 00ae6a20..01924ee8 100644 --- a/app/models/job.py +++ b/app/models/job.py @@ -20,7 +20,8 @@ class AttachmentJob(Base): id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) attachment_id: Mapped[UUID] = mapped_column( - ForeignKey("attachments.id"), unique=True, + ForeignKey("attachments.id"), + unique=True, ) staging_key: Mapped[str | None] = mapped_column(String(128)) pipeline_version: Mapped[int] = mapped_column(server_default="1") @@ -29,7 +30,9 @@ class AttachmentJob(Base): attempt_limit: Mapped[int | None] last_error_permanent: Mapped[bool | None] published_attempt: Mapped[int] = mapped_column(server_default="0") - next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + next_attempt_at: Mapped[datetime] = mapped_column( + server_default=func.clock_timestamp() + ) lease_token: Mapped[UUID | None] lease_until: Mapped[datetime | None] last_error_code: Mapped[str | None] = mapped_column(String(64)) diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py index 93911c14..629e26b9 100644 --- a/app/schemas/attachment.py +++ b/app/schemas/attachment.py @@ -35,7 +35,9 @@ class Attachment(Base): VARIANT_FILENAMES = { - "thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp", + "thumbnail": "thumbnail.jpg", + "crop": "crop.jpg", + "optimized": "optimized.webp", } diff --git a/app/schemas/job.py b/app/schemas/job.py index 7fb7f341..7b783709 100644 --- a/app/schemas/job.py +++ b/app/schemas/job.py @@ -16,5 +16,7 @@ class JobMessage(BaseModel): pipeline_version: int = Field(default=1, ge=1) attempt: int = Field(ge=1) created_at: AwareDatetime - traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + traceparent: str | None = Field( + default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" + ) tracestate: str | None = Field(default=None, max_length=512) diff --git a/app/storage/broker.py b/app/storage/broker.py index 7358a523..1932ffda 100644 --- a/app/storage/broker.py +++ b/app/storage/broker.py @@ -28,26 +28,42 @@ class JobBroker: logging.getLogger("aiormq").setLevel(logging.CRITICAL) async def ensure_ready(self) -> None: - if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed: + if ( + self.connection is None + or self.connection.is_closed + or self.channel is None + or self.channel.is_closed + ): await self.aclose() self.connection = await aio_pika.connect( - host=self.settings.broker_host, port=self.settings.broker_port, + host=self.settings.broker_host, + port=self.settings.broker_port, login=self.settings.broker_user, password=self.settings.broker_password_file.read_text().strip(), - virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30, + virtualhost=self.settings.broker_vhost, + timeout=5, + heartbeat=30, + ) + self.channel = await self.connection.channel( + publisher_confirms=True, on_return_raises=True ) - self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) await self.channel.set_qos(prefetch_count=1) - self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True) + self.queue = await self.channel.declare_queue( + self.settings.broker_queue, durable=True + ) async def publish_job(self, message: JobMessage) -> None: await self.ensure_ready() await self.channel.default_exchange.publish( aio_pika.Message( body=message.model_dump_json(exclude_none=True).encode(), - content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, message_id=f"{message.job_id}:{message.attempt}", - ), routing_key=self.settings.broker_queue, mandatory=True, timeout=10, + ), + routing_key=self.settings.broker_queue, + mandatory=True, + timeout=10, ) async def receive(self, consumer: str) -> Delivery: diff --git a/app/storage/health.py b/app/storage/health.py index d6913fe6..2d034b71 100644 --- a/app/storage/health.py +++ b/app/storage/health.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine async def check_database(engine: AsyncEngine) -> bool: async with engine.connect() as connection: - await connection.execute(text('SELECT 1')) + await connection.execute(text("SELECT 1")) return True diff --git a/app/storage/idempotency.py b/app/storage/idempotency.py index 5cfcf709..53e174c3 100644 --- a/app/storage/idempotency.py +++ b/app/storage/idempotency.py @@ -33,7 +33,8 @@ async def get_active_key( IdempotencyKey.method, IdempotencyKey.path, IdempotencyKey.key, - ) == tuple_(*scope), + ) + == tuple_(*scope), IdempotencyKey.created_at > func.clock_timestamp() - timedelta(seconds=ttl_seconds), ) @@ -45,14 +46,20 @@ async def save_result( ) -> None: """Сохраняет новый результат или заменяет истёкший.""" stmt = insert(IdempotencyKey).values( - user_id=scope[0], method=scope[1], path=scope[2], key=scope[3], - request_hash=request_hash, message_id=message_id, + user_id=scope[0], + method=scope[1], + path=scope[2], + key=scope[3], + request_hash=request_hash, + message_id=message_id, ) await session.execute( stmt.on_conflict_do_update( index_elements=[ - IdempotencyKey.user_id, IdempotencyKey.method, - IdempotencyKey.path, IdempotencyKey.key, + IdempotencyKey.user_id, + IdempotencyKey.method, + IdempotencyKey.path, + IdempotencyKey.key, ], set_={ "request_hash": request_hash, diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py index 87782c24..6780924a 100644 --- a/app/storage/job_admin.py +++ b/app/storage/job_admin.py @@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int): select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) .where(Attachment.status == "failed") - .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit) + .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id) + .limit(limit) ) return result.all() @@ -23,12 +24,24 @@ async def replay_record(session, request_id): async def insert_replay(session, **values) -> bool: - result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id)) + result = await session.scalar( + insert(JobReplay) + .values(**values) + .on_conflict_do_nothing(index_elements=[JobReplay.id]) + .returning(JobReplay.id) + ) return result is not None async def replay_history(session, job_id): - return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all() + return ( + await session.scalars( + select(JobReplay) + .where(JobReplay.job_id == job_id) + .order_by(JobReplay.created_at) + .limit(100) + ) + ).all() def reset_failed(job, attachment, now, attempt_limit): diff --git a/app/storage/jobs.py b/app/storage/jobs.py index d67545b7..e0ca118b 100644 --- a/app/storage/jobs.py +++ b/app/storage/jobs.py @@ -32,7 +32,9 @@ class JobWork: return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}" -async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: +async def create_job( + session: AsyncSession, attachment_id: UUID, staging_key: str +) -> UUID: job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key) session.add(job) await session.flush() @@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20): result = await session.execute( select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) - .where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp()) + .where( + Attachment.status == "processing", + AttachmentJob.lease_until < func.clock_timestamp(), + ) .order_by(AttachmentJob.lease_until) .limit(limit) .with_for_update(of=AttachmentJob, skip_locked=True) @@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20): return result.all() -def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork: +def start_job( + job, attachment, token: UUID, now: datetime, lease_seconds: int +) -> JobWork: job.attempt += 1 job.lease_token = token job.lease_until = now + timedelta(seconds=lease_seconds) attachment.status = "processing" attachment.updated_at = now return JobWork( - job.id, attachment.id, token, job.pipeline_version, job.staging_key, - attachment.content_type, attachment.size_bytes, attachment.sha256, + job.id, + attachment.id, + token, + job.pipeline_version, + job.staging_key, + attachment.content_type, + attachment.size_bytes, + attachment.sha256, ) -def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None: +def record_failure( + job, + attachment, + now: datetime, + error_code: str, + *, + terminal: bool, + retry_seconds: float, +) -> None: job.lease_token = None job.lease_until = None job.last_error_code = error_code @@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: attachment.updated_at = now -def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None: +def complete_job( + job, attachment, work: JobWork, variants: list[dict], now: datetime +) -> None: attachment.object_key = work.original_key attachment.thumbnail_key = work.variant_key("thumbnail") attachment.crop_key = work.variant_key("crop") @@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date job.finished_at = now -async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20): +async def terminal_staging( + session: AsyncSession, retention: int, failed_retention: int, limit: int = 20 +): result = await session.execute( select(AttachmentJob, Attachment) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) .where( AttachmentJob.staging_key.is_not(None), or_( - and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)), - and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)), + and_( + Attachment.status == "ready", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=retention), + ), + and_( + Attachment.status == "failed", + AttachmentJob.finished_at + < func.clock_timestamp() - timedelta(seconds=failed_retention), + ), ), ) .order_by(AttachmentJob.finished_at) diff --git a/app/storage/outbox.py b/app/storage/outbox.py index e791c841..be50c31b 100644 --- a/app/storage/outbox.py +++ b/app/storage/outbox.py @@ -9,7 +9,9 @@ from app.models.attachment import Attachment from app.models.job import AttachmentJob -async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None: +async def next_unpublished_job( + session: AsyncSession, *, attachment_id: UUID | None = None +) -> AttachmentJob | None: statement = ( select(AttachmentJob) .join(Attachment, Attachment.id == AttachmentJob.attachment_id) diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py index f1c76d70..b8f399f3 100644 --- a/infra/minio/bootstrap.py +++ b/infra/minio/bootstrap.py @@ -81,7 +81,10 @@ def main(): { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")], + "Resource": [ + f"arn:aws:s3:::{bucket}/{prefix}/*" + for prefix in ("originals", "staging", "variants") + ], }, ], } diff --git a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py index ea3c65fc..153425e2 100644 --- a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py +++ b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py @@ -20,21 +20,26 @@ def upgrade() -> None: sa.Column("request_hash", sa.String(64), nullable=False), sa.Column("message_id", sa.UUID(), nullable=False), sa.Column( - "created_at", sa.DateTime(timezone=True), - server_default=sa.text("clock_timestamp()"), nullable=False, + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, ), sa.ForeignKeyConstraint( ["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users") ), sa.ForeignKeyConstraint( - ["message_id"], ["messages.id"], + ["message_id"], + ["messages.id"], name=op.f("fk_idempotency_keys_message_id_messages"), ), sa.PrimaryKeyConstraint( "user_id", "method", "path", "key", name=op.f("pk_idempotency_keys") ), ) - op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]) + op.create_index( + "ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"] + ) def downgrade() -> None: diff --git a/migrations/versions/8f31b94a2d10_add_attachments.py b/migrations/versions/8f31b94a2d10_add_attachments.py index b221dccb..34d3a6d5 100644 --- a/migrations/versions/8f31b94a2d10_add_attachments.py +++ b/migrations/versions/8f31b94a2d10_add_attachments.py @@ -21,14 +21,35 @@ def upgrade() -> None: sa.Column("size_bytes", sa.Integer(), nullable=False), sa.Column("sha256", sa.String(64), nullable=False), sa.Column("status", sa.String(16), server_default="ready", nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), - sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")), - sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")), - sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), + sa.CheckConstraint( + "size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes") + ), + sa.CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", + name=op.f("ck_attachments_content_type"), + ), + sa.CheckConstraint( + "sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256") + ), sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")), - sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")), - sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats") + ), + sa.ForeignKeyConstraint( + ["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users") + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")), sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")), ) @@ -38,15 +59,31 @@ def upgrade() -> None: sa.Column("message_id", sa.UUID(), nullable=False), sa.Column("attachment_id", sa.UUID(), nullable=False), sa.Column("position", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")), - sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")), - sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + ondelete="CASCADE", + name=op.f("fk_message_attachments_message_id_messages"), + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_message_attachments_attachment_id_attachments"), + ), + sa.PrimaryKeyConstraint( + "message_id", "attachment_id", name=op.f("pk_message_attachments") + ), ) op.create_table( "upload_intents", sa.Column("id", sa.UUID(), nullable=False), sa.Column("object_key", sa.String(128), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")), sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")), ) diff --git a/migrations/versions/abd273076068_add_counters.py b/migrations/versions/abd273076068_add_counters.py index 9c809b06..cb6e08c5 100644 --- a/migrations/versions/abd273076068_add_counters.py +++ b/migrations/versions/abd273076068_add_counters.py @@ -5,29 +5,30 @@ Revises: db06fd3126ff Create Date: 2026-10-26 22:01:01.910396 """ + from typing import Sequence, Union from alembic import op import sqlalchemy as sa - -revision: str = 'abd273076068' -down_revision: Union[str, None] = 'db06fd3126ff' +revision: str = "abd273076068" +down_revision: Union[str, None] = "db06fd3126ff" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('counters', - sa.Column('name', sa.String(length=64), nullable=False), - sa.Column('value', sa.BigInteger(), nullable=False), - sa.PrimaryKeyConstraint('name', name=op.f('pk_counters')) + op.create_table( + "counters", + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("value", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('counters') + op.drop_table("counters") # ### end Alembic commands ### diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py index 302ebc6d..5d42ac5d 100644 --- a/migrations/versions/c35e91a2f706_async_attachment_jobs.py +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -12,12 +12,30 @@ depends_on = None def upgrade() -> None: op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") - op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.create_check_constraint( + op.f("ck_attachments_status"), + "attachments", + "status IN ('queued', 'processing', 'ready', 'failed')", + ) op.alter_column("attachments", "status", server_default="queued") op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True)) - op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)) - op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb"))) - op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)") + op.add_column( + "attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True) + ) + op.add_column( + "attachments", + sa.Column( + "variants", + postgresql.JSONB(), + nullable=False, + server_default=sa.text("'[]'::jsonb"), + ), + ) + op.create_check_constraint( + op.f("ck_attachments_error_code"), + "attachments", + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + ) op.create_table( "attachment_jobs", sa.Column("id", sa.UUID(), nullable=False), @@ -25,31 +43,62 @@ def upgrade() -> None: sa.Column("staging_key", sa.String(128), nullable=True), sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False), sa.Column("attempt", sa.Integer(), server_default="0", nullable=False), - sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False), - sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "published_attempt", sa.Integer(), server_default="0", nullable=False + ), + sa.Column( + "next_attempt_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.Column("lease_token", sa.UUID(), nullable=True), sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), sa.Column("last_error_code", sa.String(64), nullable=True), sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("clock_timestamp()"), + nullable=False, + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")), - sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")), - sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")), - sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")), - sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name=op.f("fk_attachment_jobs_attachment_id_attachments"), + ), + sa.UniqueConstraint( + "attachment_id", name=op.f("uq_attachment_jobs_attachment_id") + ), + sa.CheckConstraint( + "pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version") + ), + sa.CheckConstraint( + "attempt >= 0 AND published_attempt >= 0", + name=op.f("ck_attachment_jobs_attempt"), + ), ) op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"]) op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"]) # Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail. - op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments") - op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()") + op.execute( + "INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments" + ) + op.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + ) def downgrade() -> None: connection = op.get_bind() - unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")) + unfinished = connection.scalar( + sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'") + ) if unfinished: - raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения") + raise RuntimeError( + "Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения" + ) op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs") op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs") op.drop_table("attachment_jobs") @@ -58,5 +107,7 @@ def downgrade() -> None: op.drop_column("attachments", "thumbnail_key") op.drop_column("attachments", "error_code") op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") - op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'") + op.create_check_constraint( + op.f("ck_attachments_status"), "attachments", "status = 'ready'" + ) op.alter_column("attachments", "status", server_default="ready") diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py index 2576cad3..b02f44f4 100644 --- a/migrations/versions/e7048b6da913_attachment_crop_webp.py +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -11,7 +11,9 @@ depends_on = None def upgrade() -> None: op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True)) - op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True)) + op.add_column( + "attachments", sa.Column("optimized_key", sa.String(128), nullable=True) + ) # Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup. op.execute(""" INSERT INTO upload_intents (id, object_key) @@ -37,7 +39,11 @@ def upgrade() -> None: def downgrade() -> None: - active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")) + active = op.get_bind().scalar( + sa.text( + "SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')" + ) + ) if active: raise RuntimeError("Перед downgrade необходимо завершить активные задания") op.execute(""" diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py index 7e6ed24c..0c495fbf 100644 --- a/migrations/versions/f9183d27a640_job_replay.py +++ b/migrations/versions/f9183d27a640_job_replay.py @@ -10,19 +10,36 @@ depends_on = None def upgrade() -> None: - op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0")) - op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)) - op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True)) + op.add_column( + "attachment_jobs", + sa.Column( + "cycle_start_attempt", sa.Integer(), nullable=False, server_default="0" + ), + ) + op.add_column( + "attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True) + ) + op.add_column( + "attachment_jobs", + sa.Column("last_error_permanent", sa.Boolean(), nullable=True), + ) op.create_table( "job_replays", sa.Column("id", sa.Uuid(), primary_key=True), - sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column( + "job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False + ), sa.Column("actor", sa.String(64), nullable=False), sa.Column("reason", sa.String(32), nullable=False), sa.Column("previous_attempt", sa.Integer(), nullable=False), sa.Column("attempt_limit", sa.Integer(), nullable=False), sa.Column("previous_error_code", sa.String(64), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.clock_timestamp(), + ), ) op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"]) diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..d31b8a3f 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Offline structural checks; full schema validation is a separate target.""" + import ast import json from pathlib import Path @@ -7,48 +8,71 @@ import re ROOT = Path(__file__).resolve().parents[1] + def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) - assert spec['openapi']=='3.1.0' - assert spec['info']['version']=='1.%d.0'%lab + lab = json.loads((ROOT / "lab.json").read_text())["number"] + spec = json.loads((ROOT / "contracts/openapi.json").read_text()) + assert spec["openapi"] == "3.1.0" + assert spec["info"]["version"] == "1.%d.0" % lab operation_ids = set() + def walk(value): - if isinstance(value,dict): - if '$ref' in value: - pointer = value['$ref'] - assert pointer.startswith('#/'), 'unexpected external reference' + if isinstance(value, dict): + if "$ref" in value: + pointer = value["$ref"] + assert pointer.startswith("#/"), "unexpected external reference" target = spec - for part in pointer[2:].split('/'): - target = target[part.replace('~1','/').replace('~0','~')] + for part in pointer[2:].split("/"): + target = target[part.replace("~1", "/").replace("~0", "~")] for v in value.values(): walk(v) - elif isinstance(value,list): + elif isinstance(value, list): for v in value: walk(v) + walk(spec) - for route,methods in spec['paths'].items(): - placeholders = set(re.findall(r'\{([^}]+)\}',route)) - for method,op in methods.items(): - assert op['operationId'] not in operation_ids, 'duplicate operationId' - operation_ids.add(op['operationId']) - params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')} - assert placeholders==params, 'path parameter mismatch: '+route - for response in op['responses'].values(): - assert 'X-Request-Id' in response['headers'] - if lab>=3: - assert 'X-Instance-Id' in response['headers'] - assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) - assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) - for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) - for file in ROOT.rglob('*.json'): - if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): + for route, methods in spec["paths"].items(): + placeholders = set(re.findall(r"\{([^}]+)\}", route)) + for method, op in methods.items(): + assert op["operationId"] not in operation_ids, "duplicate operationId" + operation_ids.add(op["operationId"]) + params = { + p["name"] + for p in op.get("parameters", []) + if p["in"] == "path" and p.get("required") + } + assert placeholders == params, "path parameter mismatch: " + route + for response in op["responses"].values(): + assert "X-Request-Id" in response["headers"] + if lab >= 3: + assert "X-Instance-Id" in response["headers"] + assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2) + assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4) + for file in list((ROOT / "tests").glob("*.py")) + list( + (ROOT / "scripts").glob("*.py") + ): + ast.parse(file.read_text(), filename=str(file)) + for file in ROOT.rglob("*.json"): + if any( + part in {".git", ".venv", "node_modules", "vendor", "output"} + for part in file.parts + ): continue json.loads(file.read_text()) - for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): - assert (ROOT/required).is_file(), 'missing '+required - print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') + for required in ( + "README.md", + "COURSE.md", + "CONTRIBUTING.md", + "REPORT.md", + "contracts/README.md", + "tests/smoke.py", + "tests/README.md", + ): + assert (ROOT / required).is_file(), "missing " + required + print( + "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested." + ) -if __name__=='__main__': + +if __name__ == "__main__": main() diff --git a/scripts/check_jobs.py b/scripts/check_jobs.py index dc724b66..76519c51 100644 --- a/scripts/check_jobs.py +++ b/scripts/check_jobs.py @@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos from scripts.deployment import ComposeProject from tests.client import Client, check -PROBE = ''' +PROBE = """ import asyncio, hashlib, json from uuid import UUID from sqlalchemy import select @@ -43,169 +43,387 @@ async def main(): result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) finally: await storage.aclose();await engine.dispose() asyncio.run(main()) -''' +""" def probe(attachment_id): - code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE - result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30) - check(result.returncode == 0, 'Probe failed; diagnostic output hidden') + code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE + result = subprocess.run( + COMPOSE + ["exec", "-T", "api", "python", "-"], + input=code, + text=True, + capture_output=True, + cwd=ROOT, + timeout=30, + ) + check(result.returncode == 0, "Probe failed; diagnostic output hidden") return json.loads(result.stdout) def admin(*arguments, expected=0): - result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments) - check(result.returncode == expected, 'Unexpected operator command exit status') + result = compose( + "run", + "--rm", + "--no-deps", + "worker", + "python", + "-m", + "app.core.job_admin", + *arguments + ) + check(result.returncode == expected, "Unexpected operator command exit status") return json.loads(result.stdout) def verify_result(client, actor, attachment, original): ready = client.wait_ready(actor, attachment, wait_seconds=180) - check(ready['id'] == attachment['id'], 'Attachment identity changed') - path = '/api/v1/attachments/' + ready['id'] + '/content' - check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed') - check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline') - for variant in ready['variants']: - data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes'] - check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch') + check(ready["id"] == attachment["id"], "Attachment identity changed") + path = "/api/v1/attachments/" + ready["id"] + "/content" + check( + client.call("GET", path, actor=actor)["bytes"] == original, "Original changed" + ) + check( + {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}, + "Incomplete pipeline", + ) + for variant in ready["variants"]: + data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[ + "bytes" + ] + check( + hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch" + ) with Image.open(BytesIO(data)) as image: - check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch') - check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format') - check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked') - if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size') - snapshot=probe(ready['id']) - check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants') + check( + image.size == (variant["width"], variant["height"]), + "Metadata dimensions mismatch", + ) + check( + image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"), + "Wrong format", + ) + check( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp"), + "Metadata leaked", + ) + if variant["name"] == "crop": + check(image.size == (128, 128), "Wrong crop size") + snapshot = probe(ready["id"]) + check( + len(snapshot["objects"]) == 4, + "Expected exactly one original and three variants", + ) return snapshot def crash(client, actor, chat, mode): - name='messenger-worker-fault-'+uuid4().hex + name = "messenger-worker-fault-" + uuid4().hex try: - require_compose('stop','worker') - attachment,original=client.attachment(actor,chat) - result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) - phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack' - marker=json.dumps({'fault':phase,'attachment_id':attachment['id']}) - check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed') - before=probe(attachment['id']) - check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state') - check(len(before['objects'])==4,'Crash must occur after all four PUTs') + require_compose("stop", "worker") + attachment, original = client.attachment(actor, chat) + result = compose( + "run", + "--rm", + "--no-deps", + "--name", + name, + "worker", + "python", + "-m", + "app.core.job_worker", + "--" + mode, + attachment["id"], + ) + phase = ( + "after_object:optimized" if mode == "crash-after-object" else "before_ack" + ) + marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]}) + check( + result.returncode == 137 and marker in result.stdout, + "Targeted crash not observed", + ) + before = probe(attachment["id"]) + check( + before["status"] + == ("processing" if mode == "crash-after-object" else "ready"), + "Unexpected crash state", + ) + check(len(before["objects"]) == 4, "Crash must occur after all four PUTs") finally: - subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30) - require_compose('start','worker') - after=verify_result(client,actor,attachment,original) - if mode=='crash-before-ack': + subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30) + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + if mode == "crash-before-ack": # Wait for RabbitMQ to deliver and acknowledge the ready duplicate. wait_queue_empty() - after=probe(attachment['id']) - check(after==before,'Ready duplicate rewrote result') + after = probe(attachment["id"]) + check(after == before, "Ready duplicate rewrote result") else: - check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt') - check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content') - return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after) + check( + after["attempt"] == before["attempt"] + 1, + "Lease did not recover the interrupted attempt", + ) + check( + [(x["name"], x["sha256"]) for x in before["objects"]] + == [(x["name"], x["sha256"]) for x in after["objects"]], + "Replay changed object content", + ) + return dict( + scenario=mode, + attachment_id=attachment["id"], + exit_code=137, + before=before, + after=after, + ) -def retry_replay(client,actor,chat,permanent): - name='messenger-retry-fault-'+uuid4().hex - samples=[] - configuration=ComposeProject().config['services']['worker']['environment'] - retry_base=float(configuration['JOB_RETRY_SECONDS']) - retry_max=float(configuration['JOB_RETRY_MAX_SECONDS']) +def retry_replay(client, actor, chat, permanent): + name = "messenger-retry-fault-" + uuid4().hex + samples = [] + configuration = ComposeProject().config["services"]["worker"]["environment"] + retry_base = float(configuration["JOB_RETRY_SECONDS"]) + retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"]) try: - require_compose('stop','worker') - attachment,original=client.attachment(actor,chat) - mode='fail-permanent' if permanent else 'fail-transient' - require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) - deadline=time.monotonic()+180 - while time.monotonic()=1 and queued_before['durable'], 'No durable queued message before restart') - result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180) - check(result.returncode==0,'Restart hook failed') - check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker') - queued_after=queue_state() - check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart') - client.message(actor,chat,'session and chat survived broker/API restart') - finally:require_compose('start','worker') - after=verify_result(client,actor,attachment,original) - return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after) + queued_before = queue_state() + check( + queued_before["messages_ready"] >= 1 and queued_before["durable"], + "No durable queued message before restart", + ) + result = subprocess.run( + [str(ROOT / "scripts/restart.sh")], + cwd=ROOT, + capture_output=True, + timeout=180, + ) + check(result.returncode == 0, "Restart hook failed") + check( + "worker" + not in require_compose( + "ps", "--status", "running", "--services" + ).splitlines(), + "Restart started deliberately stopped worker", + ) + queued_after = queue_state() + check( + queued_after["messages_ready"] >= 1 and queued_after["durable"], + "Broker lost queued delivery across restart", + ) + client.message(actor, chat, "session and chat survived broker/API restart") + finally: + require_compose("start", "worker") + after = verify_result(client, actor, attachment, original) + return dict( + scenario="worker_stopped_restart_all", + attachment_id=attachment["id"], + queue_before=queued_before, + queue_after=queued_after, + after=after, + ) def main(): - parser=argparse.ArgumentParser() - parser.add_argument('--base-url',required=True) - parser.add_argument('--ca-file') - parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all') - parser.add_argument('--output',default='evidence/jobs.json') - args=parser.parse_args() - client=Client(args.base_url,args.ca_file,timeout=40) - aliases={r.instance_id for r in ComposeProject().replicas() if r.running} - check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL') - actor,_,_,chat=client.bootstrap() - results=[] - operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)} - selected=operations.values() if args.scenario=='all' else [operations[args.scenario]] + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + parser.add_argument( + "--scenario", + choices=[ + "all", + "persistence", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent", + ], + default="all", + ) + parser.add_argument("--output", default="evidence/jobs.json") + args = parser.parse_args() + client = Client(args.base_url, args.ca_file, timeout=40) + aliases = {r.instance_id for r in ComposeProject().replicas() if r.running} + check( + client.call("GET", "/health/live", schema="Health")["headers"].get( + "X-Instance-Id" + ) + in aliases, + "Wrong Compose project for BASE_URL", + ) + actor, _, _, chat = client.bootstrap() + results = [] + operations = { + "persistence": persistence, + "crash-after-object": lambda *x: crash(*x, "crash-after-object"), + "crash-before-ack": lambda *x: crash(*x, "crash-before-ack"), + "fail-transient": lambda *x: retry_replay(*x, False), + "fail-permanent": lambda *x: retry_replay(*x, True), + } + selected = ( + operations.values() if args.scenario == "all" else [operations[args.scenario]] + ) for operation in selected: - result=operation(client,actor,chat);results.append(result) - Path(args.output).parent.mkdir(parents=True,exist_ok=True) - Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n') - print('PASS: '+result['scenario'],flush=True) - Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n') + result = operation(client, actor, chat) + results.append(result) + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text( + json.dumps(dict(status="in_progress", results=results), indent=2) + "\n" + ) + print("PASS: " + result["scenario"], flush=True) + Path(args.output).write_text( + json.dumps(dict(status="passed", results=results), indent=2) + "\n" + ) -if __name__=='__main__':main() +if __name__ == "__main__": + main() diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py index 288257e9..c52471a4 100644 --- a/scripts/check_outbox_recovery.py +++ b/scripts/check_outbox_recovery.py @@ -19,7 +19,13 @@ COMPOSE = ["docker", "compose", "--env-file", ".env"] def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess: - return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout) + return subprocess.run( + COMPOSE + list(arguments), + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + ) def require_compose(*arguments: str) -> str: @@ -29,54 +35,112 @@ def require_compose(*arguments: str) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish") + parser = argparse.ArgumentParser( + description="Отказ publisher после COMMIT перед RabbitMQ publish" + ) parser.add_argument("--base-url", required=True) parser.add_argument("--ca-file") args = parser.parse_args() - running = set(require_compose("ps", "--status", "running", "--services").splitlines()) - check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario") + running = set( + require_compose("ps", "--status", "running", "--services").splitlines() + ) + check( + {"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, + "Start the application stack before this scenario", + ) client = Client(args.base_url, args.ca_file, timeout=40) project = ComposeProject() aliases = {replica.instance_id for replica in project.replicas() if replica.running} live = client.call("GET", "/health/live", schema="Health") - check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) alice, _, _, chat = client.bootstrap() attachment = None fault_container = "messenger-outbox-fault-" + uuid4().hex try: require_compose("stop", "publisher") attachment, original = client.attachment(alice, chat) - message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] - client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice) - killed = compose( - "run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m", - "app.core.outbox_publisher", "--crash-before-publish", attachment["id"], + message = client.message(alice, chat, "", attachments=[attachment["id"]])[ + "json" + ] + client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + expected=409, + actor=alice, ) - marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]}) - check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed") - pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"] + killed = compose( + "run", + "--rm", + "--no-deps", + "--name", + fault_container, + "publisher", + "python", + "-m", + "app.core.outbox_publisher", + "--crash-before-publish", + attachment["id"], + ) + marker = json.dumps( + {"fault": "before_publish", "attachment_id": attachment["id"]} + ) + check( + killed.returncode == 137 and marker in killed.stdout, + "Expected targeted SIGKILL before publish was not observed", + ) + pending = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}", + actor=alice, + schema="Attachment", + )["json"] check(pending["status"] == "queued", "Crash must leave accepted job queued") finally: # Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется. try: - subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30) + subprocess.run( + ["docker", "rm", "-f", fault_container], capture_output=True, timeout=30 + ) finally: require_compose("start", "publisher") ready = client.wait_ready(alice, attachment, wait_seconds=60) check(ready["id"] == attachment["id"], "Recovered a different attachment") - check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants") + check( + {variant["name"] for variant in ready["variants"]} + == {"thumbnail", "crop", "optimized"}, + "Missing required variants", + ) path = f"/api/v1/attachments/{attachment['id']}/content" - check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery") + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "Original changed after recovery", + ) for variant in ready["variants"]: - downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"] - check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch") - print(json.dumps({ - "scenario": "commit_before_publish", "attachment_id": ready["id"], - "message_id": message["id"], "publisher_exit_code": killed.returncode, - "status_after_crash": pending["status"], "status_after_restart": ready["status"], - "variants": [variant["name"] for variant in ready["variants"]], - "reuploaded": False, - })) + downloaded = client.call( + "GET", path + "?variant=" + variant["name"], actor=alice + )["bytes"] + check( + len(downloaded) == variant["size_bytes"] + and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], + "Recovered variant checksum mismatch", + ) + print( + json.dumps( + { + "scenario": "commit_before_publish", + "attachment_id": ready["id"], + "message_id": message["id"], + "publisher_exit_code": killed.returncode, + "status_after_crash": pending["status"], + "status_after_restart": ready["status"], + "variants": [variant["name"] for variant in ready["variants"]], + "reuploaded": False, + } + ) + ) if __name__ == "__main__": diff --git a/scripts/check_s3.py b/scripts/check_s3.py index f94f6a9e..eddadaf0 100644 --- a/scripts/check_s3.py +++ b/scripts/check_s3.py @@ -7,7 +7,7 @@ import time from scripts.deployment import ComposeProject from tests.client import Client, arguments, check, run -PROBE = r''' +PROBE = r""" import asyncio, json, sys from pathlib import Path from uuid import UUID @@ -62,60 +62,91 @@ async def main(): await storage.aclose() await engine.dispose() asyncio.run(main()) -''' +""" def main(): def configure(parser): - parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта') - args = arguments('MinIO private bucket and failure checks', configure) + parser.add_argument( + "--with-outage", + action="store_true", + help="временно остановить MinIO выбранного проекта", + ) + + args = arguments("MinIO private bucket and failure checks", configure) project = ComposeProject() client = Client(args.base_url, args.ca_file, timeout=40) alice, _, _, chat = client.bootstrap() aliases = {replica.instance_id for replica in project.replicas() if replica.running} - live = client.call('GET', '/health/live', schema='Health') - check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project') + live = client.call("GET", "/health/live", schema="Health") + check( + live["headers"].get("X-Instance-Id") in aliases, + "BASE_URL does not belong to the selected Compose project", + ) attachment, original = client.attachment(alice, chat) attachment = client.wait_ready(alice, attachment) # Повторный bootstrap не должен очистить bucket или сменить используемые credentials. for _ in range(2): - project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100) + project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100) replicas = project.replicas() - check(bool(replicas), 'API not running') - result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE], - input=json.dumps({'attachment_id': attachment['id']}), - text=True, capture_output=True, timeout=20) - check(result.returncode == 0, 'private bucket probe failed (details hidden)') + check(bool(replicas), "API not running") + result = subprocess.run( + ["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE], + input=json.dumps({"attachment_id": attachment["id"]}), + text=True, + capture_output=True, + timeout=20, + ) + check(result.returncode == 0, "private bucket probe failed (details hidden)") statuses = json.loads(result.stdout) - check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed') - check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access') - check(statuses['console_unreachable'], 'console reachable from API network') - check(statuses['temp_originals_absent'], 'API retained temporary originals') - check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports') - env = project.config['services']['minio']['environment'] - check(env['MINIO_BROWSER'] == 'off', 'browser enabled') - check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback') - path = '/api/v1/attachments/' + attachment['id'] + '/content' - check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original') + check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed") + check( + statuses["admin_denied"] + and statuses["other_buckets_denied"] + and statuses["root_files_absent"], + "API has excessive access", + ) + check(statuses["console_unreachable"], "console reachable from API network") + check(statuses["temp_originals_absent"], "API retained temporary originals") + check( + not project.config["services"]["minio"].get("ports"), + "MinIO publishes host ports", + ) + env = project.config["services"]["minio"]["environment"] + check(env["MINIO_BROWSER"] == "off", "browser enabled") + check( + "127.0.0.1:9001" in project.config["services"]["minio"]["command"], + "console is not bound to loopback", + ) + path = "/api/v1/attachments/" + attachment["id"] + "/content" + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "bootstrap changed original", + ) if args.with_outage: try: - project.run('stop', 'minio') - client.call('GET', path, expected=503, actor=alice) + project.run("stop", "minio") + client.call("GET", path, expected=503, actor=alice) finally: - project.run('start', 'minio') + project.run("start", "minio") deadline = time.monotonic() + 60 while True: try: - client.call('GET', '/health/ready', schema='Health') + client.call("GET", "/health/ready", schema="Health") break except Exception: if time.monotonic() >= deadline: raise time.sleep(0.5) - check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart') - print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console' - + (', MinIO outage/recovery' if args.with_outage else '')) + check( + client.call("GET", path, actor=alice)["bytes"] == original, + "original lost after MinIO restart", + ) + print( + "PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console" + + (", MinIO outage/recovery" if args.with_outage else "") + ) -if __name__ == '__main__': +if __name__ == "__main__": run(main) diff --git a/scripts/deployment.py b/scripts/deployment.py index fba70f5e..c629e6e8 100644 --- a/scripts/deployment.py +++ b/scripts/deployment.py @@ -31,8 +31,12 @@ class ComposeProject: def docker(self, *args: str, timeout: int = 90, env=None) -> str: """Выполняет Docker-команду с ограничением времени.""" result = subprocess.run( - ["docker", *args], cwd=ROOT, text=True, capture_output=True, - timeout=timeout, env={**os.environ, **(env or {})}, + ["docker", *args], + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + env={**os.environ, **(env or {})}, ) if result.returncode: raise RuntimeError(result.stderr.strip() or result.stdout.strip()) @@ -53,8 +57,17 @@ class ComposeProject: '"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}' ) rows = self.docker("inspect", "--format", template, *ids).splitlines() - configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip() + configured = str( + self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") + or "" + ).strip() return [ - Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"]) + Replica( + row["id"], + row["image"], + configured or row["hostname"], + row["running"], + row["health"], + ) for row in map(json.loads, rows) ] diff --git a/scripts/init_local.py b/scripts/init_local.py index a3fb687a..7477cd09 100644 --- a/scripts/init_local.py +++ b/scripts/init_local.py @@ -14,7 +14,15 @@ def main() -> None: destination.chmod(0o600) directory = root / "secrets" directory.mkdir(mode=0o700, exist_ok=True) - for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): + for name in ( + "db_password", + "redis_password", + "broker_password", + "minio_root_user", + "minio_root_password", + "s3_access_key_id", + "s3_secret_access_key", + ): path = directory / name try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 7e1d1afc..8351b1c4 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -1,23 +1,27 @@ #!/usr/bin/env python3 """Full OpenAPI and JSON Schema checks; install requirements-dev.txt first.""" + import json from pathlib import Path from openapi_spec_validator import validate from jsonschema import Draft202012Validator root = Path(__file__).resolve().parents[1] -spec = json.loads((root/'contracts/openapi.json').read_text()) +spec = json.loads((root / "contracts/openapi.json").read_text()) validate(spec) from referencing import Registry, Resource + # The base URI resolves local #/components references inside examples. -resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) -registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource)) -for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): - schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name} +resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"}) +registry = Registry().with_resource( + "urn:messenger:openapi", Resource.from_contents(resource) +) +for name, example in json.loads((root / "contracts/examples.json").read_text()).items(): + schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name} Draft202012Validator(schema, registry=registry).validate(example) -for path in (root/'contracts').glob('*.schema.json'): +for path in (root / "contracts").glob("*.schema.json"): schema = json.loads(path.read_text()) Draft202012Validator.check_schema(schema) - for example in schema.get('examples',[]): + for example in schema.get("examples", []): Draft202012Validator(schema).validate(example) -print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.') +print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 20237f1f..dba63f94 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -28,7 +28,7 @@ from app.storage import attachments as repository from app.storage.s3 import ObjectInfo from helpers import bearer, create_chat, create_user, idempotency_header -PNG = Path('tests/fixtures/sample.png').read_bytes() +PNG = Path("tests/fixtures/sample.png").read_bytes() class MemoryStorage: @@ -55,12 +55,15 @@ class MemoryStorage: info = None if key in self.objects: data, metadata = self.objects[key] - info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc)) + info = ObjectInfo( + key, len(data), metadata["content_type"], datetime.now(timezone.utc) + ) return info async def open_object(self, key): async def chunks(): yield self.objects[key][0] + return chunks() async def check_available(self): @@ -76,23 +79,40 @@ async def context(client, app): return storage, alice, bob, eve, chat -async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'): +async def upload( + client, actor, chat, data=PNG, filename="фото.png", content_type="image/png" +): return await client.post( - f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename}, - headers={**bearer(actor), 'Content-Type': content_type}, content=data, + f"/api/v1/chats/{chat['id']}/attachments", + params={"filename": filename}, + headers={**bearer(actor), "Content-Type": content_type}, + content=data, ) async def finish_attachment(client, storage, actor, attachment_id): engine = client._transport.app.state.engine async with AsyncSession(engine) as session: - job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) - message = JobMessage(job_id=job.id, attachment_id=job.attachment_id, - pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at) + job = await session.scalar( + select(AttachmentJob).where( + AttachmentJob.attachment_id == UUID(attachment_id) + ) + ) + message = JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt + 1, + created_at=job.created_at, + ) await session.rollback() - assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None)) - result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor)) - assert result.json()['status'] == 'ready' + assert await process_attachment_job( + session, storage, message, JobSettings(_env_file=None) + ) + result = await client.get( + f"/api/v1/attachments/{attachment_id}", headers=bearer(actor) + ) + assert result.json()["status"] == "ready" return result.json() @@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context): response = await upload(client, alice, chat) assert response.status_code == 202, response.text attachment = response.json() - assert attachment['status'] == 'queued' - before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)) + assert attachment["status"] == "queued" + before = await client.get( + f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice) + ) assert before.status_code == 409 - attachment = await finish_attachment(client, storage, alice, attachment['id']) - assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest() - assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'} - assert attachment['error_code'] is None - assert 'object_key' not in attachment - id = attachment['id'] + attachment = await finish_attachment(client, storage, alice, attachment["id"]) + assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest() + assert {v["name"] for v in attachment["variants"]} == { + "thumbnail", + "crop", + "optimized", + } + assert attachment["error_code"] is None + assert "object_key" not in attachment + id = attachment["id"] for actor in (alice, bob): - content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor)) + content = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(actor) + ) assert content.status_code == 200 and content.content == PNG - assert content.headers['content-type'] == 'image/png' + assert content.headers["content-type"] == "image/png" key = idempotency_header() - body = {'text': '', 'attachment_ids': [id]} + body = {"text": "", "attachment_ids": [id]} path = f"/api/v1/chats/{chat['id']}/messages" message = await client.post(path, json=body, headers={**bearer(bob), **key}) assert message.status_code == 201, message.text repeat = await client.post(path, json=body, headers={**bearer(bob), **key}) assert repeat.json() == message.json() history = await client.get(path, headers=bearer(alice)) - assert history.json()['items'][0]['attachment_ids'] == [id] - for suffix in ('', '/content'): - denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve)) + assert history.json()["items"][0]["attachment_ids"] == [id] + for suffix in ("", "/content"): + denied = await client.get( + f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve) + ) assert denied.status_code == 404 @pytest.mark.parametrize( - 'data,content_type', - [(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')], - ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'], + "data,content_type", + [ + (b"not an image", "image/png"), + (PNG[:30], "image/png"), + (PNG, "image/jpeg"), + (PNG, "application/octet-stream"), + ], + ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"], ) async def test_invalid_image_and_mime_are_415(client, context, data, content_type): storage, alice, _, _, chat = context @@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ async def test_jpeg_is_stored_without_reencoding(client, context): storage, alice, _, _, chat = context output = BytesIO() - Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG') + Image.new("RGB", (32, 32), "blue").save(output, format="JPEG") original = output.getvalue() - response = await upload(client, alice, chat, original, content_type='image/jpeg') + response = await upload(client, alice, chat, original, content_type="image/jpeg") assert response.status_code == 202, response.text - id = response.json()['id'] + id = response.json()["id"] await finish_attachment(client, storage, alice, id) - downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + downloaded = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) assert downloaded.content == original -@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256]) +@pytest.mark.parametrize( + "filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256] +) async def test_unsafe_filename_is_400(client, context, filename): storage, alice, _, _, chat = context response = await upload(client, alice, chat, filename=filename) @@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename): async def test_oversize_chunked_upload_is_413(client, context): storage, alice, _, _, chat = context + async def chunks(): for _ in range(11): - yield b'x' * 1048576 + yield b"x" * 1048576 + response = await upload(client, alice, chat, chunks()) assert response.status_code == 413 assert not storage.objects @@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context): _, alice, _, _, chat = context response = await client.post( f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", - headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG, + headers={ + **bearer(alice), + "Content-Type": "image/png", + "Content-Length": "10485761", + }, + content=PNG, ) assert response.status_code == 413 @@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context): maximum = 10_485_760 # Валидный ancillary chunk вместо данных после IEND. payload = bytes(maximum - len(PNG) - 12) - kind = b'npAD' - padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload)) + kind = b"npAD" + padding = ( + struct.pack("!I", len(payload)) + + kind + + payload + + struct.pack("!I", zlib.crc32(kind + payload)) + ) image = PNG[:-12] + padding + PNG[-12:] response = await upload(client, alice, chat, image) assert response.status_code == 202, response.text - assert response.json()['size_bytes'] == maximum - assert response.json()['sha256'] == hashlib.sha256(image).hexdigest() + assert response.json()["size_bytes"] == maximum + assert response.json()["sha256"] == hashlib.sha256(image).hexdigest() async def test_exact_pixel_limit_accepts_valid_png(client, context): _, alice, _, _, chat = context output = BytesIO() - with Image.new('RGB', (8000, 5000), 'blue') as image: - image.save(output, format='PNG') + with Image.new("RGB", (8000, 5000), "blue") as image: + image.save(output, format="PNG") response = await upload(client, alice, chat, output.getvalue()) assert response.status_code == 202, response.text async def test_pixel_limit_before_decoding(client, context): _, alice, _, _, chat = context + def chunk(kind, data): - return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data)) - header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0) - image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'') + return ( + struct.pack("!I", len(data)) + + kind + + data + + struct.pack("!I", zlib.crc32(kind + data)) + ) + + header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0) + image = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(b"\0")) + + chunk(b"IEND", b"") + ) response = await upload(client, alice, chat, image) assert response.status_code == 413, response.text @@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context): _, alice, bob, _, chat = context attachment = (await upload(client, alice, chat)).json() other_chat = await create_chat(client, alice, bob) - for id in (attachment['id'], str(uuid4())): + for id in (attachment["id"], str(uuid4())): response = await client.post( f"/api/v1/chats/{other_chat['id']}/messages", - json={'text': '', 'attachment_ids': [id]}, + json={"text": "", "attachment_ids": [id]}, headers={**bearer(alice), **idempotency_header()}, ) assert response.status_code == 404 @@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context): async def test_duplicate_ids_are_400(client, context): _, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] - response = await client.post(f"/api/v1/chats/{chat['id']}/messages", - json={'text': '', 'attachment_ids': [id, id]}, - headers={**bearer(alice), **idempotency_header()}) + id = (await upload(client, alice, chat)).json()["id"] + response = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [id, id]}, + headers={**bearer(alice), **idempotency_header()}, + ) assert response.status_code == 400 @@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): _, alice, _, _, chat = context headers = {**bearer(alice), **idempotency_header()} path = f"/api/v1/chats/{chat['id']}/messages" - first = await client.post(path, json={'text': 'same'}, headers=headers) - second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers) + first = await client.post(path, json={"text": "same"}, headers=headers) + second = await client.post( + path, json={"text": "same", "attachment_ids": []}, headers=headers + ) assert first.status_code == second.status_code == 201 assert first.json() == second.json() async def test_s3_outage_is_503(client, context): storage, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] + id = (await upload(client, alice, chat)).json()["id"] await finish_attachment(client, storage, alice, id) storage.down = True - response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) + response = await client.get( + f"/api/v1/attachments/{id}/content", headers=bearer(alice) + ) assert response.status_code == 503 - assert (await client.get('/health/ready')).status_code == 503 + assert (await client.get("/health/ready")).status_code == 503 async def test_uncertain_put_is_compensated(client, context, db_engine): @@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine): async def test_failure_after_s3_write_compensates(client, context, monkeypatch): storage, alice, _, _, chat = context + async def fail(*args, **kwargs): - raise OperationalError('injected', {}, None) - monkeypatch.setattr(repository, 'create_attachment', fail) + raise OperationalError("injected", {}, None) + + monkeypatch.setattr(repository, "create_attachment", fail) response = await upload(client, alice, chat) assert response.status_code == 503 assert not storage.objects -async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch): +async def test_ambiguous_successful_commit_keeps_original( + client, context, db_engine, monkeypatch +): storage, alice, _, _, chat = context original = AsyncSession.commit count = 0 + async def uncertain(session): nonlocal count count += 1 await original(session) if count == 2: - raise OperationalError('lost commit acknowledgement', {}, None) - monkeypatch.setattr(AsyncSession, 'commit', uncertain) + raise OperationalError("lost commit acknowledgement", {}, None) + + monkeypatch.setattr(AsyncSession, "commit", uncertain) response = await upload(client, alice, chat) assert response.status_code == 503 assert len(storage.objects) == 1 @@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_ storage, _, _, _, _ = context old_id, fresh_id = uuid4(), uuid4() async with AsyncSession(db_engine) as session: - session.add_all([ - UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)), - UploadIntent(id=fresh_id, object_key='originals/fresh'), - ]) + session.add_all( + [ + UploadIntent( + id=old_id, + object_key="originals/old", + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ), + UploadIntent(id=fresh_id, object_key="originals/fresh"), + ] + ) await session.commit() async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner: await repository.lock_upload(uploader, old_id) @@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_ async def test_client_disconnect_before_put_has_no_side_effects(client, context): storage, alice, _, _, chat = context + async def interrupted(): yield PNG[:16] raise ClientDisconnect() + response = await upload(client, alice, chat, interrupted()) assert response.status_code == 400 assert not storage.objects @@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context) async def test_upload_deadline_cancels_validation(client, context, monkeypatch): storage, alice, _, _, chat = context - storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02}) + storage.settings = storage.settings.model_copy( + update={"upload_timeout_seconds": 0.02} + ) + async def stalled(*args): await asyncio.sleep(10) - monkeypatch.setattr(domain, 'validate_image', stalled) + + monkeypatch.setattr(domain, "validate_image", stalled) response = await upload(client, alice, chat) assert response.status_code == 400 assert not storage.objects @@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa storage, alice, _, _, chat = context entered = asyncio.Event() original = storage.put_file + async def interrupted(*args, **kwargs): await original(*args, **kwargs) entered.set() await asyncio.sleep(10) - monkeypatch.setattr(storage, 'put_file', interrupted) + + monkeypatch.setattr(storage, "put_file", interrupted) task = asyncio.create_task(upload(client, alice, chat)) await asyncio.wait_for(entered.wait(), timeout=10) task.cancel() @@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): storage, alice, _, _, chat = context - id = (await upload(client, alice, chat)).json()['id'] + id = (await upload(client, alice, chat)).json()["id"] await finish_attachment(client, storage, alice, id) async with AsyncSession(db_engine) as session: attachment = await session.get(Attachment, UUID(id)) key = attachment.object_key # Искусственно оставляем старый журнал у уже подтверждённого объекта. - session.add(UploadIntent(id=uuid4(), object_key=key, - created_at=datetime.now(timezone.utc) - timedelta(days=2))) + session.add( + UploadIntent( + id=uuid4(), + object_key=key, + created_at=datetime.now(timezone.utc) - timedelta(days=2), + ) + ) await session.commit() result = await domain.reconcile_attachments(session, storage, dry_run=False) assert result.examined == 1 and result.deleted == 0 diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py index 66860669..bf07e6f8 100644 --- a/student-tests/test_concurrency.py +++ b/student-tests/test_concurrency.py @@ -8,7 +8,14 @@ from sqlalchemy import text from app.main import create_app from conftest import close_app, make_client -from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register +from helpers import ( + PASSWORD, + basic_header, + bearer, + create_user, + idempotency_header, + register, +) async def test_parallel_registration_of_same_username_creates_exactly_one_user( diff --git a/student-tests/test_history_plan.py b/student-tests/test_history_plan.py index 704c9ccd..b19afcac 100644 --- a/student-tests/test_history_plan.py +++ b/student-tests/test_history_plan.py @@ -29,14 +29,12 @@ async def seed_messages( ) -> None: async with engine.begin() as connection: await connection.execute( - text( - """ + text(""" INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g, now() - make_interval(secs => :count - g) FROM generate_series(1, :count) AS g - """ - ), + """), {"chat_id": chat_id, "sender_id": sender_id, "count": count}, ) await connection.execute(text("ANALYZE messages")) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py index 9455e3d4..7c3aa49d 100644 --- a/student-tests/test_idempotency.py +++ b/student-tests/test_idempotency.py @@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m @pytest.fixture async def replicas(request, make_settings): overrides = getattr(request, "param", {}) - apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")] + apps = [ + create_app(make_settings(instance_id=name, **overrides)) + for name in ("api-a", "api-b") + ] try: async with AsyncExitStack() as stack: - clients = [await stack.enter_async_context(make_client(app)) for app in apps] + clients = [ + await stack.enter_async_context(make_client(app)) for app in apps + ] yield clients finally: for app in apps: @@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine): original = await send_message(first, user, chat, "Привет", "same-key") repeated = await second.post( f"/api/v1/chats/{chat['id']}/messages", - headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"}, + headers={ + **bearer(user), + **idempotency_header("same-key"), + "Content-Type": "application/json", + }, content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }', ) conflict = await send_message(second, user, chat, "другой текст", "same-key") @@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine): assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"] assert conflict.status_code == 409 async with db_engine.connect() as connection: - assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 - assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute( + text("SELECT value FROM counters WHERE name='messages'") + ) + ).scalar_one() == 1 async def test_parallel_replays_create_one_message(replicas, db_engine): user = await create_user(replicas[0]) chat = await create_chat(replicas[0], user) - responses = await asyncio.gather(*( - send_message(replicas[index % 2], user, chat, "parallel", "one-key") - for index in range(24) - )) + responses = await asyncio.gather( + *( + send_message(replicas[index % 2], user, chat, "parallel", "one-key") + for index in range(24) + ) + ) assert all(response.status_code == 201 for response in responses) assert len({response.json()["id"] for response in responses}) == 1 - assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"} + assert {response.headers["X-Instance-Id"] for response in responses} == { + "api-a", + "api-b", + } async with db_engine.connect() as connection: - assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1 - assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM messages")) + ).scalar_one() == 1 + assert ( + await connection.execute(text("SELECT count(*) FROM idempotency_keys")) + ).scalar_one() == 1 async def test_key_scope_separates_users_and_chats(client): @@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng chat = await create_chat(client, user) first = await send_message(client, user, chat, "old", "expiring-key") async with db_engine.begin() as connection: - await connection.execute(text( - "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" - )) + await connection.execute( + text( + "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'" + ) + ) second = await send_message(client, user, chat, "new", "expiring-key") assert first.status_code == second.status_code == 201 assert first.json()["id"] != second.json()["id"] @@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp assert failed.status_code == 503 async with db_engine.connect() as connection: for table in ("messages", "idempotency_keys", "counters"): - assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + assert ( + await connection.execute(text(f"SELECT count(*) FROM {table}")) + ).scalar_one() == 0 monkeypatch.setattr(key_storage, "save_result", original) recovered = await send_message(client, user, chat, "retry", "retry-key") assert recovered.status_code == 201 @@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas): async def test_replay_checks_session_and_membership(client, db_engine): user = await create_user(client) chat = await create_chat(client, user) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 201 async with db_engine.begin() as connection: await connection.execute(text("DELETE FROM chat_members")) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 404 await client.delete("/api/v1/auth/sessions/current", headers=bearer(user)) - assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401 + assert ( + await send_message(client, user, chat, "private", "private-key") + ).status_code == 401 diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py index bbed62f2..f317fa59 100644 --- a/student-tests/test_image_pipeline.py +++ b/student-tests/test_image_pipeline.py @@ -18,54 +18,96 @@ from app.storage.jobs import JobWork from test_attachments import MemoryStorage -@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)]) -async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent): - source=tmp_path/'source.png' - with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image: - exif=Image.Exif();exif[270]='private description' - image.save(source,format='PNG',exif=exif) - result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None)) - variants={v.name:v for v in result.variants} - assert (variants['crop'].width,variants['crop'].height)==(128,128) - assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size) - assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size) - for name in ('thumbnail.jpg','crop.jpg','optimized.webp'): - with Image.open(tmp_path/name) as image: - assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp') +@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)]) +async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent): + source = tmp_path / "source.png" + with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image: + exif = Image.Exif() + exif[270] = "private description" + image.save(source, format="PNG", exif=exif) + result = create_variants( + source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None) + ) + variants = {v.name: v for v in result.variants} + assert (variants["crop"].width, variants["crop"].height) == (128, 128) + assert (variants["thumbnail"].width, variants["thumbnail"].height) == ( + (256, 128) if size[0] > 256 else size + ) + assert (variants["optimized"].width, variants["optimized"].height) == ( + (1280, 640) if size[0] > 1280 else size + ) + for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"): + with Image.open(tmp_path / name) as image: + assert ( + not image.getexif() + and not image.info.get("icc_profile") + and not image.info.get("xmp") + ) if transparent: - if name.endswith('webp'):assert image.getpixel((0,0))[3]==0 - else:assert min(image.getpixel((0,0)))>=250 + if name.endswith("webp"): + assert image.getpixel((0, 0))[3] == 0 + else: + assert min(image.getpixel((0, 0))) >= 250 async def test_exif_orientation_applied(tmp_path): - source=tmp_path/'source.jpg' - with Image.new('RGB',(400,200),'blue') as image: - exif=Image.Exif();exif[274]=6 - image.save(source,exif=exif) - result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None)) - assert (result.variants[0].width,result.variants[0].height)==(128,256) + source = tmp_path / "source.jpg" + with Image.new("RGB", (400, 200), "blue") as image: + exif = Image.Exif() + exif[274] = 6 + image.save(source, exif=exif) + result = create_variants( + source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None) + ) + assert (result.variants[0].width, result.variants[0].height) == (128, 256) -@pytest.mark.parametrize('failure', ['memory','cpu']) -async def test_worker_os_limits(failure,tmp_path): - body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass' - code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())" - result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'], - env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15) - assert result.returncode==(3 if failure=='memory' else -9) - assert not (tmp_path/'metadata').exists() +@pytest.mark.parametrize("failure", ["memory", "cpu"]) +async def test_worker_os_limits(failure, tmp_path): + body = ( + "return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass" + ) + code = ( + "from app.core import thumbnail_worker as w\ndef fail(*args):\n " + + body + + "\nw.create_variants=fail\nraise SystemExit(w.main())" + ) + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-c", + code, + "source", + str(tmp_path), + str(tmp_path / "metadata"), + "image/png", + "40000000", + ], + env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"}, + capture_output=True, + timeout=15, + ) + assert result.returncode == (3 if failure == "memory" else -9) + assert not (tmp_path / "metadata").exists() -async def test_render_timeout_kills_child(monkeypatch,tmp_path): - create=asyncio.create_subprocess_exec - children=[] - async def slow_child(*args,**kwargs): - child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs) +async def test_render_timeout_kills_child(monkeypatch, tmp_path): + create = asyncio.create_subprocess_exec + children = [] + + async def slow_child(*args, **kwargs): + child = await create( + sys.executable, "-c", "import time;time.sleep(60)", **kwargs + ) children.append(child) return child - monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child) - work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child) + work = JobWork( + uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64 + ) with pytest.raises(TimeoutError): - async with asyncio.timeout(.2): - await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path) - assert len(children)==1 and children[0].returncode is not None + async with asyncio.timeout(0.2): + await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path) + assert len(children) == 1 and children[0].returncode is not None diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py index 8b0d932b..1f262474 100644 --- a/student-tests/test_instance.py +++ b/student-tests/test_instance.py @@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors( assert response.headers["X-Instance-Id"] == "api-test" assert response.headers["X-Request-Id"] if status >= 400: - assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + assert ( + response.json()["error"]["request_id"] == response.headers["X-Request-Id"] + ) async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client): @@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli second = await instance_client.get("/__test__/request-id") assert first.status_code == second.status_code == 200 - assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + assert ( + first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test" + ) assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"] for response in (first, second): assert response.json()["request_id"] == response.headers["X-Request-Id"] diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py index 4a08bc26..3582a09a 100644 --- a/student-tests/test_jobs.py +++ b/student-tests/test_jobs.py @@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.jobs import JobSettings from app.domain import attachment_jobs -from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result +from app.domain.attachment_jobs import ( + claim_job, + process_attachment_job, + cleanup_staging, + build_result, +) from app.domain.job_admin import replay_failed_job from app.domain.job_retry import classify_failure, retry_delay from app.errors.exceptions import JobOperationError, ProcessingError @@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment async def envelope(session, attachment_id): - job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) - return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1, - attempt=job.attempt + 1, created_at=job.created_at) + job = await session.scalar( + select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)) + ) + return JobMessage( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=1, + attempt=job.attempt + 1, + created_at=job.created_at, + ) async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() settings = JobSettings(_env_file=None, job_max_attempts=3) + def fail(_): raise TimeoutError + async with AsyncSession(db_engine, expire_on_commit=False) as session: - first = await envelope(session, attachment['id']) + first = await envelope(session, attachment["id"]) for attempt in range(1, 4): - message = await envelope(session, attachment['id']) - assert await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + assert await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, first.job_id) assert job.attempt == attempt delay = (job.next_attempt_at - item.updated_at).total_seconds() assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1) - assert item.status == ('failed' if attempt == 3 else 'queued') + assert item.status == ("failed" if attempt == 3 else "queued") assert job.last_error_permanent is False - job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1) + job.next_attempt_at = await jobs.database_time(session) - timedelta( + seconds=1 + ) await session.commit() - assert item.error_code == 'retry_exhausted' + assert item.error_code == "retry_exhausted" request_id = uuid4() - result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, - actor='test', reason='dependency_restored') - assert result['attempt_limit'] == 6 + result = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert result["attempt_limit"] == 6 # Old envelope cannot consume the fresh replay budget. assert await process_attachment_job(session, storage, first, settings) job, item = await jobs.lock_job(session, first.job_id) - assert job.attempt == 3 and item.status == 'queued' + assert job.attempt == 3 and item.status == "queued" await session.commit() - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) assert await process_attachment_job(session, storage, message, settings) before = dict(storage.objects) assert await process_attachment_job(session, storage, message, settings) assert storage.objects == before - repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, - actor='test', reason='dependency_restored') - assert repeat['already_applied'] - with pytest.raises(JobOperationError, match='job_not_failed'): - await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review') + repeat = await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=request_id, + actor="test", + reason="dependency_restored", + ) + assert repeat["already_applied"] + with pytest.raises(JobOperationError, match="job_not_failed"): + await replay_failed_job( + session, + storage, + settings, + first.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) await session.rollback() assert len((await session.scalars(select(JobReplay))).all()) == 1 @@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng async def test_permanent_failure_and_missing_source_replay(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + def fail(_): - raise ProcessingError('image_limits') + raise ProcessingError("image_limits") + settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) - await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, message.job_id) - assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent + assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent job.staging_key = None await session.commit() - with pytest.raises(JobOperationError, match='source_expired'): - await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review') + with pytest.raises(JobOperationError, match="source_expired"): + await replay_failed_job( + session, + storage, + settings, + job.id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) -async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine): +async def test_stale_worker_cannot_write_and_cleanup_keeps_active( + client, context, db_engine +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) claim = await claim_job(session, message, settings) job, item = await jobs.lock_job(session, message.job_id) job.lease_token = uuid4() @@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex async def test_cleanup_ready_preserves_final_objects(client, context, db_engine): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() - await finish_attachment(client, storage, alice, attachment['id']) + await finish_attachment(client, storage, alice, attachment["id"]) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) job, item = await jobs.lock_job(session, message.job_id) staging = job.staging_key job.finished_at = await jobs.database_time(session) - timedelta(days=2) await session.commit() - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=False + ) + == 1 + ) assert staging in storage.objects - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 1 + ) assert staging not in storage.objects and len(storage.objects) == 4 - assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0 + assert ( + await cleanup_staging( + session, storage, JobSettings(_env_file=None), apply=True + ) + == 0 + ) async def test_retry_classification_and_cap(): @@ -131,44 +200,64 @@ async def test_retry_classification_and_cap(): for attempt in (1, 2, 6, 100): cap = min(120, 5 * 2 ** min(30, attempt - 1)) assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100)) - assert classify_failure(TimeoutError()) == ('processing_timeout', False) - assert classify_failure(OSError(errno.EACCES, 'denied'))[1] - assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1] + assert classify_failure(TimeoutError()) == ("processing_timeout", False) + assert classify_failure(OSError(errno.EACCES, "denied"))[1] + assert not classify_failure(OSError(errno.ENOSPC, "full"))[1] -async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch): +async def test_timeout_records_retry_and_clears_lease( + client, context, db_engine, monkeypatch +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + async def stalled(*args, **kwargs): await asyncio.sleep(60) - monkeypatch.setattr(attachment_jobs, 'build_result', stalled) - settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05}) + + monkeypatch.setattr(attachment_jobs, "build_result", stalled) + settings = JobSettings(_env_file=None).model_copy( + update={"job_timeout_seconds": 0.05} + ) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) + message = await envelope(session, attachment["id"]) assert await process_attachment_job(session, storage, message, settings) job, item = await jobs.lock_job(session, message.job_id) - assert item.status == 'queued' and job.last_error_code == 'processing_timeout' + assert item.status == "queued" and job.last_error_code == "processing_timeout" assert job.lease_token is None and job.lease_until is None -async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine): +async def test_failed_retention_and_source_checksum_protect_replay( + client, context, db_engine +): storage, alice, _, _, chat = context attachment = (await upload(client, alice, chat)).json() + def fail(_): - raise ProcessingError('invalid_image') + raise ProcessingError("invalid_image") + settings = JobSettings(_env_file=None) async with AsyncSession(db_engine, expire_on_commit=False) as session: - message = await envelope(session, attachment['id']) - await process_attachment_job(session, storage, message, settings, before_processing=fail) + message = await envelope(session, attachment["id"]) + await process_attachment_job( + session, storage, message, settings, before_processing=fail + ) job, item = await jobs.lock_job(session, message.job_id) key = job.staging_key job.finished_at = await jobs.database_time(session) - timedelta(days=2) await session.commit() assert await cleanup_staging(session, storage, settings, apply=True) == 0 data, metadata = storage.objects[key] - storage.objects[key] = (b'x' * len(data), metadata) - with pytest.raises(JobOperationError, match='source_mismatch'): - await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review') + storage.objects[key] = (b"x" * len(data), metadata) + with pytest.raises(JobOperationError, match="source_mismatch"): + await replay_failed_job( + session, + storage, + settings, + message.job_id, + request_id=uuid4(), + actor="test", + reason="manual_review", + ) await session.rollback() assert not (await session.scalars(select(JobReplay))).all() job, item = await jobs.lock_job(session, message.job_id) diff --git a/student-tests/test_lifecycle.py b/student-tests/test_lifecycle.py index 1a659b74..11e4dcb1 100644 --- a/student-tests/test_lifecycle.py +++ b/student-tests/test_lifecycle.py @@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings): app.state.lifecycle.start_draining() try: async with make_client(app) as client: - assert (await client.get('/health/live')).status_code == 200 - assert (await client.get('/health/ready')).status_code == 503 - assert (await client.get('/api/v1/users/me')).status_code == 503 + assert (await client.get("/health/live")).status_code == 200 + assert (await client.get("/health/ready")).status_code == 503 + assert (await client.get("/api/v1/users/me")).status_code == 503 finally: await close_app(app) @@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey finally: cancelled.append(True) - monkeypatch.setattr(health, 'check_database', stalled) - monkeypatch.setattr(health, 'check_redis', stalled) + monkeypatch.setattr(health, "check_database", stalled) + monkeypatch.setattr(health, "check_redis", stalled) app = create_app(make_settings(readiness_timeout_seconds=0.03)) try: async with make_client(app) as client: started = time.monotonic() - response = await client.get('/health/ready') + response = await client.get("/health/ready") assert response.status_code == 503 assert time.monotonic() - started < 1 assert len(cancelled) == 2 diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py index 3bd68c0d..9cfca1d3 100644 --- a/student-tests/test_limits.py +++ b/student-tests/test_limits.py @@ -6,7 +6,13 @@ import httpx from app.main import create_app from conftest import close_app, make_client -from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header +from helpers import ( + assert_request_id, + bearer, + create_chat, + create_user, + idempotency_header, +) def _assert_error(response: httpx.Response, status: int, code: str) -> None: @@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se for index in range(2): ok = await client.post( - url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"} + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"confirmed-{index}"}, ) assert ok.status_code == 201 rejected = await client.post( - url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"} + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "rejected"}, ) _assert_error(rejected, 503, "unavailable") @@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat user = await create_user(client) chat = await create_chat(client, user) - monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message) + monkeypatch.setattr( + message_endpoint, "send_message_idempotent", blocking_send_message + ) url = f"/api/v1/chats/{chat['id']}/messages" held = [ asyncio.create_task( - client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"}) + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"held-{i}"}, + ) ) for i in range(2) ] @@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat rejected = await asyncio.gather( *( - client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"}) + client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": f"rejected-{i}"}, + ) for i in range(4) ) ) @@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat release.set() assert all(r.status_code == 201 for r in await asyncio.gather(*held)) - recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"}) + recovered = await client.post( + url, + headers={**bearer(user), **idempotency_header()}, + json={"text": "after"}, + ) assert recovered.status_code == 201 assert_request_id(recovered) await close_app(app) diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py index f8efed1b..c515ade1 100644 --- a/student-tests/test_migrations.py +++ b/student-tests/test_migrations.py @@ -8,7 +8,19 @@ from sqlalchemy import inspect, text from app.models import Base from conftest import reset_schema, upgrade_head -EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"} +EXPECTED_TABLES = { + "users", + "chats", + "chat_members", + "messages", + "counters", + "idempotency_keys", + "attachments", + "message_attachments", + "upload_intents", + "attachment_jobs", + "job_replays", +} @pytest.fixture @@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine): async with migrated_engine.connect() as connection: diff = await connection.run_sync(compare) - assert diff == [], ( - f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" - ) + assert ( + diff == [] + ), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}" diff --git a/student-tests/test_s3_stream.py b/student-tests/test_s3_stream.py index b168a626..67fc364b 100644 --- a/student-tests/test_s3_stream.py +++ b/student-tests/test_s3_stream.py @@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync async def test_sdk_thread_finishes_before_cancellation_returns(): entered, release, finished = Event(), Event(), Event() + def operation(): entered.set() release.wait(timeout=3) finished.set() + task = asyncio.create_task(run_sync(operation)) await asyncio.to_thread(entered.wait, 2) task.cancel() @@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled(): entered, release = Event(), Event() discarded = [] response = object() + def operation(): entered.set() release.wait(timeout=3) return response + task = asyncio.create_task(run_sync(operation, discard_result=discarded.append)) await asyncio.to_thread(entered.wait, 2) task.cancel() @@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled(): async def test_stream_closes_even_before_first_iteration(): calls = [] - response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release')) + response = SimpleNamespace( + close=lambda: calls.append("close"), + release_conn=lambda: calls.append("release"), + ) storage = SimpleNamespace(settings=S3Settings(_env_file=None)) stream = ObjectStream(storage, response) await stream.aclose() await stream.aclose() - assert calls == ['close', 'release'] + assert calls == ["close", "release"] diff --git a/tests/client.py b/tests/client.py index f0499a29..c9ca1ae9 100644 --- a/tests/client.py +++ b/tests/client.py @@ -1,4 +1,5 @@ """HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" + import base64 import datetime as dt import hashlib @@ -13,205 +14,336 @@ import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] -LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +SPEC = json.loads((ROOT / "contracts/openapi.json").read_text()) + class Failure(AssertionError): pass + def check(condition, message): if not condition: raise Failure(message) -def timestamp(value): - check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z') - try: - return dt.datetime.fromisoformat(value[:-1] + '+00:00') - except ValueError as exc: - raise Failure('invalid RFC3339 timestamp') from exc -def validate(value, schema, where='response'): +def timestamp(value): + check( + isinstance(value, str) and value.endswith("Z"), + "timestamp must be UTC ending in Z", + ) + try: + return dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise Failure("invalid RFC3339 timestamp") from exc + + +def validate(value, schema, where="response"): """Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI.""" - if '$ref' in schema: + if "$ref" in schema: target = SPEC - for part in schema['$ref'].split('/')[1:]: + for part in schema["$ref"].split("/")[1:]: target = target[part] return validate(value, target, where) - kinds = schema.get('type', []) + kinds = schema.get("type", []) if isinstance(kinds, str): kinds = [kinds] - matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)} + matches = { + "null": value is None, + "boolean": isinstance(value, bool), + "integer": isinstance(value, int) and not isinstance(value, bool), + "number": isinstance(value, (int, float)) and not isinstance(value, bool), + "string": isinstance(value, str), + "array": isinstance(value, list), + "object": isinstance(value, dict), + } if kinds: - check(any(matches.get(k,False) for k in kinds), where + ': incorrect type') - if 'enum' in schema: - check(value in schema['enum'], where + ': invalid enum') - if 'const' in schema: - check(value == schema['const'], where + ': invalid const') - if isinstance(value,dict): - props = schema.get('properties',{}) - check(all(k in value for k in schema.get('required',[])), where + ': missing required fields') - if schema.get('additionalProperties') is False: - check(not set(value).difference(props), where + ': unexpected fields') + check(any(matches.get(k, False) for k in kinds), where + ": incorrect type") + if "enum" in schema: + check(value in schema["enum"], where + ": invalid enum") + if "const" in schema: + check(value == schema["const"], where + ": invalid const") + if isinstance(value, dict): + props = schema.get("properties", {}) + check( + all(k in value for k in schema.get("required", [])), + where + ": missing required fields", + ) + if schema.get("additionalProperties") is False: + check(not set(value).difference(props), where + ": unexpected fields") for key, sub in props.items(): if key in value: - validate(value[key], sub, where + '.' + key) - if isinstance(value,list): - check(len(value) >= schema.get('minItems',0), where + ': too few items') - check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items') - if schema.get('uniqueItems'): - encoded = [json.dumps(x,sort_keys=True) for x in value] - check(len(encoded)==len(set(encoded)), where + ': duplicate items') - if 'items' in schema: + validate(value[key], sub, where + "." + key) + if isinstance(value, list): + check(len(value) >= schema.get("minItems", 0), where + ": too few items") + check( + len(value) <= schema.get("maxItems", float("inf")), + where + ": too many items", + ) + if schema.get("uniqueItems"): + encoded = [json.dumps(x, sort_keys=True) for x in value] + check(len(encoded) == len(set(encoded)), where + ": duplicate items") + if "items" in schema: for item in value: - validate(item, schema['items'], where + '[]') - if isinstance(value,str): - check(len(value) >= schema.get('minLength',0), where + ': too short') - check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long') - if 'pattern' in schema: - check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch') - if schema.get('format') == 'uuid': + validate(item, schema["items"], where + "[]") + if isinstance(value, str): + check(len(value) >= schema.get("minLength", 0), where + ": too short") + check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long") + if "pattern" in schema: + check( + re.search(schema["pattern"], value) is not None, + where + ": pattern mismatch", + ) + if schema.get("format") == "uuid": try: uuid.UUID(value) except ValueError as exc: - raise Failure(where + ': invalid UUID') from exc - if schema.get('format') == 'date-time': + raise Failure(where + ": invalid UUID") from exc + if schema.get("format") == "date-time": timestamp(value) - if isinstance(value,(int,float)) and not isinstance(value,bool): - check(value >= schema.get('minimum',-float('inf')), where + ': below minimum') - check(value <= schema.get('maximum',float('inf')), where + ': above maximum') - if 'anyOf' in schema: - for alternative in schema['anyOf']: + if isinstance(value, (int, float)) and not isinstance(value, bool): + check(value >= schema.get("minimum", -float("inf")), where + ": below minimum") + check(value <= schema.get("maximum", float("inf")), where + ": above maximum") + if "anyOf" in schema: + for alternative in schema["anyOf"]: try: validate(value, alternative, where) break except Failure: pass else: - raise Failure(where + ': no anyOf branch matches') + raise Failure(where + ": no anyOf branch matches") + def shape(value, name): - validate(value, SPEC['components']['schemas'][name]) + validate(value, SPEC["components"]["schemas"][name]) return value + class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None + class Client: def __init__(self, base_url=None, ca_file=None, timeout=10): - self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') - check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') - check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') + self.base = ( + base_url + or os.environ.get("BASE_URL") + or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080") + ).rstrip("/") + check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)") + check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS") self.timeout = timeout - context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) - self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context)) + context = ssl.create_default_context( + cafile=ca_file or os.environ.get("CA_FILE") or None + ) + self.opener = urllib.request.build_opener( + NoRedirect(), urllib.request.HTTPSHandler(context=context) + ) - def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None): - h = {'Accept':'application/json'} + def call( + self, + method, + path, + expected=200, + body=None, + actor=None, + headers=None, + raw=None, + schema=None, + ): + h = {"Accept": "application/json"} if actor: - h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token'] + h["X-User-Id" if LAB == 1 else "Authorization"] = ( + actor["id"] if LAB == 1 else "Bearer " + actor["token"] + ) if body is not None: - raw = json.dumps(body,ensure_ascii=False).encode('utf-8') - h['Content-Type'] = 'application/json' + raw = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" h.update(headers or {}) - req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method) + req = urllib.request.Request( + self.base + path, data=raw, headers=h, method=method + ) start = time.monotonic() try: response = self.opener.open(req, timeout=self.timeout) except urllib.error.HTTPError as error: response = error - except (urllib.error.URLError,TimeoutError,OSError) as exc: + except (urllib.error.URLError, TimeoutError, OSError) as exc: # Do not print URL query, headers, request body, or credentials. - raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None + raise Failure( + method + + " " + + path.split("?")[0] + + ": connection/TLS failure (" + + type(exc).__name__ + + ")" + ) from None with response: status = response.code rh = response.headers - data = response.read(16*1024*1024 + 1) - check(len(data) <= 16*1024*1024, 'response exceeds test safety limit') - allowed = [expected] if isinstance(expected,int) else expected - check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) - check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id') - if LAB>=3: - check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id') + data = response.read(16 * 1024 * 1024 + 1) + check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit") + allowed = [expected] if isinstance(expected, int) else expected + check( + status in allowed, + method + + " " + + path.split("?")[0] + + ": expected " + + str(allowed) + + ", received " + + str(status), + ) + check(bool(rh.get("X-Request-Id")), "missing X-Request-Id") + if LAB >= 3: + check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id") payload = None - if schema or status>=400: - check(rh.get_content_type()=='application/json', 'expected application/json') + if schema or status >= 400: + check( + rh.get_content_type() == "application/json", "expected application/json" + ) try: payload = json.loads(data) - except (ValueError,UnicodeError): - raise Failure('invalid response JSON') from None - shape(payload, 'Error' if status>=400 else schema) - if status>=400: - check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header') - if status==401 and LAB>=2: - scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer' - check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge') - if status==204: - check(not data, '204 must not contain body') - return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start} + except (ValueError, UnicodeError): + raise Failure("invalid response JSON") from None + shape(payload, "Error" if status >= 400 else schema) + if status >= 400: + check( + payload["error"]["request_id"] == rh.get("X-Request-Id"), + "error request_id differs from header", + ) + if status == 401 and LAB >= 2: + scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer" + check( + rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()), + "incorrect WWW-Authenticate challenge", + ) + if status == 204: + check(not data, "204 must not contain body") + return { + "status": status, + "headers": rh, + "json": payload, + "bytes": data, + "elapsed": time.monotonic() - start, + } def user(self, label): - username = 'u_' + label + '_' + uuid.uuid4().hex[:12] - password = 'T3st-' + uuid.uuid4().hex - body = {'username':username,'display_name':'Студент ' + label} - if LAB>=2: - body['password'] = password - user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json'] - check(user['username']==username,'username changed') - actor = {'id':user['id'],'user':user,'registration':body} - if LAB>=2: - credential = base64.b64encode((username+':'+password).encode()).decode() - session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json'] - check(session['user']==user,'session user mismatch') - check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired') - actor['token'] = session['token'] + username = "u_" + label + "_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": "Студент " + label} + if LAB >= 2: + body["password"] = password + user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["username"] == username, "username changed") + actor = {"id": user["id"], "user": user, "registration": body} + if LAB >= 2: + credential = base64.b64encode((username + ":" + password).encode()).decode() + session = self.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + credential}, + schema="Session", + )["json"] + check(session["user"] == user, "session user mismatch") + check( + timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc), + "session already expired", + ) + actor["token"] = session["token"] return actor def bootstrap(self): - self.call('GET','/health/live',schema='Health') - self.call('GET','/health/ready',schema='Health') - alice, bob, eve = (self.user(label) for label in ('alice','bob','eve')) - chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] - check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch') - return alice,bob,eve,chat + self.call("GET", "/health/live", schema="Health") + self.call("GET", "/health/ready", schema="Health") + alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve")) + chat = self.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + check( + set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch" + ) + return alice, bob, eve, chat def message(self, actor, chat, text, key=None, attachments=None): - body = {'text':text} + body = {"text": text} if attachments is not None: - body['attachment_ids'] = attachments - headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {} - return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + body["attachment_ids"] = attachments + headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {} + return self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + actor=actor, + body=body, + headers=headers, + schema="Message", + ) def attachment(self, actor, chat): - content = (ROOT / 'tests/fixtures/sample.png').read_bytes() - att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json'] - check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status') - check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch') - check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch') - return att,content + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + att = self.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 201 if LAB == 4 else 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + check( + att["status"] == ("ready" if LAB == 4 else "queued"), + "incorrect upload status", + ) + check( + att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"], + "attachment ownership mismatch", + ) + check( + att["sha256"] == hashlib.sha256(content).hexdigest() + and att["size_bytes"] == len(content), + "original checksum/size mismatch", + ) + return att, content def wait_ready(self, actor, att, wait_seconds=60): - deadline = time.monotonic()+wait_seconds - while time.monotonic()=2 and i+length<=len(data),'invalid JPEG segment length') + check(i + 2 <= len(data), "truncated JPEG segment") + length = int.from_bytes(data[i : i + 2], "big") + check(length >= 2 and i + length <= len(data), "invalid JPEG segment length") if marker in sof: - check(length>=8,'invalid JPEG SOF') - return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big')) + check(length >= 8, "invalid JPEG SOF") + return ( + int.from_bytes(data[i + 5 : i + 7], "big"), + int.from_bytes(data[i + 3 : i + 5], "big"), + ) i += length - raise Failure('JPEG dimensions not found') + raise Failure("JPEG dimensions not found") diff --git a/tests/load.py b/tests/load.py index ad81d596..8d022bad 100644 --- a/tests/load.py +++ b/tests/load.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Small closed-loop HTTP write benchmark; not an open-loop capacity proof.""" + from concurrent.futures import ThreadPoolExecutor import json import math @@ -11,48 +12,78 @@ from client import Client, Failure, LAB, arguments, check, run def main(): def configure(p): - p.add_argument('--duration',type=float,default=30) - p.add_argument('--concurrency',type=int,default=4) - p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens') - args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure) - check(1<=args.duration<=3600,'duration must be 1..3600 seconds') - check(1<=args.concurrency<=128,'concurrency must be 1..128') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() + p.add_argument("--duration", type=float, default=30) + p.add_argument("--concurrency", type=int, default=4) + p.add_argument( + "--output", default=None, help="Optional JSON metrics, no session tokens" + ) + + args = arguments( + "Synthetic POST message load, closed loop; creates users/messages", configure + ) + check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds") + check(1 <= args.concurrency <= 128, "concurrency must be 1..128") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() barrier = threading.Barrier(args.concurrency) + def worker(worker_id): - local = Client(args.base_url,args.ca_file,timeout=5) + local = Client(args.base_url, args.ca_file, timeout=5) barrier.wait() - deadline = time.monotonic()+args.duration - latencies,errors = [],0 + deadline = time.monotonic() + args.duration + latencies, errors = [], 0 index = 0 - while time.monotonic()=2,'persistence requires lab2+') - check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + p.add_argument("--mode", choices=["persistence", "failover"], required=True) + p.add_argument( + "--action-script", + required=True, + help="Executable script authored by student; restarts stack or stops ONE API replica", + ) + + args = arguments( + "Explicit restart/failover scenario; modifies only synthetic API data", + configure, + ) + check(LAB >= 2, "persistence requires lab2+") + check(args.mode != "failover" or LAB >= 3, "failover requires lab3+") script = Path(args.action_script).expanduser().resolve() - check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') - check('.example.' not in script.name,'copy and implement the example hook first') - c = Client(args.base_url,args.ca_file,timeout=2) - alice,bob,eve,chat = c.bootstrap() - sent = c.message(alice,chat,'Сохранить при отказе') - original = sent['json'] - target = sent['headers'].get('X-Instance-Id','') + check( + script.is_file() and os.access(script, os.X_OK), + "action script must exist and be executable (chmod +x)", + ) + check(".example." not in script.name, "copy and implement the example hook first") + c = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, eve, chat = c.bootstrap() + sent = c.message(alice, chat, "Сохранить при отказе") + original = sent["json"] + target = sent["headers"].get("X-Instance-Id", "") attachment = content = None - if LAB>=4: - attachment,content = c.attachment(alice,chat) - if LAB>=5: - attachment = c.wait_ready(alice,attachment) + if LAB >= 4: + attachment, content = c.attachment(alice, chat) + if LAB >= 5: + attachment = c.wait_ready(alice, attachment) env = dict(os.environ) - env['TARGET_INSTANCE_ID'] = target + env["TARGET_INSTANCE_ID"] = target observed = set() - if args.mode=='failover': + if args.mode == "failover": for _ in range(40): - observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) - check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') - print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') + observed.add( + c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[ + "headers" + ]["X-Instance-Id"] + ) + check( + len(observed) >= 2, + "less than two API instances observed before failure; verify LB/session affinity", + ) + print( + "Executing explicit " + + args.mode + + " hook; hook output suppressed to avoid leaking secrets." + ) # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + proc = subprocess.Popen( + [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) started = time.monotonic() - successes,failures,after_instances = [],0,[] + successes, failures, after_instances = [], 0, [] try: - if args.mode=='persistence': + if args.mode == "persistence": try: code = proc.wait(timeout=90) except subprocess.TimeoutExpired: - raise Failure('restart hook exceeded 90 seconds') from None - check(code==0,'restart hook failed (inspect your script locally)') - deadline = time.monotonic()+90 + raise Failure("restart hook exceeded 90 seconds") from None + check(code == 0, "restart hook failed (inspect your script locally)") + deadline = time.monotonic() + 90 while True: try: - c.call('GET','/health/ready',schema='Health') - c.call('GET','/api/v1/users/me',actor=alice,schema='User') + c.call("GET", "/health/ready", schema="Health") + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") break except Failure: - check(time.monotonic()=5,'too few successful writes after stop') - check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') - print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + check(proc.poll() == 0, "stop-one hook did not finish successfully") + check( + first_recovery is not None and first_recovery <= 10, + "writes did not recover within 10 seconds from hook start", + ) + check(len(after_instances) >= 5, "too few successful writes after stop") + check( + target not in after_instances[-5:], + "target replica still serves requests; hook must leave it stopped", + ) + print( + json.dumps( + { + "observed_instances_before": len(observed), + "successful_writes": len(successes), + "failed_attempts": failures, + "first_recovery_seconds": round(first_recovery, 3), + }, + ensure_ascii=False, + ) + ) # Existing token, no re-login. Read all confirmations and reject duplicates. - messages = history(c,bob,chat) - ids = [m['id'] for m in messages] - check(len(ids)==len(set(ids)),'duplicated message IDs in history') - check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + messages = history(c, bob, chat) + ids = [m["id"] for m in messages] + check(len(ids) == len(set(ids)), "duplicated message IDs in history") + check( + any(m == original for m in messages), + "confirmed pre-failure message lost or altered", + ) for msg in successes: - check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') + check( + any(m == msg for m in messages), + "confirmed in-flight message lost or altered", + ) # Bodies are unique in this scenario: an uncertain request may exist once, never twice. - texts = [m['text'] for m in messages] - check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + texts = [m["text"] for m in messages] + check( + len(texts) == len(set(texts)), "uncertain retry created duplicate message" + ) if attachment: - c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') - downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) - check(downloaded['bytes']==content,'attachment lost or changed across failure') - print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + c.call( + "GET", + "/api/v1/attachments/" + attachment["id"], + actor=bob, + schema="Attachment", + ) + downloaded = c.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob + ) + check( + downloaded["bytes"] == content, + "attachment lost or changed across failure", + ) + print( + "PASS: " + + args.mode + + " preserved confirmed data and existing session. Hook scope needs operator evidence." + ) finally: if proc.poll() is None: proc.terminate() @@ -125,5 +202,6 @@ def main(): proc.kill() proc.wait() -if __name__=='__main__': + +if __name__ == "__main__": run(main) diff --git a/tests/smoke.py b/tests/smoke.py index 4cbfa558..19db4747 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """One complete public acceptance scenario, cumulative by lab number.""" + import base64 import hashlib import uuid @@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp def main(): - args = arguments('Messenger public smoke; creates synthetic users and messages') - c = Client(args.base_url,args.ca_file) - alice,bob,eve,chat = c.bootstrap() - path = '/api/v1/chats/'+chat['id'] - c.call('POST','/api/v1/users',409,body=alice['registration']) - c.call('GET','/api/v1/users/me',401) - c.call('GET',path,404,actor=eve) - me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json'] - check(me==alice['user'],'current user mismatch') - listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json'] - check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') - hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] - check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + args = arguments("Messenger public smoke; creates synthetic users and messages") + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + c.call("POST", "/api/v1/users", 409, body=alice["registration"]) + c.call("GET", "/api/v1/users/me", 401) + c.call("GET", path, 404, actor=eve) + me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + check(me == alice["user"], "current user mismatch") + listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[ + "json" + ] + check( + any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat" + ) + hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[ + "json" + ] + check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider") key = uuid.uuid4().hex - first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] - second = c.message(bob,chat,'Сообщение Боба')['json'] - check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor') - check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message') - check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') - page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] - check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') + first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"] + second = c.message(bob, chat, "Сообщение Боба")["json"] + check( + first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"], + "sender was not derived from actor", + ) + check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message") + check( + first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба", + "message text changed", + ) + page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[ + "json" + ] + check( + len(page1["items"]) == 1 and page1["next_cursor"], + "first page must have one item and cursor", + ) from urllib.parse import quote - page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] - check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') - ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int)) - check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message') - h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {} - c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h) - c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h) - c.call('GET',path+'/messages?limit=0',400,actor=alice) - if LAB>=2: - bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode() - c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad}) - good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode() - c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good}) - c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']}) - me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json'] - check(me['id']==alice['id'],'X-User-Id overrode bearer identity') - if LAB>=3: - replay = c.message(alice,chat,first['text'],key=key)['json'] - check(replay==first,'idempotency replay differs') - c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key}) - c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'}) - if LAB>=4: - att,content = c.attachment(alice,chat) - # A queued attachment must already be linkable to its own chat. - attached = c.message(alice,chat,'',attachments=[att['id']])['json'] - check(attached['attachment_ids']==[att['id']],'message lost attachment') - ready = c.wait_ready(bob,att) if LAB>=5 else att - check(ready['error_code'] is None,'ready attachment has an error') - download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) - check(download['bytes']==content,'download differs from original') - check(download['headers'].get_content_type()=='image/png','original content type mismatch') - for suffix in ('','/content'): - c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) - if LAB>=5: - variants = [x for x in ready['variants'] if x['name']=='thumbnail'] - check(len(variants)==1,'expected exactly one thumbnail') - v = variants[0] - check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions') - thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob) - check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch') - check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch') - from image_probe import jpeg_size - check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch') - if LAB>=2: - c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) - c.call('GET','/api/v1/users/me',401,actor=alice) - c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice) - print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB) -if __name__=='__main__': + page2 = c.call( + "GET", + path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""), + actor=alice, + schema="MessagePage", + )["json"] + check( + len(page2["items"]) == 1 and page2["next_cursor"] is None, + "second page must finish history", + ) + ordered = sorted( + [first, second], + key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int), + ) + check( + page1["items"] + page2["items"] == ordered, + "pagination lost, duplicated or reordered a message", + ) + h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {} + c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h) + c.call( + "POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h + ) + c.call("GET", path + "/messages?limit=0", 400, actor=alice) + if LAB >= 2: + bad = base64.b64encode( + (alice["registration"]["username"] + ":wrong-password").encode() + ).decode() + c.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + bad}, + ) + good = base64.b64encode( + ( + alice["registration"]["username"] + + ":" + + alice["registration"]["password"] + ).encode() + ).decode() + c.call( + "GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good} + ) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + me = c.call( + "GET", + "/api/v1/users/me", + actor=alice, + headers={"X-User-Id": eve["id"]}, + schema="User", + )["json"] + check(me["id"] == alice["id"], "X-User-Id overrode bearer identity") + if LAB >= 3: + replay = c.message(alice, chat, first["text"], key=key)["json"] + check(replay == first, "idempotency replay differs") + c.call( + "POST", + path + "/messages", + 409, + actor=alice, + body={"text": "другой body"}, + headers={"Idempotency-Key": key}, + ) + c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"}) + if LAB >= 4: + att, content = c.attachment(alice, chat) + # A queued attachment must already be linkable to its own chat. + attached = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(attached["attachment_ids"] == [att["id"]], "message lost attachment") + ready = c.wait_ready(bob, att) if LAB >= 5 else att + check(ready["error_code"] is None, "ready attachment has an error") + download = c.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob + ) + check(download["bytes"] == content, "download differs from original") + check( + download["headers"].get_content_type() == "image/png", + "original content type mismatch", + ) + for suffix in ("", "/content"): + c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve) + if LAB >= 5: + variants = [x for x in ready["variants"] if x["name"] == "thumbnail"] + check(len(variants) == 1, "expected exactly one thumbnail") + v = variants[0] + check( + (v["width"], v["height"]) == (256, 160), + "thumbnail metadata has wrong dimensions", + ) + thumb = c.call( + "GET", + "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", + actor=bob, + ) + check( + thumb["headers"].get_content_type() == "image/jpeg", + "thumbnail content type mismatch", + ) + check( + len(thumb["bytes"]) == v["size_bytes"] + and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"], + "thumbnail checksum mismatch", + ) + from image_probe import jpeg_size + + check( + jpeg_size(thumb["bytes"]) == (256, 160), + "actual JPEG dimensions mismatch", + ) + if LAB >= 2: + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("GET", "/api/v1/users/me", 401, actor=alice) + c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice) + print( + "PASS: lab%d public API scenario; infrastructure and grading evidence remain separate." + % LAB + ) + + +if __name__ == "__main__": run(main) -- 2.54.0 From 2639c089449cc5a2f3673597bfeb2d530e023bee Mon Sep 17 00:00:00 2001 From: gytall Date: Thu, 12 Nov 2026 22:50:05 +0300 Subject: [PATCH 5/6] final vers --- REPORT.md | 7 ++----- app/domain/job_admin.py | 42 ++++++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/REPORT.md b/REPORT.md index afac7ebc..268e54d3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 5 - Автор, группа: Григорян Илья Мурадович, БПИ2303. -- Целевая оценка; выбранный трек 5, если нужен: **5**, трек — **TODO**. Все пункты уровней **3 и 4** реализованы и проверены. +- Целевая оценка: **4**. Все пункты уровней **3 и 4** реализованы и проверены. - Commit/PR: TODO после публикации. - Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI, SQLAlchemy, Pillow, aio-pika; [RabbitMQ](https://www.rabbitmq.com/docs/reliability), [Pillow](https://pillow.readthedocs.io/en/stable/); помощь Codex при реализации и проверках. @@ -79,7 +79,6 @@ JOB_ID и REQUEST_UUID заменить UUID. Повтор команды с т | Разрыв COMMIT → publish | make test-outbox | SIGKILL publisher, затем ready без нового upload | evidence/verification.json | | Backoff/jitter, failed registry, replay | make test-jobs | Постоянные/временные ошибки различаются, replay безопасен | evidence/jobs.json | | Crash после PUT до ack, lease и cleanup | make test-jobs; make test-pipeline | Тот же результат, prefetch=1; активный staging сохранён | evidence/jobs.json; evidence/verification.json | -| Выбранный трек 5 | TODO | TODO | TODO | ## Эксперименты и отказы @@ -89,10 +88,8 @@ JOB_ID и REQUEST_UUID заменить UUID. Повтор команды с т | Crash после COMMIT до ack | Готовый дубль без обработки | Metadata и Last-Modified не изменились | После доставки; потерь нет | | Временная / постоянная ошибка | Ограниченный retry / немедленный failed | Пять / одна попытка; явный replay завершился ready | В пределах бюджета; потерь нет | -Трек 5 — **TODO**: гипотеза, методика, измерения и выводы. - ## Ограничения и следующие шаги Один Docker-хост, без репликации хранилищ и защиты от потери диска. Очередь не имеет admission control; pipeline фиксирован на версии 1. Доставка как минимум один раз, -с идемпотентным результатом. Исследование для уровня 5 — **TODO**. +с идемпотентным результатом. diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py index 6a6be443..5bfb82e2 100644 --- a/app/domain/job_admin.py +++ b/app/domain/job_admin.py @@ -75,28 +75,25 @@ def same_request(record, job_id, actor, reason): } -async def replay_failed_job( - session, storage, settings, job_id, *, request_id, actor, reason -): +def validate_replay_request(actor: str, reason: str) -> None: + """Проверить данные оператора для записи аудита.""" if ( re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS ): raise JobOperationError("invalid_audit_fields") - row = await jobs.lock_job(session, job_id) - if row is None: - raise JobOperationError("unknown_job") - record = await job_admin.replay_record(session, request_id) - if record is not None: - result = same_request(record, job_id, actor, reason) - await session.rollback() - return result - job, attachment = row + + +def validate_replay_job(job, attachment) -> None: + """Разрешить новый replay только для failed с сохранённым исходником.""" if attachment.status != "failed": raise JobOperationError("job_not_failed") if job.staging_key is None: raise JobOperationError("source_expired") - # Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT. + + +async def verify_replay_source(storage, settings, job, attachment) -> None: + """Проверить размер и SHA-256 исходника в S3 под блокировкой job.""" work = JobWork( job.id, attachment.id, @@ -117,6 +114,25 @@ async def replay_failed_job( if error.code in {"NoSuchKey", "NoSuchObject"}: raise JobOperationError("source_missing") from None raise + + +async def replay_failed_job( + session, storage, settings, job_id, *, request_id, actor, reason +): + """Организовать replay; блокировка, rollback и commit остаются здесь.""" + validate_replay_request(actor, reason) + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + record = await job_admin.replay_record(session, request_id) + if record is not None: + result = same_request(record, job_id, actor, reason) + await session.rollback() + return result + job, attachment = row + validate_replay_job(job, attachment) + # Блокировка сохраняется во время проверки S3, аудита и изменения job. + await verify_replay_source(storage, settings, job, attachment) limit = job.attempt + settings.job_max_attempts inserted = await job_admin.insert_replay( session, -- 2.54.0 From 2d4031331814c43cac1d5e85ca27463310bc385f Mon Sep 17 00:00:00 2001 From: gytall Date: Thu, 12 Nov 2026 23:02:32 +0300 Subject: [PATCH 6/6] fixed makefile and add new evidence --- Makefile | 2 +- evidence/make_check | 2 ++ evidence/make_tests | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 evidence/make_check create mode 100644 evidence/make_tests diff --git a/Makefile b/Makefile index 4bad55df..b7b71b09 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ validate: $(PYTHON) scripts/validate_openapi.py test: - $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)" $(TLS_ARGS) # Все сохранённые регрессионные тесты запускаются явно; PYTEST_ARGS выбирает подмножество. test-unit: diff --git a/evidence/make_check b/evidence/make_check new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/make_check @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/make_tests b/evidence/make_tests new file mode 100644 index 00000000..2861dce4 --- /dev/null +++ b/evidence/make_tests @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:28080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. -- 2.54.0