From 35753ff11d1fda1885d316cb89e13cce5b63f83c Mon Sep 17 00:00:00 2001 From: kirillka222 Date: Fri, 20 Nov 2026 18:44:04 +0300 Subject: [PATCH] =?UTF-8?q?lab5:=20=D0=B0=D1=81=D0=B8=D0=BD=D1=85=D1=80?= =?UTF-8?q?=D0=BE=D0=BD=D0=BD=D1=8B=D0=B9=20=D0=B2=D0=BE=D1=80=D0=BA=D0=B5?= =?UTF-8?q?=D1=80,=20=D0=B1=D1=80=D0=BE=D0=BA=D0=B5=D1=80=20=D1=81=D0=BE?= =?UTF-8?q?=D0=BE=D0=B1=D1=89=D0=B5=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 26 +- .github/pull_request_template.md | 40 +- Dockerfile | 24 + Makefile | 150 +++- REPORT.md | 85 ++- alembic.ini | 117 ++++ alembic/README | 1 + alembic/env.py | 92 +++ alembic/script.py.mako | 26 + .../versions/08fe661df060_create_tables.py | 30 + .../12043b148948_add_outbox_events.py | 35 + .../157ed8ce0e22_add_attachment_lease.py | 33 + .../1c0bb32fbe88_add_status_to_attachments.py | 28 + .../2a1b3c4d5e6f_add_thumbnail_fields.py | 23 + ...3fbed8a46_add_timestamps_to_attachments.py | 31 + ...f2c_add_owner_and_sha256_to_attachments.py | 35 + .../versions/b24373e3b620_create_tables.py | 67 ++ .../bc35c1b6b748_seed_explain_test_10k.py | 43 ++ ...7aaedac6_add_attachment_ids_to_messages.py | 30 + alembic/versions/e090e7a3e071_add_crop_key.py | 20 + .../ea4f73899364_add_attachments_table.py | 40 ++ .../f9b85be2b710_seed_load_test_data.py | 69 ++ alembic/versions/ffdfa8869ddc_add_webp_key.py | 20 + app/__init__.py | 0 app/config.py | 28 + app/database.py | 13 + app/dependencies.py | 6 + app/exceptions.py | 37 + app/image_processor.py | 62 ++ app/load_test.py | 336 +++++++++ app/main.py | 31 + app/middleware.py | 11 + app/models/__init__.py | 0 app/models/attachment.py | 48 ++ app/models/base.py | 4 + app/models/chat.py | 35 + app/models/message.py | 36 + app/models/outbox.py | 33 + app/models/user.py | 24 + app/outbox_dispatcher.py | 137 ++++ app/pagination.py | 20 + app/redis_client.py | 9 + app/replay_failed.py | 91 +++ app/routers/__init__.py | 0 app/routers/attachments.py | 286 ++++++++ app/routers/auth.py | 83 +++ app/routers/chats.py | 139 ++++ app/routers/health.py | 34 + app/routers/messages.py | 181 +++++ app/routers/users.py | 55 ++ app/security.py | 81 +++ app/storage.py | 22 + app/worker.py | 639 ++++++++++++++++++ backpressure_test.py | 259 +++++++ benchmark_07ebc3e5.csv | 16 + benchmark_413d66b3.csv | 16 + benchmark_8742d471.csv | 16 + benchmark_results/backpressure_queue.csv | 64 ++ benchmark_results/benchmark_16f64efa.csv | 61 ++ benchmark_results/benchmark_7499bd42.csv | 61 ++ benchmark_results/benchmark_be804c9e.csv | 61 ++ benchmark_results/rss_1_1.csv | 74 ++ benchmark_results/rss_2_2.csv | 43 ++ benchmark_results/rss_4_4.csv | 25 + benchmark_results/summary.csv | 4 + benchmark_rss.py | 297 ++++++++ docker-compose.yml | 197 ++++++ nginx/nginx.conf | 27 + requirements-dev.txt | 17 + schemas/__init__.py | 0 schemas/chat.py | 14 + schemas/message.py | 6 + schemas/user.py | 7 + scripts/__init__.py | 0 scripts/failover-test.sh | 21 + scripts/load-50.py | 76 +++ scripts/plot.py | 23 + scripts/ram-experiment.py | 90 +++ scripts/reconcile.py | 146 ++++ scripts/rolling-load.sh | 12 + scripts/rolling-update.sh | 20 + scripts/run-grace-period-test.sh | 78 +++ scripts/run-reconciler-test.sh | 153 +++++ scripts/run-used-attachment-test.sh | 76 +++ scripts/test-stream-limit.py | 61 ++ tests/check_attachment_status | 29 + tests/check_attachment_status.py | 29 + tests/test_worker_retry.py | 154 +++++ 88 files changed, 5726 insertions(+), 53 deletions(-) create mode 100644 Dockerfile create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/08fe661df060_create_tables.py create mode 100644 alembic/versions/12043b148948_add_outbox_events.py create mode 100644 alembic/versions/157ed8ce0e22_add_attachment_lease.py create mode 100644 alembic/versions/1c0bb32fbe88_add_status_to_attachments.py create mode 100644 alembic/versions/2a1b3c4d5e6f_add_thumbnail_fields.py create mode 100644 alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py create mode 100644 alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py create mode 100644 alembic/versions/b24373e3b620_create_tables.py create mode 100644 alembic/versions/bc35c1b6b748_seed_explain_test_10k.py create mode 100644 alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py create mode 100644 alembic/versions/e090e7a3e071_add_crop_key.py create mode 100644 alembic/versions/ea4f73899364_add_attachments_table.py create mode 100644 alembic/versions/f9b85be2b710_seed_load_test_data.py create mode 100644 alembic/versions/ffdfa8869ddc_add_webp_key.py create mode 100644 app/__init__.py create mode 100644 app/config.py create mode 100644 app/database.py create mode 100644 app/dependencies.py create mode 100644 app/exceptions.py create mode 100644 app/image_processor.py create mode 100644 app/load_test.py create mode 100644 app/main.py create mode 100644 app/middleware.py create mode 100644 app/models/__init__.py create mode 100644 app/models/attachment.py create mode 100644 app/models/base.py create mode 100644 app/models/chat.py create mode 100644 app/models/message.py create mode 100644 app/models/outbox.py create mode 100644 app/models/user.py create mode 100644 app/outbox_dispatcher.py create mode 100644 app/pagination.py create mode 100644 app/redis_client.py create mode 100644 app/replay_failed.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/attachments.py create mode 100644 app/routers/auth.py create mode 100644 app/routers/chats.py create mode 100644 app/routers/health.py create mode 100644 app/routers/messages.py create mode 100644 app/routers/users.py create mode 100644 app/security.py create mode 100644 app/storage.py create mode 100644 app/worker.py create mode 100644 backpressure_test.py create mode 100644 benchmark_07ebc3e5.csv create mode 100644 benchmark_413d66b3.csv create mode 100644 benchmark_8742d471.csv create mode 100644 benchmark_results/backpressure_queue.csv create mode 100644 benchmark_results/benchmark_16f64efa.csv create mode 100644 benchmark_results/benchmark_7499bd42.csv create mode 100644 benchmark_results/benchmark_be804c9e.csv create mode 100644 benchmark_results/rss_1_1.csv create mode 100644 benchmark_results/rss_2_2.csv create mode 100644 benchmark_results/rss_4_4.csv create mode 100644 benchmark_results/summary.csv create mode 100644 benchmark_rss.py create mode 100644 docker-compose.yml create mode 100644 nginx/nginx.conf create mode 100644 schemas/__init__.py create mode 100644 schemas/chat.py create mode 100644 schemas/message.py create mode 100644 schemas/user.py create mode 100644 scripts/__init__.py create mode 100755 scripts/failover-test.sh create mode 100644 scripts/load-50.py create mode 100644 scripts/plot.py create mode 100644 scripts/ram-experiment.py create mode 100644 scripts/reconcile.py create mode 100755 scripts/rolling-load.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/run-grace-period-test.sh create mode 100755 scripts/run-reconciler-test.sh create mode 100755 scripts/run-used-attachment-test.sh create mode 100644 scripts/test-stream-limit.py create mode 100644 tests/check_attachment_status create mode 100644 tests/check_attachment_status.py create mode 100644 tests/test_worker_retry.py diff --git a/.env.example b/.env.example index 119384b5..519a85a7 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,19 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +POSTGRES_USER=messenger +POSTGRES_PASSWORD=messenger_password +POSTGRES_DB=messenger_db +DATABASE_URL=postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db +REDIS_URL=redis://redis:6379/0 +SESSION_STORE=redis IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=api-user +S3_SECRET_ACCESS_KEY=api-user-secret-2026 MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +MAX_IMAGE_PIXELS=40000000 \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..3a73f372 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,30 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +**Автор / группа:** Долда Кирилл, БПИ2301 +**Целевая оценка:** 5 +**Трек 5:** Трек 1 — «Очередь под перегрузкой». -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +**Что реализовано:** асинхронная обработка изображений через RabbitMQ, Worker, Transactional Outbox, retry, восстановление после сбоев, backpressure и ограничение очереди до 100 активных заданий. Из лабораторной 4 сохранены API, PostgreSQL, Redis, MinIO и система вложений. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +**Запуск из чистого clone:** +```bash +cp .env.example .env +docker compose up -d --build +docker compose exec app1 alembic upgrade head +make check +make test +``` + +**Результаты проверок:** обработано 120/120 изображений без ошибок. Throughput вырос с 1,53 до 5,14 изображений/сек. Проверены восстановление после сбоя, очистка staging и ограничение приёма заданий. Подробности — [REPORT.md](REPORT.md). + +**Ограничения:** увеличение параллелизма повышает потребление RAM; queue lag измеряется приблизительно. + +**Источники и ИИ:** FastAPI, SQLAlchemy, RabbitMQ, MinIO, Pillow, Docker; + +**Чек-лист:** +- [x] Сохранена функциональность предыдущих лабораторных. +- [x] Проверены сценарии отказов и перегрузки. +- [x] Реализован трек 5. +- [x] Финально выполнить `make check` и `make test`, сохранить вывод. +- [x] Проверить отсутствие секретов в Git. +- [x] Подготовлен `REPORT.md`. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..787808fe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.13-slim + +WORKDIR /app + +COPY requirements-dev.txt . +RUN pip install --no-cache-dir -r requirements-dev.txt + +RUN useradd --create-home --shell /bin/bash appuser + +COPY app/ app/ + +COPY alembic.ini . +COPY alembic/ alembic/ +RUN chown -R appuser:appuser /app/alembic +COPY scripts/ /app/scripts/ + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/live')" || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10", "--reload"] diff --git a/Makefile b/Makefile index 5d86d171..a474491c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -25,6 +25,148 @@ test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) -test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +explain-10k: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222' ORDER BY created_at, id LIMIT 50 OFFSET 0;" + +explain-composite: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_composite.txt + +explain-simple: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX IF EXISTS ix_messages_chat_created_id;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_only ON messages (chat_id); ANALYZE messages;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_simple.txt + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX ix_messages_chat_only;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_created_id ON messages (chat_id, created_at, id);" + +backup-restore: + @echo "Делаем бекап pg_dump" + time docker exec -it mtusi_messenger_db pg_dump -U messenger -d messenger_db > evidence/backup.sql + @echo "Создаем новую базу messenger_restore" + docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS messenger_restore;" + docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "CREATE DATABASE messenger_restore;" + @echo "Копируем бекап в контейнер и восстанавливаем" + docker cp evidence/backup.sql mtusi_messenger_db:/tmp/ + time docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -f /tmp/backup.sql > /dev/null + @echo "В основной базе messenger_db:" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -t -c "SELECT count(*) FROM messages;" + @echo "В восстановленной базе messenger_restore:" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -t -c "SELECT count(*) FROM messages;" + +test-race: + @echo "Запускаем два одновременных запроса на регистрацию" + @RACE_USER="race_$$(date +%s)"; \ + curl -s -o /dev/null -w "Запрос 1: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \ + curl -s -o /dev/null -w "Запрос 2: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \ + wait + +test-redis: + @echo "Ставим Redis на паузу" + docker pause mtusi_messenger_redis + @echo "Запрос" + time curl -s -w "\nHTTP Статус: %{http_code}\n" http://localhost:8080/api/v1/users/me -H "Authorization: Bearer test_token" + @echo "Снимаем паузу" + docker unpause mtusi_messenger_redis + +test-health-db: + @echo "Останавливаем БД" + docker compose stop db + @echo "Проверяем health ready" + curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready + docker compose start db + +test-health-redis: + @echo "Останавливаем" + docker compose stop redis + @echo "Проверяем health ожидая 503 " + curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready + docker compose start redis + +test-dependencies: + $(PYTHON) -m pytest tests/test_resilience.py -v + +failover: + ./scripts/failover-test.sh app1 + +failover-app2: + ./scripts/failover-test.sh app2 + +rolling-load: + ./scripts/rolling-load.sh + +rolling-update: + ./scripts/rolling-update.sh + +test-lab3: + $(PYTHON) -m pytest tests/test_lab3.py -v + +test-attachments: + $(PYTHON) -m pytest tests/test_attachments.py -v + +reconcile-dry: + docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run + +reconcile-execute: + docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute + +reconcile-test: + ./scripts/run-reconciler-test.sh + +ram-experiment: + @echo "В другом терминале для определения нагрузки: docker stats --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}'" + mkdir -p evidence + docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" > evidence/ram-experiment.txt + docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/ram-experiment.py 2>&1 | tee -a evidence/ram-experiment.txt + docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/ram-experiment.txt + +stream-limit: + mkdir -p evidence + @echo "=== ДО ===" > evidence/stream-limit.txt + @docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt + @echo "" >> evidence/stream-limit.txt + docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/test-stream-limit.py 2>&1 | tee -a evidence/stream-limit.txt + @echo "" >> evidence/stream-limit.txt + @echo "=== ПОСЛЕ ===" >> evidence/stream-limit.txt + @docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt + +private-bucket: + mkdir -p evidence + @OBJECT_KEY=$$(docker compose exec -T db psql -U messenger -d messenger_db -t -c "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1;" | tr -d ' \n'); \ + echo "Object key: $$OBJECT_KEY"; \ + { \ + echo "=== Проверка private bucket ==="; \ + echo "Object key: $$OBJECT_KEY"; \ + echo ""; \ + echo "=== Прямой GET без авторизации (ожидаем 403) ==="; \ + curl -i "http://localhost:9000/messenger-attachments/$$OBJECT_KEY"; \ + } > evidence/private-bucket.txt 2>&1; \ + cat evidence/private-bucket.txt + +s3-user-check: + @echo "=== 1. api-user пытается создать пользователя (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user add myminio hacker hacker123 2>&1' | head -3 + @echo "" + @echo "=== 2. api-user пытается изменить политику (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin policy attach myminio readwrite --user hacker 2>&1' | head -3 + @echo "" + @echo "=== 3. api-user пытается удалить пользователя (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user remove myminio minioadmin 2>&1' | head -3 + +reconcile-cost: + @echo "=== Стоимость обхода reconciler ===" + time docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run 2>&1 | tee evidence/track2-cost-50.txt + +grace-period-test: + ./scripts/run-grace-period-test.sh + +used-attachment-test: + ./scripts/run-used-attachment-test.sh + +test-retry: + $(PYTHON) -m pytest tests/test_worker_retry.py -v -s + +test-queue-upload: + $(PYTHON) -c 'import sys; sys.path.insert(0, "tests"); from client import Client; c = Client(); alice, bob, eve, chat = c.bootstrap(); att, content = c.attachment(alice, chat); print("Attachment ID:", att["id"]); print("Status:", att["status"])' + +test-queue-status: + @test -n "$(ATTACHMENT_ID)" || (echo "Укажи ATTACHMENT_ID"; exit 1) + docker compose exec -T worker python - "$(ATTACHMENT_ID)" < tests/check_attachment_status.py \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..0d68d009 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,84 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Долда Кирилл, БПИ2301 +- **Целевая оценка:** 5, трек 1 «Очередь под перегрузкой» +- **Commit/PR:** TODO после публикации +- **Источники, библиотеки и помощь ИИ:** README лабораторной, FastAPI, SQLAlchemy, asyncpg, Alembic, RabbitMQ (aio-pika), PostgreSQL, Redis, MinIO, Pillow, Docker Compose. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Требования:** Docker Desktop, Docker Compose, Git, make, curl. Для локальной разработки — Python 3.13. + +**Основные образы:** `python:3.13-slim`, `postgres:16-alpine`, `redis:7-alpine`, `rabbitmq:3.12-management-alpine`, `nginx:alpine`, `pgsty/silo:latest`, `pgsty/mc:latest`. + +**Подготовка, сборка, миграции и проверка:** + +```bash +cp .env.example .env +docker compose up -d --build +docker compose exec -T app1 alembic upgrade head +make check +make test +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** клиент → Nginx → FastAPI (app1/app2) → PostgreSQL -> MinIO (исходники и варианты) → Outbox dispatcher → RabbitMQ → Worker → subprocess Pillow → MinIO + PostgreSQL. Redis используется для сессий. + +**Источник истины:** PostgreSQL хранит состояние задания (`queued`, `processing`, `ready`, `failed`), ошибки и lease. MinIO хранит файлы; RabbitMQ доставляет задания, но не определяет их итоговый статус. + +**Транзакции и надёжность:** + +- API сохраняет исходник в MinIO, затем создаёт `Attachment` и `OutboxEvent` в одной транзакции БД; возвращает HTTP `202`. +- Диспетчер публикует события в RabbitMQ с publisher confirm. Если публикация прервалась, неопубликованное событие остаётся в outbox. +- Worker подтверждает сообщение (`ack`) после фиксации результата. Повторная доставка уже готового задания безопасна (идемпотентность). +- Для зависших задач используется lease; для временных ошибок — ограниченные повторы (до 5) с backoff и jitter, затем `failed` с безопасным `error_code`. + +**Обработка:** отдельный subprocess с таймаутом и ограничением памяти создаёт thumbnail JPEG 256×256, crop JPEG 128×128 и WebP. + +**Лимиты:** загрузка до 10 МБ и 40 млн пикселей; `WORKER_CONCURRENCY` и `WORKER_PREFETCH` управляют параллелизмом. API принимает не более **100 активных** (`queued` + `processing`) задач; при переполнении — HTTP `429`. + +**Из lab4 перенесено:** FastAPI, Nginx, PostgreSQL, Redis, MinIO, авторизация, ограничения загрузки, миграции, reconciler. **Добавлено в lab5:** RabbitMQ, worker, outbox, варианты изображений, retry, lease, prefetch и нагрузочные эксперименты. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / сценарий | Наблюдаемый результат | Доказательство | +| --- | --- |--------------------------------------------------| --- | +| Публичный smoke | `make test` | PASS | `evidence/smoke-lab5.txt` | +| Асинхронная обработка | Upload → Outbox → RabbitMQ → Worker | Задания переходят в `ready` | `app/worker.py`, `app/outbox_dispatcher.py` | +| Retry и `failed` | Ошибка обработки, до 5 попыток | Реализовано; отдельный финальный лог не сохранён | `app/worker.py` | +| Варианты и изоляция | Обработка PNG/JPEG | Thumbnail, crop, WebP; subprocess с лимитами | `app/image_processor.py` | +| Outbox и идемпотентность | Crash после COMMIT до ack | `REDELIVERED: True`, `CRASH TEST: PASSED` | Вывод ручного теста | +| Трек 1: три режима | `benchmark_rss.py`, 60 задач на режим | 60/60 `ready` в каждом, 0 `failed` | `benchmark_results/summary.csv`, `rss_*.csv` | +| Трек 1: перегрузка | `backpressure_test.py`, 120 задач | `Recovery: PASSED`, потерь нет | `benchmark_results/backpressure_queue.csv` | +| Лимит приёма | Имитация 100/99 активных задач | HTTP `429` / разрешённая загрузка; `PASSED` | Вывод ручного теста | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Трек 1: очередь под перегрузкой -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Гипотеза:** большее число параллельных обработчиков повышает throughput, но увеличивает расход RAM. При перегрузке RabbitMQ должен накопить задания, а затем разобрать очередь без потерь. + +**Методика:** `app/load_test.py` создаёт JPEG трёх размеров: 512×512 (~172 КБ), 1024×1024 (~688 КБ), 2048×2048 (~2751 КБ). На каждый режим — 60 задач (20 каждого размера). Время до `ready` и queue lag оценивались опросом БД; RSS — по процессам worker-контейнера. + +| Concurrency / Prefetch | Ready / Failed | Время | Throughput | Пик RSS | p95 ready: 512 / 1024 / 2048, с | +| --- | --- | --- | --- | --- | --- | +| 1 / 1 | 60 / 0 | 39.34 с | 1.53 img/s | 214.80 МБ | 36.85 / 38.84 / 36.62 | +| 2 / 2 | 60 / 0 | 21.72 с | 2.76 img/s | 337.21 МБ | 16.67 / 17.15 / 20.43 | +| 4 / 4 | 60 / 0 | 11.67 с | 5.14 img/s | 536.34 МБ | 9.84 / 10.13 / 11.67 | + +**Средний наблюдаемый queue lag (512 / 1024 / 2048), с:** 1/1 — 18.98 / 22.74 / 18.76; 2/2 — 7.42 / 9.58 / 11.14; 4/4 — 5.72 / 5.14 / 5.70. + +**Вывод:** переход с 1/1 на 4/4 повысил throughput в 3.36 раза и сократил время обработки пакета примерно на 70%, но увеличил пиковый RSS в 2.5 раза. Режим 2/2 — компромисс. + +**Перегрузка (120 задач, 1/1):** 120/120 `ready`, 0 ошибок; 79.06 с, 1.52 img/s. Максимум в очереди — **119**, максимум `unacked` — **1**. После обработки `queued=0`, `unacked=0`, `Recovery: PASSED`. Это подтверждает backpressure по prefetch и восстановление после всплеска нагрузки. + +| Воздействие | Что наблюдали | Результат | +| --- | --- | --- | +| Перегрузка 120 изображениями | Очередь выросла до 119, затем опустела | 120/120 готовы, потерь нет | +| Истечение lease | Зависшая задача возвращена в обработку | Ручная проверка пройдена | +| 100 активных задач | Новый upload отклонён | HTTP 429, тест `PASSED` | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Подтверждено:** три режима worker обработали все задания; при перегрузке очередь восстановилась без потерь; повторная доставка после COMMIT безопасна; ветки лимита 100/99 работают. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..0e50bdb1 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,117 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..20ff915b --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,92 @@ +from app.models.base import Base +from app.models.user import User + +from app.models.chat import Chat, ChatMember +from app.models.message import Message +from app.config import DATABASE_URL + +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from app.models.attachment import Attachment +from app.models.outbox import OutboxEvent + + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +config.set_main_option("sqlalchemy.url", DATABASE_URL.replace("+asyncpg", "+psycopg2")) + + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/08fe661df060_create_tables.py b/alembic/versions/08fe661df060_create_tables.py new file mode 100644 index 00000000..51c3cf25 --- /dev/null +++ b/alembic/versions/08fe661df060_create_tables.py @@ -0,0 +1,30 @@ +"""create tables + +Revision ID: 08fe661df060 +Revises: bc35c1b6b748 +Create Date: 2026-10-31 11:52:26.342048 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '08fe661df060' +down_revision: Union[str, None] = 'bc35c1b6b748' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/12043b148948_add_outbox_events.py b/alembic/versions/12043b148948_add_outbox_events.py new file mode 100644 index 00000000..03694f8b --- /dev/null +++ b/alembic/versions/12043b148948_add_outbox_events.py @@ -0,0 +1,35 @@ +from alembic import op +import sqlalchemy as sa + +revision = "12043b148948" +down_revision = "ffdfa8869ddc" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "outbox_events", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("attachment_id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False + ), + sa.Column( + "published_at", + sa.DateTime(timezone=True), + nullable=True + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"] + ), + sa.PrimaryKeyConstraint("id") + ) + + +def downgrade() -> None: + op.drop_table("outbox_events") \ No newline at end of file diff --git a/alembic/versions/157ed8ce0e22_add_attachment_lease.py b/alembic/versions/157ed8ce0e22_add_attachment_lease.py new file mode 100644 index 00000000..3769c009 --- /dev/null +++ b/alembic/versions/157ed8ce0e22_add_attachment_lease.py @@ -0,0 +1,33 @@ + +"""add attachment lease + +Revision ID: 157ed8ce0e22 +Revises: 12043b148948 +Create Date: 2026-11-20 13:07:19.055182 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "157ed8ce0e22" +down_revision: Union[str, None] = "12043b148948" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "attachments", + sa.Column( + "lease_expires_at", + sa.DateTime(timezone=True), + nullable=True, + ), + ) + + +def downgrade() -> None: + op.drop_column("attachments", "lease_expires_at") diff --git a/alembic/versions/1c0bb32fbe88_add_status_to_attachments.py b/alembic/versions/1c0bb32fbe88_add_status_to_attachments.py new file mode 100644 index 00000000..df49b1c0 --- /dev/null +++ b/alembic/versions/1c0bb32fbe88_add_status_to_attachments.py @@ -0,0 +1,28 @@ +"""add status to attachments + +Revision ID: 1c0bb32fbe88 +Revises: 8203fbed8a46 +Create Date: 2026-11-17 15:38:59.622410 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1c0bb32fbe88' +down_revision: Union[str, None] = '8203fbed8a46' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('attachments', sa.Column('status', sa.String(), server_default='queued', nullable=False)) + op.add_column('attachments', sa.Column('error_code', sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column('attachments', 'error_code') + op.drop_column('attachments', 'status') \ No newline at end of file diff --git a/alembic/versions/2a1b3c4d5e6f_add_thumbnail_fields.py b/alembic/versions/2a1b3c4d5e6f_add_thumbnail_fields.py new file mode 100644 index 00000000..20151c60 --- /dev/null +++ b/alembic/versions/2a1b3c4d5e6f_add_thumbnail_fields.py @@ -0,0 +1,23 @@ +from alembic import op +import sqlalchemy as sa + +revision = "2a1b3c4d5e6f" +down_revision = "1c0bb32fbe88" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("attachments", sa.Column("thumbnail_key", sa.String(), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_sha256", sa.String(), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_size_bytes", sa.Integer(), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_width", sa.Integer(), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_height", sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column("attachments", "thumbnail_height") + op.drop_column("attachments", "thumbnail_width") + op.drop_column("attachments", "thumbnail_size_bytes") + op.drop_column("attachments", "thumbnail_sha256") + op.drop_column("attachments", "thumbnail_key") \ No newline at end of file diff --git a/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py b/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py new file mode 100644 index 00000000..d7e3a2d7 --- /dev/null +++ b/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py @@ -0,0 +1,31 @@ +"""add timestamps to attachments + +Revision ID: 8203fbed8a46 +Revises: b0f31212df2c +Create Date: 2026-11-11 15:59:23.045039 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8203fbed8a46' +down_revision: Union[str, None] = 'b0f31212df2c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Добавляем колонки created_at и updated_at + op.add_column('attachments', sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + op.add_column('attachments', sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + # УДАЛЕНО: op.alter_column('attachments', 'user_id', ...) - чтобы не упасть на старых записях + + +def downgrade() -> None: + op.drop_column('attachments', 'updated_at') + op.drop_column('attachments', 'created_at') + # УДАЛЕНО: op.alter_column('attachments', 'user_id', nullable=True) \ No newline at end of file diff --git a/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py b/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py new file mode 100644 index 00000000..c095702f --- /dev/null +++ b/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py @@ -0,0 +1,35 @@ +"""add owner and sha256 to attachments + +Revision ID: b0f31212df2c +Revises: d78a7aaedac6 +Create Date: 2026-11-11 15:39:07.391800 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b0f31212df2c' +down_revision: Union[str, None] = 'd78a7aaedac6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Добавляем user_id с nullable=True, чтобы не упасть на старых данных + op.add_column('attachments', sa.Column('user_id', sa.UUID(), nullable=True)) + + # 2. Добавляем sha256 + op.add_column('attachments', sa.Column('sha256', sa.String(), nullable=True)) + + # 3. Создаем внешний ключ + op.create_foreign_key(None, 'attachments', 'users', ['user_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + op.drop_constraint(None, 'attachments', type_='foreignkey') + op.drop_column('attachments', 'sha256') + op.drop_column('attachments', 'user_id') \ No newline at end of file diff --git a/alembic/versions/b24373e3b620_create_tables.py b/alembic/versions/b24373e3b620_create_tables.py new file mode 100644 index 00000000..f5bb0dea --- /dev/null +++ b/alembic/versions/b24373e3b620_create_tables.py @@ -0,0 +1,67 @@ +"""create tables + +Revision ID: b24373e3b620 +Revises: +Create Date: 2026-10-24 06:54:22.471827 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b24373e3b620' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('chats', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True) + op.create_table('chat_members', + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('chat_id', 'user_id') + ) + op.create_table('messages', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('sender_id', sa.UUID(), nullable=False), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_messages_chat_created_id', table_name='messages') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_index(op.f('ix_users_username'), table_name='users') + op.drop_table('users') + op.drop_table('chats') + # ### end Alembic commands ### diff --git a/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py b/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py new file mode 100644 index 00000000..31f3f3c0 --- /dev/null +++ b/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py @@ -0,0 +1,43 @@ +"""seed explain test 10k + +Revision ID: bc35c1b6b748 +Revises: f9b85be2b710 +Create Date: 2026-10-24 07:04:09.294404 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'bc35c1b6b748' +down_revision: Union[str, None] = 'f9b85be2b710' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('11111111-1111-1111-1111-111111111111', 'explain_test', 'Test', 'hash', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO chats (id, title, created_at) + VALUES ('22222222-2222-2222-2222-222222222222', 'Explain Chat', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '22222222-2222-2222-2222-222222222222', '11111111-1111-1111-1111-111111111111', 'Message ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + +def downgrade() -> None: + op.execute("DELETE FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222';") + op.execute("DELETE FROM chats WHERE id = '22222222-2222-2222-2222-222222222222';") + op.execute("DELETE FROM users WHERE id = '11111111-1111-1111-1111-111111111111';") diff --git a/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py b/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py new file mode 100644 index 00000000..6e284653 --- /dev/null +++ b/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py @@ -0,0 +1,30 @@ +"""add attachment_ids to messages + +Revision ID: d78a7aaedac6 +Revises: ea4f73899364 +Create Date: 2026-11-11 15:16:50.773392 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd78a7aaedac6' +down_revision: Union[str, None] = 'ea4f73899364' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('messages', sa.Column('attachment_ids', sa.JSON(), nullable=True)) + + op.execute("UPDATE messages SET attachment_ids = '[]'::json WHERE attachment_ids IS NULL") + + op.alter_column('messages', 'attachment_ids', nullable=False, server_default='[]') + + +def downgrade() -> None: + op.drop_column('messages', 'attachment_ids') \ No newline at end of file diff --git a/alembic/versions/e090e7a3e071_add_crop_key.py b/alembic/versions/e090e7a3e071_add_crop_key.py new file mode 100644 index 00000000..4371e058 --- /dev/null +++ b/alembic/versions/e090e7a3e071_add_crop_key.py @@ -0,0 +1,20 @@ + +from alembic import op +import sqlalchemy as sa + + +revision = "e090e7a3e071" +down_revision = "2a1b3c4d5e6f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "attachments", + sa.Column("crop_key", sa.String(), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("attachments", "crop_key") diff --git a/alembic/versions/ea4f73899364_add_attachments_table.py b/alembic/versions/ea4f73899364_add_attachments_table.py new file mode 100644 index 00000000..ffb86529 --- /dev/null +++ b/alembic/versions/ea4f73899364_add_attachments_table.py @@ -0,0 +1,40 @@ +"""add attachments table + +Revision ID: ea4f73899364 +Revises: 08fe661df060 +Create Date: 2026-11-11 14:37:24.096041 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ea4f73899364' +down_revision: Union[str, None] = '08fe661df060' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('attachments', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('object_key', sa.String(), nullable=False), + sa.Column('filename', sa.String(), nullable=False), + sa.Column('mime_type', sa.String(), nullable=False), + sa.Column('size_bytes', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('object_key') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('attachments') + # ### end Alembic commands ### diff --git a/alembic/versions/f9b85be2b710_seed_load_test_data.py b/alembic/versions/f9b85be2b710_seed_load_test_data.py new file mode 100644 index 00000000..a39b72ee --- /dev/null +++ b/alembic/versions/f9b85be2b710_seed_load_test_data.py @@ -0,0 +1,69 @@ +"""seed load test data + +Revision ID: f9b85be2b710 +Revises: b24373e3b620 +Create Date: ... +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'f9b85be2b710' +down_revision: Union[str, None] = 'b24373e3b620' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('55555555-5555-4555-8555-555555555555', 'load_test', 'Load', 'hash', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO chats (id, title, created_at) VALUES + ('66666666-6666-4666-8666-666666666666', 'Big Chat', NOW()), + ('77777777-7777-4777-8777-777777777777', 'Small Chat 1', NOW()), + ('88888888-8888-4888-8888-888888888888', 'Small Chat 2', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '66666666-6666-4666-8666-666666666666', '55555555-5555-4555-8555-555555555555', 'Big ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 80000) AS i; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '77777777-7777-4777-8777-777777777777', '55555555-5555-4555-8555-555555555555', 'Small1 ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '88888888-8888-4888-8888-888888888888', '55555555-5555-4555-8555-555555555555', 'Small2 ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + + +def downgrade() -> None: + op.execute(""" + DELETE FROM messages WHERE chat_id IN ( + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '88888888-8888-4888-8888-888888888888' + ); + """) + op.execute(""" + DELETE FROM chats WHERE id IN ( + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '88888888-8888-4888-8888-888888888888' + ); + """) + op.execute("DELETE FROM users WHERE id = '55555555-5555-4555-8555-555555555555';") diff --git a/alembic/versions/ffdfa8869ddc_add_webp_key.py b/alembic/versions/ffdfa8869ddc_add_webp_key.py new file mode 100644 index 00000000..439a9852 --- /dev/null +++ b/alembic/versions/ffdfa8869ddc_add_webp_key.py @@ -0,0 +1,20 @@ + +from alembic import op +import sqlalchemy as sa + + +revision = "ffdfa8869ddc" +down_revision = "e090e7a3e071" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "attachments", + sa.Column("webp_key", sa.String(), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("attachments", "webp_key") diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..7c029933 --- /dev/null +++ b/app/config.py @@ -0,0 +1,28 @@ +import os + +DATABASE_URL = os.getenv( + "DATABASE_URL", + "postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db", +) +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + +CODE_MAP = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "minioadmin") +S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadmin") +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) \ No newline at end of file diff --git a/app/database.py b/app/database.py new file mode 100644 index 00000000..b0447a18 --- /dev/null +++ b/app/database.py @@ -0,0 +1,13 @@ +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from app.config import DATABASE_URL + +engine = create_async_engine( + DATABASE_URL, + pool_size=5, + max_overflow=10, + pool_timeout=30, + pool_recycle=1800, + echo=False, +) + +async_session_maker = async_sessionmaker(engine, expire_on_commit=False) \ No newline at end of file diff --git a/app/dependencies.py b/app/dependencies.py new file mode 100644 index 00000000..647232bc --- /dev/null +++ b/app/dependencies.py @@ -0,0 +1,6 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.database import async_session_maker + +async def get_session(): + async with async_session_maker() as session: + yield session \ No newline at end of file diff --git a/app/exceptions.py b/app/exceptions.py new file mode 100644 index 00000000..0e6b18a2 --- /dev/null +++ b/app/exceptions.py @@ -0,0 +1,37 @@ +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError +from app.config import CODE_MAP + + +def error_response(request: Request, status_code: int, code: str, message: str, extra_headers: dict = None): + request_id = getattr(request.state, "request_id", "unknown") + + + headers = {"X-Request-Id": request_id} + if extra_headers: + headers.update(extra_headers) + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers, + ) + + +async def http_exception_handler(request: Request, exc: HTTPException): + code = CODE_MAP.get(exc.status_code, "internal_error") + return error_response(request, exc.status_code, code, str(exc.detail), extra_headers=exc.headers) + + +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Ошибка валидации данных") + +async def internal_error_handler(request: Request, exc: Exception): + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") \ No newline at end of file diff --git a/app/image_processor.py b/app/image_processor.py new file mode 100644 index 00000000..cc42d41a --- /dev/null +++ b/app/image_processor.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_INPUT_PIXELS = 20_000_000 + + +def main(): + input_path = Path(sys.argv[1]) + output_dir = Path(sys.argv[2]) + + try: + with Image.open(input_path) as source: + if source.width * source.height > MAX_INPUT_PIXELS: + return 20 + + source.load() + img = source.convert("RGB") + + crop = ImageOps.fit( + img, + (128, 128), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5) + ) + + crop.save( + output_dir / "crop.jpg", + format="JPEG", + quality=85 + ) + + img.save( + output_dir / "optimized.webp", + format="WEBP", + quality=80, + method=6 + ) + + img.thumbnail((256, 256)) + + img.save( + output_dir / "thumbnail.jpg", + format="JPEG", + quality=85 + ) + + except ( + UnidentifiedImageError, + OSError, + ValueError, + Image.DecompressionBombError, + ): + return 21 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/app/load_test.py b/app/load_test.py new file mode 100644 index 00000000..b01d88a0 --- /dev/null +++ b/app/load_test.py @@ -0,0 +1,336 @@ +import argparse +import asyncio +import csv +import hashlib +import io +import statistics +import time +import uuid +from pathlib import Path + +from PIL import Image +from sqlalchemy import select + +from app.config import S3_BUCKET +from app.models.attachment import Attachment +from app.models.outbox import OutboxEvent +from app.outbox_dispatcher import session_maker +from app.worker import minio_client + + +IMAGE_SIZES = [512, 1024, 2048] + + +def make_image(size): + image = Image.effect_noise((size, size), 80).convert("RGB") + + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=85) + + return buffer.getvalue() + + +def percentile(values, percent): + values = sorted(values) + + if not values: + return None + + position = (len(values) - 1) * percent / 100 + lower = int(position) + upper = min(lower + 1, len(values) - 1) + + return values[lower] + ( + values[upper] - values[lower] + ) * (position - lower) + + +async def get_source(): + async with session_maker() as session: + result = await session.execute( + select(Attachment) + .where(Attachment.status == "ready") + .order_by(Attachment.created_at.desc()) + .limit(1) + ) + + source = result.scalar_one_or_none() + + if source is None: + raise RuntimeError( + "Нет готовой фотографии для создания тестовых записей" + ) + + return source.chat_id, source.user_id + + +async def create_jobs(count_per_size, run_id): + chat_id, user_id = await get_source() + + jobs = [] + + for size in IMAGE_SIZES: + data = make_image(size) + + print( + f"[LOAD] Изображение {size}x{size}: " + f"{len(data) / 1024:.1f} KB", + flush=True, + ) + + for index in range(count_per_size): + key = f"benchmark/{run_id}/{size}_{index}.jpg" + + minio_client.put_object( + S3_BUCKET, + key, + io.BytesIO(data), + len(data), + content_type="image/jpeg", + ) + + jobs.append({ + "size": size, + "key": key, + "data": data, + }) + + async with session_maker() as session: + async with session.begin(): + for job in jobs: + data = job["data"] + + attachment = Attachment( + object_key=job["key"], + filename=f"benchmark_{job['size']}.jpg", + mime_type="image/jpeg", + size_bytes=len(data), + chat_id=chat_id, + user_id=user_id, + sha256=hashlib.sha256(data).hexdigest(), + status="queued", + ) + + session.add(attachment) + await session.flush() + + session.add( + OutboxEvent(attachment_id=attachment.id) + ) + + job["id"] = attachment.id + del job["data"] + + return jobs + + +async def monitor_jobs(jobs, timeout): + ids = [job["id"] for job in jobs] + + started = time.perf_counter() + last_report = started + + first_processing = {} + ready_times = {} + failed = set() + + peak_processing = 0 + + while True: + now = time.perf_counter() + elapsed = now - started + + async with session_maker() as session: + result = await session.execute( + select(Attachment.id, Attachment.status) + .where(Attachment.id.in_(ids)) + ) + + statuses = result.all() + + processing_count = 0 + + for attachment_id, status in statuses: + if status == "processing": + processing_count += 1 + + if attachment_id not in first_processing: + first_processing[attachment_id] = elapsed + + elif status == "ready": + if attachment_id not in ready_times: + ready_times[attachment_id] = elapsed + + elif status == "failed": + failed.add(attachment_id) + + peak_processing = max( + peak_processing, + processing_count, + ) + + completed = len(ready_times) + len(failed) + + if now - last_report >= 5: + print( + f"[LOAD] Готово: {len(ready_times)}/{len(jobs)}, " + f"failed={len(failed)}, " + f"processing={processing_count}, " + f"elapsed={elapsed:.1f}s", + flush=True, + ) + last_report = now + + if completed == len(jobs): + break + + if elapsed >= timeout: + print("[LOAD] Достигнут таймаут", flush=True) + break + + await asyncio.sleep(0.1) + + return { + "ready_times": ready_times, + "first_processing": first_processing, + "failed": failed, + "peak_processing": peak_processing, + "elapsed": time.perf_counter() - started, + } + + +def save_results(jobs, results, run_id): + output = Path(f"benchmark_{run_id}.csv") + + with output.open("w", newline="") as file: + writer = csv.writer(file) + + writer.writerow([ + "attachment_id", + "image_size", + "status", + "queue_lag_estimate_s", + "time_to_ready_s", + ]) + + for job in jobs: + attachment_id = job["id"] + + ready = results["ready_times"].get(attachment_id) + lag = results["first_processing"].get(attachment_id) + + status = ( + "ready" if ready is not None + else "failed" if attachment_id in results["failed"] + else "unfinished" + ) + + writer.writerow([ + str(attachment_id), + job["size"], + status, + round(lag, 3) if lag is not None else "", + round(ready, 3) if ready is not None else "", + ]) + + return output + + +def print_results(jobs, results): + ready_times = results["ready_times"] + first_processing = results["first_processing"] + + total = len(jobs) + completed = len(ready_times) + + throughput = ( + completed / results["elapsed"] + if results["elapsed"] > 0 + else 0 + ) + + print("\n========== BENCHMARK RESULTS ==========") + print(f"Total jobs: {total}") + print(f"Ready: {completed}") + print(f"Failed: {len(results['failed'])}") + print(f"Duration: {results['elapsed']:.2f} s") + print(f"Throughput: {throughput:.2f} images/s") + print(f"Peak observed processing: {results['peak_processing']}") + + for size in IMAGE_SIZES: + size_jobs = [ + job for job in jobs + if job["size"] == size + ] + + times = [ + ready_times[job["id"]] + for job in size_jobs + if job["id"] in ready_times + ] + + lags = [ + first_processing[job["id"]] + for job in size_jobs + if job["id"] in first_processing + ] + + print(f"\nImage size: {size}x{size}") + print(f"Completed: {len(times)}/{len(size_jobs)}") + + if times: + print(f"Mean ready: {statistics.mean(times):.2f} s") + print(f"P95 ready: {percentile(times, 95):.2f} s") + + if lags: + print(f"Mean observed queue lag: {statistics.mean(lags):.2f} s") + print(f"Queue lag samples: {len(lags)}/{len(size_jobs)}") + + print("\n=======================================") + + +async def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--count", + type=int, + default=10, + help="Количество изображений каждого размера", + ) + + parser.add_argument( + "--timeout", + type=int, + default=300, + ) + + args = parser.parse_args() + + if args.count < 1 or args.timeout < 1: + parser.error("--count и --timeout должны быть положительными") + + run_id = uuid.uuid4().hex[:8] + + print(f"[LOAD] Run ID: {run_id}") + print("[LOAD] Подготовка изображений...") + + jobs = await create_jobs(args.count, run_id) + + print(f"[LOAD] Создано заданий: {len(jobs)}") + print("[LOAD] Ожидание обработки...") + + results = await monitor_jobs(jobs, args.timeout) + + print_results(jobs, results) + + output = save_results(jobs, results, run_id) + + print(f"\nCSV: {output}") + + if len(results["ready_times"]) != len(jobs): + raise SystemExit(1) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..14277d99 --- /dev/null +++ b/app/main.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from fastapi import HTTPException +import uuid +from app.middleware import RequestIdMiddleware +from app.exceptions import http_exception_handler, validation_exception_handler +from app.routers import health, users, chats, messages, auth, attachments + + +app = FastAPI( + title="MTUSI Messenger", + version="1.1.0", +) + +INSTANCE_ID = str(uuid.uuid4()) +@app.middleware("http") +async def add_instance_id_header(request, call_next): + response = await call_next(request) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + +app.add_middleware(RequestIdMiddleware) +app.add_exception_handler(HTTPException, http_exception_handler) +app.add_exception_handler(RequestValidationError, validation_exception_handler) +app.include_router(health.router) +app.include_router(users.router) +app.include_router(chats.router) +app.include_router(messages.router) +app.include_router(auth.router) +app.include_router(attachments.router_upload) +app.include_router(attachments.router_download) \ No newline at end of file diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..142c49e4 --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,11 @@ +import uuid as uuid_lib +from starlette.middleware.base import BaseHTTPMiddleware + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + request_id = str(uuid_lib.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + return response \ No newline at end of file diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 00000000..c5eeebef --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,48 @@ + +import uuid +from datetime import datetime + +from sqlalchemy import String, Integer, ForeignKey, DateTime, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class Attachment(Base): + __tablename__ = "attachments" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + object_key: Mapped[str] = mapped_column(String, unique=True) + filename: Mapped[str] = mapped_column(String) + mime_type: Mapped[str] = mapped_column(String) + size_bytes: Mapped[int] = mapped_column(Integer) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + sha256: Mapped[str] = mapped_column(String, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now() + ) + + status: Mapped[str] = mapped_column(String, default="queued") + error_code: Mapped[str] = mapped_column(String, nullable=True) + + lease_expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True + ) + + thumbnail_key: Mapped[str] = mapped_column(String, nullable=True) + thumbnail_sha256: Mapped[str] = mapped_column(String, nullable=True) + thumbnail_size_bytes: Mapped[int] = mapped_column(Integer, nullable=True) + thumbnail_width: Mapped[int] = mapped_column(Integer, nullable=True) + thumbnail_height: Mapped[int] = mapped_column(Integer, nullable=True) + + crop_key: Mapped[str] = mapped_column(String, nullable=True) + webp_key: Mapped[str] = mapped_column(String, nullable=True) diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 00000000..22784168 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,4 @@ +from sqlalchemy.orm import DeclarativeBase + +class Base(DeclarativeBase): + pass \ No newline at end of file diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 00000000..7bfed508 --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,35 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, DateTime, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class Chat(Base): + __tablename__ = "chats" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + title: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + +class ChatMember(Base): + __tablename__ = "chat_members" + + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + primary_key=True, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + primary_key=True, + ) \ No newline at end of file diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 00000000..291e5668 --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,36 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Text, DateTime, ForeignKey, Index, func, JSON +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class Message(Base): + __tablename__ = "messages" + + attachment_ids: Mapped[list] = mapped_column(JSON, default=list) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + ) + sender_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + text: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + __table_args__ = ( + Index("ix_messages_chat_created_id", "chat_id", "created_at", "id"), + ) \ No newline at end of file diff --git a/app/models/outbox.py b/app/models/outbox.py new file mode 100644 index 00000000..a675b186 --- /dev/null +++ b/app/models/outbox.py @@ -0,0 +1,33 @@ +import uuid + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.attachment import Base + + +class OutboxEvent(Base): + __tablename__ = "outbox_events" + + id: Mapped[uuid.UUID] = mapped_column( + primary_key=True, + default=uuid.uuid4 + ) + + attachment_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("attachments.id"), + nullable=False + ) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False + ) + + published_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True + ) \ No newline at end of file diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 00000000..03ab1ace --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,24 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, DateTime, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class User(Base): + __tablename__ = "users" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + username: Mapped[str] = mapped_column( + String(32), unique=True, nullable=False, index=True + ) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) \ No newline at end of file diff --git a/app/outbox_dispatcher.py b/app/outbox_dispatcher.py new file mode 100644 index 00000000..ed6bc0b5 --- /dev/null +++ b/app/outbox_dispatcher.py @@ -0,0 +1,137 @@ + +import asyncio +import json +import os +from datetime import datetime, timezone + +import aio_pika +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.config import DATABASE_URL +from app.models.attachment import Attachment +from app.models.chat import Chat, ChatMember +from app.models.outbox import OutboxEvent +from app.models.user import User + +RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/") +QUEUE_NAME = "image_tasks" +POLL_INTERVAL_SECONDS = 1 + +engine = create_async_engine(DATABASE_URL) +session_maker = async_sessionmaker(engine, expire_on_commit=False) + + +async def recover_expired_lease(): + now = datetime.now(timezone.utc) + + async with session_maker() as session: + async with session.begin(): + result = await session.execute( + select(Attachment) + .where( + Attachment.status == "processing", + Attachment.lease_expires_at.is_not(None), + Attachment.lease_expires_at <= now, + ) + .order_by(Attachment.lease_expires_at) + .limit(1) + .with_for_update(skip_locked=True) + ) + + att = result.scalar_one_or_none() + + if att is None: + return False + + att.status = "queued" + att.lease_expires_at = None + + session.add( + OutboxEvent(attachment_id=att.id) + ) + + attachment_id = att.id + + print(f"[Outbox] Recovered expired lease: {attachment_id}", flush=True) + return True + + +async def dispatch_one(channel): + async with session_maker() as session: + async with session.begin(): + result = await session.execute( + select(OutboxEvent) + .where(OutboxEvent.published_at.is_(None)) + .order_by(OutboxEvent.created_at, OutboxEvent.id) + .limit(1) + .with_for_update(skip_locked=True) + ) + + event = result.scalar_one_or_none() + + if event is None: + return False + + confirmation = await channel.default_exchange.publish( + aio_pika.Message( + body=json.dumps( + {"attachment_id": str(event.attachment_id)} + ).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=str(event.id), + content_type="application/json", + ), + routing_key=QUEUE_NAME, + mandatory=True, + ) + + if confirmation is False: + raise RuntimeError("RabbitMQ did not confirm publication") + + event.published_at = datetime.now(timezone.utc) + + print(f"[Outbox] Published {event.attachment_id}", flush=True) + return True + + +async def main(): + try: + while True: + try: + connection = await aio_pika.connect_robust(RABBITMQ_URL) + + async with connection: + channel = await connection.channel( + publisher_confirms=True, + on_return_raises=True, + ) + + await channel.declare_queue( + QUEUE_NAME, + durable=True, + ) + + while True: + recovered = await recover_expired_lease() + published = await dispatch_one(channel) + + if not recovered and not published: + await asyncio.sleep(POLL_INTERVAL_SECONDS) + + except asyncio.CancelledError: + raise + + except Exception as exc: + print( + f"[Outbox] Retry after error: {type(exc).__name__}", + flush=True, + ) + await asyncio.sleep(2) + + finally: + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/app/pagination.py b/app/pagination.py new file mode 100644 index 00000000..ca387009 --- /dev/null +++ b/app/pagination.py @@ -0,0 +1,20 @@ +def paginate(items: list, limit: int, cursor: str | None): + start = 0 + if cursor is not None: + try: + start = int(cursor) + except ValueError: + start = 0 + + if start < 0: + start = 0 + + page = items[start : start + limit] + next_index = start + limit + + if next_index < len(items): + next_cursor = str(next_index) + else: + next_cursor = None + + return page, next_cursor \ No newline at end of file diff --git a/app/redis_client.py b/app/redis_client.py new file mode 100644 index 00000000..0d6aa009 --- /dev/null +++ b/app/redis_client.py @@ -0,0 +1,9 @@ +import redis.asyncio as redis +from app.config import REDIS_URL + +redis_client = redis.from_url( + REDIS_URL, + decode_responses=True, + socket_timeout=3, + socket_connect_timeout=3, +) \ No newline at end of file diff --git a/app/replay_failed.py b/app/replay_failed.py new file mode 100644 index 00000000..2ba367fa --- /dev/null +++ b/app/replay_failed.py @@ -0,0 +1,91 @@ + +import asyncio +import sys +import uuid + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.config import DATABASE_URL +from app.models.attachment import Attachment +from app.models.outbox import OutboxEvent +from app.models.user import User +from app.models.chat import Chat, ChatMember + + +RETRYABLE_ERROR_CODES = { + "s3_download_error", + "s3_upload_error", + "image_processing_timeout", + "image_processor_start_error", + "image_processor_error", + "image_processor_output_error", + "worker_error", +} + + +async def replay_failed(attachment_id: uuid.UUID): + engine = create_async_engine(DATABASE_URL) + session_maker = async_sessionmaker(engine, expire_on_commit=False) + + try: + async with session_maker() as session: + async with session.begin(): + att = await session.get( + Attachment, + attachment_id, + with_for_update=True, + ) + + if att is None: + print("Файл не найден") + return + + if att.status != "failed": + print(f"Replay запрещён: status={att.status}") + return + + if att.error_code not in RETRYABLE_ERROR_CODES: + print( + f"Replay запрещён: error_code={att.error_code}" + ) + return + + pending = await session.scalar( + select(OutboxEvent.id) + .where( + OutboxEvent.attachment_id == attachment_id, + OutboxEvent.published_at.is_(None), + ) + .limit(1) + ) + + if pending is not None: + print("Задание уже ожидает публикации") + return + + att.status = "queued" + att.error_code = None + + session.add( + OutboxEvent(attachment_id=attachment_id) + ) + + print(f"Задание {attachment_id} отправлено на повторную обработку") + + finally: + await engine.dispose() + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Использование: python -m app.replay_failed UUID") + sys.exit(1) + + try: + attachment_id = uuid.UUID(sys.argv[1]) + except ValueError: + print("Некорректный UUID") + sys.exit(1) + + asyncio.run(replay_failed(attachment_id)) diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/attachments.py b/app/routers/attachments.py new file mode 100644 index 00000000..e8744128 --- /dev/null +++ b/app/routers/attachments.py @@ -0,0 +1,286 @@ +import uuid +import filetype +import io +import hashlib +from datetime import datetime, timezone +from PIL import Image +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, func, text + +from app.dependencies import get_session +from app.security import get_current_user +from app.models.user import User +from app.models.attachment import Attachment +from app.models.outbox import OutboxEvent +from app.models.chat import ChatMember +from app.storage import minio_client, BUCKET + + +router_upload = APIRouter( + prefix="/api/v1/chats/{chat_id}/attachments", + tags=["attachments"] +) + +router_download = APIRouter( + prefix="/api/v1/attachments", + tags=["attachments"] +) + +MAX_SIZE = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +MAX_ACTIVE_ATTACHMENTS = 100 +ADMISSION_LOCK_KEY = 823164001 + +def format_utc_timestamp(value: datetime | None) -> str: + if value is None: + value = datetime.now(timezone.utc) + + elif value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + + return ( + value.astimezone(timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + + +def build_attachment_response(att: Attachment) -> dict: + variants = [] + + if ( + att.status == "ready" + and att.thumbnail_key + and att.thumbnail_sha256 + and att.thumbnail_size_bytes is not None + and att.thumbnail_width is not None + and att.thumbnail_height is not None + ): + variants.append({ + "name": "thumbnail", + "content_type": "image/jpeg", + "size_bytes": att.thumbnail_size_bytes, + "width": att.thumbnail_width, + "height": att.thumbnail_height, + "sha256": att.thumbnail_sha256, + }) + + return { + "id": str(att.id), + "chat_id": str(att.chat_id), + "owner_id": str(att.user_id), + "filename": att.filename, + "content_type": att.mime_type, + "size_bytes": att.size_bytes, + "sha256": att.sha256, + "status": att.status, + "variants": variants, + "error_code": att.error_code, + "created_at": format_utc_timestamp(att.created_at), + "updated_at": format_utc_timestamp(att.updated_at), + } + + +@router_upload.post("", status_code=202) +async def upload_attachment( + chat_id: uuid.UUID, + filename: str = Query( + ..., + min_length=1, + max_length=255, + pattern=r"^[^/\\\x00]+$" + ), + request: Request = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == chat_id, + ChatMember.user_id == user.id, + ) + ) + + if not result.scalar_one_or_none(): + raise HTTPException(404, "Чат не найден") + + sha_hasher = hashlib.sha256() + buffer = io.BytesIO() + total = 0 + first_bytes = b"" + + async for chunk in request.stream(): + if not chunk: + continue + + total += len(chunk) + + if total > MAX_SIZE: + raise HTTPException(413, "File too large") + + sha_hasher.update(chunk) + buffer.write(chunk) + + if len(first_bytes) < 262: + need = 262 - len(first_bytes) + first_bytes += chunk[:need] + + kind = filetype.guess(first_bytes) + + if not kind or kind.mime not in ("image/png", "image/jpeg"): + raise HTTPException(400, "Only PNG/JPEG allowed") + + contents = buffer.getvalue() + + try: + img = Image.open(io.BytesIO(contents)) + + if img.width * img.height > MAX_PIXELS: + raise HTTPException(400, "Image too large (pixels)") + + except HTTPException: + raise + except Exception: + raise HTTPException(400, "Corrupted image") + + # Serialize admission checks across all API instances. + # This transaction-level PostgreSQL lock is released on commit/rollback. + await session.execute( + text("SELECT pg_advisory_xact_lock(:lock_key)"), + {"lock_key": ADMISSION_LOCK_KEY}, + ) + + active_count = await session.scalar( + select(func.count()) + .select_from(Attachment) + .where(Attachment.status.in_(("queued", "processing"))) + ) + + if active_count >= MAX_ACTIVE_ATTACHMENTS: + await session.rollback() + raise HTTPException( + status_code=429, + detail="Image processing queue is full; try again later", + headers={"Retry-After": "30"}, + ) + + object_key = f"{uuid.uuid4()}.{kind.extension}" + + minio_client.put_object( + BUCKET, + object_key, + io.BytesIO(contents), + len(contents), + content_type=kind.mime, + ) + + sha_hash = sha_hasher.hexdigest() + + att = Attachment( + object_key=object_key, + filename=filename, + mime_type=kind.mime, + size_bytes=len(contents), + chat_id=chat_id, + user_id=user.id, + sha256=sha_hash, + status="queued" + ) + + session.add(att) + await session.flush() + + event = OutboxEvent(attachment_id=att.id) + session.add(event) + + await session.commit() + await session.refresh(att) + + return build_attachment_response(att) + + +@router_download.get("/{attachment_id}/content") +async def download_attachment_content( + attachment_id: uuid.UUID, + variant: str = "original", + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + att = await session.get(Attachment, attachment_id) + + if not att: + raise HTTPException(404, "Not found") + + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == att.chat_id, + ChatMember.user_id == user.id, + ) + ) + + if not result.scalar_one_or_none(): + raise HTTPException(404, "Not found") + + if att.status != "ready": + raise HTTPException(409, "File is not ready yet") + + if variant == "thumbnail": + if not att.thumbnail_key: + raise HTTPException(404, "Thumbnail not found") + + key_to_download = att.thumbnail_key + mime = "image/jpeg" + + elif variant == "original": + key_to_download = att.object_key + mime = att.mime_type + + else: + raise HTTPException(404, "Variant not found") + + try: + response = minio_client.get_object( + BUCKET, + key_to_download + ) + + return StreamingResponse( + response, + media_type=mime, + headers={ + "Content-Disposition": + f'attachment; filename="{att.filename}"' + }, + ) + + except Exception: + raise HTTPException( + 404, + "File not found in storage" + ) + + +@router_download.get("/{attachment_id}") +async def get_attachment_metadata( + attachment_id: uuid.UUID, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + att = await session.get(Attachment, attachment_id) + + if not att: + raise HTTPException(404, "Not found") + + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == att.chat_id, + ChatMember.user_id == user.id, + ) + ) + + if not result.scalar_one_or_none(): + raise HTTPException(404, "Not found") + + return build_attachment_response(att) diff --git a/app/routers/auth.py b/app/routers/auth.py new file mode 100644 index 00000000..439a5982 --- /dev/null +++ b/app/routers/auth.py @@ -0,0 +1,83 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPAuthorizationCredentials +from pydantic import BaseModel, ConfigDict, field_serializer +from app.redis_client import redis_client +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +import uuid +from datetime import datetime + +from app.database import async_session_maker +from app.models.user import User +from app.security import verify_password, create_session, delete_session, bearer_scheme + +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) +security = HTTPBasic(realm="messenger") + + +class UserResponse(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +class SessionResponse(BaseModel): + token: str + token_type: str + expires_at: str + user: UserResponse + + @field_serializer('expires_at') + def serialize_dt(self, dt_str: str): + if dt_str and dt_str.endswith("+00:00"): + return dt_str.replace("+00:00", "Z") + return dt_str + + +# Basic Auth в Bearer Token +@router.post("/sessions", status_code=201, response_model=SessionResponse) +async def login(credentials: HTTPBasicCredentials = Depends(security)): + async with async_session_maker() as session: + result = await session.execute(select(User).where(User.username == credentials.username)) + user = result.scalar_one_or_none() + + if not user or not verify_password(credentials.password, user.password_hash): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + + session_data = await create_session(user.id) + + session_data["user"] = user + + return session_data + +@router.delete("/sessions/current", status_code=204) +async def logout(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + if not credentials: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + token = credentials.credentials + + user_id = await redis_client.get(f"session:{token}") + if not user_id: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + await delete_session(token) + return None diff --git a/app/routers/chats.py b/app/routers/chats.py new file mode 100644 index 00000000..64299117 --- /dev/null +++ b/app/routers/chats.py @@ -0,0 +1,139 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends +from pydantic import BaseModel, Field, field_validator, ConfigDict, field_serializer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select + +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.security import get_current_user +from app.dependencies import get_session +from app.pagination import paginate + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v): + if len(set(v)) != len(v): + raise ValueError("member_ids должны быть уникальными") + return v + +class ChatResponse(BaseModel): + id: uuid.UUID + title: str + member_ids: list[uuid.UUID] + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +@router.post("", status_code=201) +async def create_chat( + payload: CreateChat, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if str(user.id) not in payload.member_ids: + raise HTTPException(status_code=400, detail="Пользователь должен быть участником чата") + + for member_id in payload.member_ids: + result = await session.execute(select(User).where(User.id == member_id)) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=400, detail=f"несуществующий пользователь: {member_id}") + + chat = Chat(title=payload.title) + session.add(chat) + await session.flush() + + for member_id in payload.member_ids: + session.add(ChatMember(chat_id=chat.id, user_id=member_id)) + + await session.commit() + await session.refresh(chat) + + return { + "id": chat.id, + "title": chat.title, + "member_ids": payload.member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + } + + +@router.get("/{chat_id}") +async def get_chat( + chat_id: str, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + result = await session.execute(select(Chat).where(Chat.id == chat_id)) + chat = result.scalar_one_or_none() + + if not chat: + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute( + select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user.id) + ) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id)) + member_ids = [] + for row in result.all(): + user_id = row[0] + member_ids.append(str(user_id)) + + return { + "id": chat.id, + "title": chat.title, + "member_ids": member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + } + + +@router.get("") +async def list_chats( + limit: int = 20, + cursor: str | None = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100") + + result = await session.execute( + select(Chat) + .join(ChatMember, Chat.id == ChatMember.chat_id) + .where(ChatMember.user_id == user.id) + .order_by(Chat.created_at, Chat.id) + ) + chats = result.scalars().all() + + my_chats = [] + for chat in chats: + result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id)) + member_ids = [] + for row in result.all(): + user_id = row[0] + member_ids.append(str(user_id)) + + my_chats.append({ + "id": chat.id, + "title": chat.title, + "member_ids": member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + }) + + page, next_cursor = paginate(my_chats, limit, cursor) + return {"items": page, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/routers/health.py b/app/routers/health.py new file mode 100644 index 00000000..abcef176 --- /dev/null +++ b/app/routers/health.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter +from fastapi import HTTPException +from sqlalchemy import text +from app.database import async_session_maker +from app.redis_client import redis_client + + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +async def ready(): + try: + async with async_session_maker() as session: + await session.execute(text("SELECT 1")) + + await redis_client.ping() + + return {"status": "ok"} + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + +@router.get("/") +def home(): + return { + "service": "MTUSI Messenger", + "version": "1.1.0", + "docs": "/docs", + } \ No newline at end of file diff --git a/app/routers/messages.py b/app/routers/messages.py new file mode 100644 index 00000000..baa2c11a --- /dev/null +++ b/app/routers/messages.py @@ -0,0 +1,181 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends, Header +from pydantic import BaseModel, Field, ConfigDict, field_serializer, model_validator +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from app.redis_client import redis_client +import json + +from app.models.message import Message +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.models.attachment import Attachment +from app.security import get_current_user +from app.dependencies import get_session + +router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"]) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[uuid.UUID] = [] + + @model_validator(mode="after") + def check_text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Сообщение должно содержать текст или вложение") + return self + + +class MessageResponse(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + sender_id: uuid.UUID + text: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + attachment_ids: list[uuid.UUID] = [] + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +async def check_chat_access(chat_id: str, user_id: uuid.UUID, session: AsyncSession): + chat = await session.get(Chat, chat_id) + if not chat: + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute( + select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user_id) + ) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="чат не найден") + return chat + + +@router.post("", status_code=201, response_model=MessageResponse) +async def create_message( + chat_id: str, + payload: CreateMessage, + idempotency_key: str = Header( + ..., + alias="Idempotency-Key", + min_length=1, + max_length=128, + pattern=r"^[A-Za-z0-9._:-]+$", + ), + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + await check_chat_access(chat_id, user.id, session) + + if payload.attachment_ids: + result = await session.execute( + select(Attachment.id).where( + Attachment.id.in_(payload.attachment_ids), + Attachment.chat_id == chat_id, + ) + ) + found_ids = {str(row[0]) for row in result.all()} + requested_ids = {str(a) for a in payload.attachment_ids} + if found_ids != requested_ids: + raise HTTPException( + status_code=400, + detail="attachment_ids содержат файлы из другого чата или несуществующие" + ) + + redis_key = f"idem:{user.id}:{chat_id}:{idempotency_key}" + body = payload.model_dump(mode="json") + + raw = await redis_client.get(redis_key) + if raw: + stored = json.loads(raw) + if stored["state"] == "done": + if stored["request_body"] == body: + return stored["response"] + raise HTTPException(status_code=409, detail="Idempotency-Key переиспользован с другим телом запроса") + raise HTTPException(status_code=409, detail="Запрос с этим Idempotency-Key ещё обрабатывается") + + acquired = await redis_client.set( + redis_key, + json.dumps({"state": "processing", "request_body": body}), + nx=True, + ex=86400, + ) + if not acquired: + raise HTTPException(409, "Одновременный запрос с тем же самым Idempotency-Key") + + message = Message( + chat_id=chat_id, + sender_id=user.id, + text=payload.text, + attachment_ids=[str(a) for a in payload.attachment_ids] + ) + session.add(message) + await session.commit() + await session.refresh(message) + + response_data = { + "id": str(message.id), + "chat_id": str(message.chat_id), + "sender_id": str(message.sender_id), + "text": message.text, + "created_at": message.created_at.isoformat().replace("+00:00", "Z"), + "attachment_ids": message.attachment_ids, + } + + await redis_client.set( + redis_key, + json.dumps({"state": "done", + "request_body": body, + "response": response_data}), + ex=86400, + ) + + return response_data + + +@router.get("") +async def list_messages( + chat_id: str, + limit: int = 50, + cursor: str | None = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100") + + await check_chat_access(chat_id, user.id, session) + + offset = int(cursor) if cursor else 0 + + result = await session.execute( + select(Message) + .where(Message.chat_id == chat_id) + .order_by(Message.created_at, Message.id) + .limit(limit + 1) + .offset(offset) + ) + msgs = result.scalars().all() + + has_next = len(msgs) > limit + if has_next: + msgs = msgs[:limit] + + my_messages = [ + { + "id": m.id, + "chat_id": m.chat_id, + "sender_id": m.sender_id, + "text": m.text, + "created_at": m.created_at.isoformat().replace("+00:00", "Z"), + "attachment_ids": m.attachment_ids, + } for m in msgs + ] + + next_cursor = str(offset + limit) if has_next else None + return {"items": my_messages, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/routers/users.py b/app/routers/users.py new file mode 100644 index 00000000..cc77b513 --- /dev/null +++ b/app/routers/users.py @@ -0,0 +1,55 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends +from pydantic import BaseModel, Field, ConfigDict, field_serializer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from app.models.user import User +from app.security import get_password_hash, get_current_user +from app.dependencies import get_session + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=5, max_length=128) + +class UserResponse(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + +@router.post("", status_code=201, response_model=UserResponse) +async def register_user(payload: CreateUser, session: AsyncSession = Depends(get_session)): + result = await session.execute(select(User).where(User.username == payload.username)) + if result.scalar_one_or_none(): + raise HTTPException(status_code=409, detail="Имя пользователя уже существует") + + user = User( + username=payload.username, + display_name=payload.display_name, + password_hash=get_password_hash(payload.password) + ) + session.add(user) + + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise HTTPException(status_code=409, detail="Имя пользователя уже существует") + + await session.refresh(user) + return user + +@router.get("/me", response_model=UserResponse) +async def current_user(user: User = Depends(get_current_user)): + return user \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..e6f5588f --- /dev/null +++ b/app/security.py @@ -0,0 +1,81 @@ +import secrets +from datetime import datetime, timezone, timedelta +from passlib.context import CryptContext +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from sqlalchemy import select +from app.database import async_session_maker +from app.models.user import User +from app.redis_client import redis_client +from app.config import SESSION_TTL_SECONDS + +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") +bearer_scheme = HTTPBearer(auto_error=False) + +def get_password_hash(password: str) -> str: + return pwd_context.hash(password) + + +def verify_password(plain: str, hashed: str) -> bool: + return pwd_context.verify(plain, hashed) + + +async def create_session(user_id: str) -> dict: + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + + await redis_client.setex( + name=f"session:{token}", + time=SESSION_TTL_SECONDS, + value=str(user_id), + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + } + + +async def delete_session(token: str) -> None: + await redis_client.delete(f"session:{token}") + + +async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + if not credentials: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + token = credentials.credentials + + + try: + user_id = await redis_client.get(f"session:{token}") + except Exception: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Внутренняя ошибка сервера" + ) + + if not user_id: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + async with async_session_maker() as session: + result = await session.execute(select(User).where(User.id == user_id)) + user = result.scalar_one_or_none() + + if not user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + return user \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..5abf57a6 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,22 @@ +import threading +from minio import Minio +from app.config import S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET + +users_lock = threading.Lock() +messages_lock = threading.Lock() + +users_by_id: dict[str, dict] = {} +users_by_username: dict[str, dict] = {} +chats_by_id: dict[str, dict] = {} +messages_by_chat: dict[str, list[dict]] = {} + +endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "") + +minio_client = Minio( + endpoint, + access_key=S3_ACCESS_KEY_ID, + secret_key=S3_SECRET_ACCESS_KEY, + secure=False, +) + +BUCKET = S3_BUCKET \ No newline at end of file diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..9f90e364 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,639 @@ +import asyncio +import hashlib +import io +import json +import os +import random +import sys +import tempfile +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import aio_pika +from minio import Minio +from PIL import Image +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker + +from app.models.attachment import Attachment +from app.models.user import User +from app.models.chat import Chat, ChatMember +from app.config import ( + DATABASE_URL, + S3_ENDPOINT, + S3_ACCESS_KEY_ID, + S3_SECRET_ACCESS_KEY, + S3_BUCKET, +) + +RABBITMQ_URL = os.getenv( + "RABBITMQ_URL", + "amqp://guest:guest@rabbitmq:5672/", +) + +QUEUE_NAME = "image_tasks" + +MAX_ATTEMPTS = max( + 1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5")) +) + +CONCURRENCY = max( + 1, int(os.getenv("WORKER_CONCURRENCY", "1")) +) + +PREFETCH = max( + 1, int(os.getenv("WORKER_PREFETCH", "1")) +) + +MAX_IMAGE_BYTES = 10 * 1024 * 1024 +PROCESS_TIMEOUT_SECONDS = 30 +LEASE_SECONDS = 300 + +endpoint = S3_ENDPOINT.replace( + "http://", "" +).replace( + "https://", "" +) + +minio_client = Minio( + endpoint, + access_key=S3_ACCESS_KEY_ID, + secret_key=S3_SECRET_ACCESS_KEY, + secure=False, +) + +engine = create_async_engine(DATABASE_URL) + +async_session_maker = async_sessionmaker( + engine, + expire_on_commit=False, +) + + +class RetryableProcessingError(Exception): + def __init__(self, error_code: str): + self.error_code = error_code + super().__init__(error_code) + + +class PermanentProcessingError(Exception): + def __init__(self, error_code: str): + self.error_code = error_code + super().__init__(error_code) + + +class LeaseBusyError(Exception): + pass + + +class LeaseLostError(Exception): + pass + + +def get_rss_mb(): + try: + with open("/proc/self/status") as file: + for line in file: + if line.startswith("VmRSS:"): + kb = int(line.split()[1]) + return round(kb / 1024, 2) + except (OSError, ValueError, IndexError): + pass + + return None + + +async def claim_attachment(attachment_id: str): + async with async_session_maker() as session: + async with session.begin(): + att = await session.get( + Attachment, + uuid.UUID(attachment_id), + with_for_update=True, + ) + + if att is None: + print( + f"[Worker] Файл {attachment_id} не найден", + flush=True, + ) + return None + + if att.status == "ready" and att.thumbnail_key: + print( + f"[Worker] Файл {attachment_id} уже обработан", + flush=True, + ) + return None + + if att.status == "failed": + print( + f"[Worker] Файл {attachment_id} уже failed", + flush=True, + ) + return None + + now = datetime.now(timezone.utc) + + if ( + att.status == "processing" + and att.lease_expires_at is not None + and att.lease_expires_at > now + ): + raise LeaseBusyError() + + lease_expires_at = now + timedelta( + seconds=LEASE_SECONDS + ) + + att.status = "processing" + att.error_code = None + att.lease_expires_at = lease_expires_at + + object_key = att.object_key + + return object_key, lease_expires_at + + +async def release_lease( + attachment_id: str, + lease_expires_at: datetime, +): + async with async_session_maker() as session: + async with session.begin(): + att = await session.get( + Attachment, + uuid.UUID(attachment_id), + with_for_update=True, + ) + + if ( + att is not None + and att.status == "processing" + and att.lease_expires_at == lease_expires_at + ): + att.status = "queued" + att.lease_expires_at = None + + +async def mark_failed( + attachment_id: str, + error_code: str, +): + async with async_session_maker() as session: + async with session.begin(): + att = await session.get( + Attachment, + uuid.UUID(attachment_id), + with_for_update=True, + ) + + if att is None or att.status == "ready": + return + + if att.status == "processing": + raise LeaseBusyError() + + att.status = "failed" + att.error_code = error_code + att.lease_expires_at = None + + print( + f"[Worker] Файл {attachment_id}: " + f"failed, error_code={error_code}", + flush=True, + ) + + +async def process_image(image_data: bytes): + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + + input_path = directory / "original" + input_path.write_bytes(image_data) + + try: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "app.image_processor", + str(input_path), + str(directory), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except OSError as exc: + raise RetryableProcessingError( + "image_processor_start_error" + ) from exc + + try: + await asyncio.wait_for( + process.communicate(), + timeout=PROCESS_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError as exc: + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + + await process.communicate() + + raise RetryableProcessingError( + "image_processing_timeout" + ) from exc + + if process.returncode == 20: + raise PermanentProcessingError( + "image_too_many_pixels" + ) + + if process.returncode == 21: + raise PermanentProcessingError( + "image_processing_error" + ) + + if process.returncode != 0: + raise RetryableProcessingError( + "image_processor_error" + ) + + try: + thumb_bytes = ( + directory / "thumbnail.jpg" + ).read_bytes() + + crop_bytes = ( + directory / "crop.jpg" + ).read_bytes() + + webp_bytes = ( + directory / "optimized.webp" + ).read_bytes() + + with Image.open( + io.BytesIO(thumb_bytes) + ) as thumbnail: + thumb_width, thumb_height = thumbnail.size + + except OSError as exc: + raise RetryableProcessingError( + "image_processor_output_error" + ) from exc + + return ( + thumb_bytes, + crop_bytes, + webp_bytes, + thumb_width, + thumb_height, + ) + + +async def process_attachment(attachment_id: str): + print( + f"[Worker] Обработка файла: {attachment_id}", + flush=True, + ) + + claim = await claim_attachment(attachment_id) + + if claim is None: + return + + object_key, lease_expires_at = claim + + try: + try: + response = minio_client.get_object( + S3_BUCKET, + object_key, + ) + + try: + image_data = response.read( + MAX_IMAGE_BYTES + 1 + ) + finally: + response.close() + response.release_conn() + + except Exception as exc: + raise RetryableProcessingError( + "s3_download_error" + ) from exc + + if len(image_data) > MAX_IMAGE_BYTES: + raise PermanentProcessingError( + "image_too_large" + ) + + ( + thumb_bytes, + crop_bytes, + webp_bytes, + thumb_width, + thumb_height, + ) = await process_image(image_data) + + thumb_key = f"thumbnails/{object_key}" + crop_key = f"crops/{attachment_id}.jpg" + webp_key = f"webp/{attachment_id}.webp" + + try: + minio_client.put_object( + S3_BUCKET, + thumb_key, + io.BytesIO(thumb_bytes), + len(thumb_bytes), + content_type="image/jpeg", + ) + + minio_client.put_object( + S3_BUCKET, + crop_key, + io.BytesIO(crop_bytes), + len(crop_bytes), + content_type="image/jpeg", + ) + + minio_client.put_object( + S3_BUCKET, + webp_key, + io.BytesIO(webp_bytes), + len(webp_bytes), + content_type="image/webp", + ) + + except Exception as exc: + raise RetryableProcessingError( + "s3_upload_error" + ) from exc + + async with async_session_maker() as session: + async with session.begin(): + att = await session.get( + Attachment, + uuid.UUID(attachment_id), + with_for_update=True, + ) + + if ( + att is None + or att.status != "processing" + or att.lease_expires_at != lease_expires_at + ): + raise LeaseLostError() + + att.thumbnail_key = thumb_key + att.thumbnail_sha256 = hashlib.sha256( + thumb_bytes + ).hexdigest() + att.thumbnail_size_bytes = len(thumb_bytes) + att.thumbnail_width = thumb_width + att.thumbnail_height = thumb_height + att.crop_key = crop_key + att.webp_key = webp_key + att.status = "ready" + att.error_code = None + att.lease_expires_at = None + + print( + f"[Worker] Успешно обработан {attachment_id}", + flush=True, + ) + + except Exception: + await release_lease( + attachment_id, + lease_expires_at, + ) + raise + + +async def main(): + connection = await aio_pika.connect_robust( + RABBITMQ_URL + ) + + async with connection: + channel = await connection.channel( + publisher_confirms=True + ) + + await channel.set_qos( + prefetch_count=PREFETCH + ) + + queue = await channel.declare_queue( + QUEUE_NAME, + durable=True, + ) + + semaphore = asyncio.Semaphore(CONCURRENCY) + + print( + f"[Worker] CONCURRENCY={CONCURRENCY}, " + f"PREFETCH={PREFETCH}", + flush=True, + ) + + async def on_message( + message: aio_pika.IncomingMessage + ): + async with semaphore: + started = time.perf_counter() + + try: + data = json.loads( + message.body.decode() + ) + + attachment_id = str( + uuid.UUID(data["attachment_id"]) + ) + + except ( + ValueError, + KeyError, + TypeError, + ): + print( + "[Worker] Некорректное сообщение", + flush=True, + ) + + await message.reject( + requeue=False + ) + return + + headers = dict(message.headers or {}) + + try: + attempt = max( + 1, + int(headers.get("x-attempt", 1)), + ) + except (ValueError, TypeError): + attempt = 1 + + print( + f"[Worker] Попытка " + f"{attempt}/{MAX_ATTEMPTS} " + f"для {attachment_id}", + flush=True, + ) + + try: + await process_attachment( + attachment_id + ) + + except LeaseBusyError: + print( + "[Worker] Задание уже " + "обрабатывается, ждём Lease", + flush=True, + ) + + await asyncio.sleep(5) + + await message.nack( + requeue=True + ) + return + + except LeaseLostError: + print( + "[Worker] Lease перешёл " + "другому Worker", + flush=True, + ) + + await message.ack() + return + + except PermanentProcessingError as exc: + try: + await mark_failed( + attachment_id, + exc.error_code, + ) + except Exception: + await message.nack( + requeue=True + ) + return + + await message.ack() + return + + except Exception as exc: + error_code = ( + exc.error_code + if isinstance( + exc, + RetryableProcessingError, + ) + else "worker_error" + ) + + if attempt < MAX_ATTEMPTS: + next_attempt = attempt + 1 + + print( + f"[Worker] Ошибка {error_code}. " + f"Повтор " + f"{next_attempt}/{MAX_ATTEMPTS}", + flush=True, + ) + + headers["x-attempt"] = next_attempt + + base_delay = min( + 30, + 2 ** (attempt - 1), + ) + + delay = base_delay + random.uniform( + 0, + base_delay / 2, + ) + + print( + f"[Worker] Повтор через " + f"{delay:.1f} сек.", + flush=True, + ) + + await asyncio.sleep(delay) + + try: + await channel.default_exchange.publish( + aio_pika.Message( + body=message.body, + headers=headers, + delivery_mode=( + aio_pika.DeliveryMode.PERSISTENT + ), + content_type="application/json", + ), + routing_key=QUEUE_NAME, + ) + + except Exception: + await message.nack( + requeue=True + ) + return + + await message.ack() + return + + print( + f"[Worker] Попытки исчерпаны: " + f"{attachment_id}", + flush=True, + ) + + try: + await mark_failed( + attachment_id, + error_code, + ) + except Exception: + await message.nack( + requeue=True + ) + return + + await message.ack() + return + + await message.ack() + + duration = time.perf_counter() - started + rss_mb = get_rss_mb() + + print( + f"[Metrics] attachment_id={attachment_id} " + f"duration_s={duration:.3f} " + f"rss_mb={rss_mb}", + flush=True, + ) + + print( + "[Worker] Ждем сообщения из RabbitMQ...", + flush=True, + ) + + await queue.consume(on_message) + await asyncio.Future() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/backpressure_test.py b/backpressure_test.py new file mode 100644 index 00000000..7f9e74d6 --- /dev/null +++ b/backpressure_test.py @@ -0,0 +1,259 @@ +import csv +import re +import subprocess +import time +from pathlib import Path + +from benchmark_rss import command, start_worker, remove_test_worker + +RESULTS_DIR = Path("benchmark_results") +RESULTS_DIR.mkdir(exist_ok=True) + +QUEUE_NAME = "image_tasks" +COUNT_PER_SIZE = 40 +TOTAL = COUNT_PER_SIZE * 3 + + +def queue_stats(): + try: + result = subprocess.run( + [ + "docker", "exec", + "mtusi_messenger_rabbitmq", + "rabbitmqctl", "-q", "list_queues", + "name", + "messages_ready", + "messages_unacknowledged", + ], + capture_output=True, + text=True, + timeout=20, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + "RabbitMQ не ответил на запрос статистики" + ) from exc + + if result.returncode != 0: + raise RuntimeError( + "Не удалось получить статистику RabbitMQ:\n" + + result.stderr + ) + + for line in result.stdout.splitlines(): + parts = line.split() + + if len(parts) >= 3 and parts[0] == QUEUE_NAME: + return int(parts[1]), int(parts[2]) + + raise RuntimeError( + f"Очередь {QUEUE_NAME} не найдена:\n" + + result.stdout + ) + + +def run_test(): + print("Останавливаем обычный Worker...") + command(["docker", "compose", "stop", "worker"]) + + print("Запускаем Worker 1/1...") + start_worker(1, 1) + + initial_ready, initial_unacked = queue_stats() + + print( + f"Начальная очередь: ready={initial_ready}, " + f"unacked={initial_unacked}" + ) + + if initial_ready or initial_unacked: + raise RuntimeError( + "Очередь не пустая. Для чистого эксперимента " + "сначала нужно завершить предыдущие задания." + ) + + print(f"\nЗапускаем нагрузку: {TOTAL} фотографий") + + log_path = RESULTS_DIR / "backpressure_load.log" + samples = [] + + with log_path.open("w") as log_file: + process = subprocess.Popen( + [ + "docker", "compose", "exec", + "-T", + "-w", "/tmp", + "-e", "PYTHONPATH=/app", + "app1", + "python", "-m", "app.load_test", + "--count", str(COUNT_PER_SIZE), + "--timeout", "600", + ], + stdout=log_file, + stderr=subprocess.STDOUT, + ) + + started = time.monotonic() + + try: + while process.poll() is None: + ready, unacked = queue_stats() + elapsed = time.monotonic() - started + + samples.append({ + "elapsed_s": round(elapsed, 2), + "queue_ready": ready, + "unacked": unacked, + "total_pending": ready + unacked, + }) + + print( + f"[QUEUE] {elapsed:6.1f}s | " + f"ready={ready:3} | " + f"unacked={unacked} | " + f"pending={ready + unacked:3}", + flush=True, + ) + + time.sleep(1) + + return_code = process.wait() + + except BaseException: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise + + output = log_path.read_text() + + if return_code != 0: + print(output) + raise RuntimeError("Нагрузочный тест завершился с ошибкой") + + print("\n" + output) + + ready_match = re.search( + r"^Ready:\s*(\d+)", + output, + re.MULTILINE, + ) + + failed_match = re.search( + r"^Failed:\s*(\d+)", + output, + re.MULTILINE, + ) + + completed = ( + int(ready_match.group(1)) + if ready_match else -1 + ) + + failed = ( + int(failed_match.group(1)) + if failed_match else -1 + ) + + print("Проверяем окончательное состояние очереди...") + + final_ready = None + final_unacked = None + + for _ in range(15): + final_ready, final_unacked = queue_stats() + + if final_ready == 0 and final_unacked == 0: + break + + time.sleep(1) + + peak_ready = max( + (row["queue_ready"] for row in samples), + default=0, + ) + + peak_unacked = max( + (row["unacked"] for row in samples), + default=0, + ) + + csv_path = RESULTS_DIR / "backpressure_queue.csv" + + with csv_path.open("w", newline="") as file: + writer = csv.DictWriter( + file, + fieldnames=[ + "elapsed_s", + "queue_ready", + "unacked", + "total_pending", + ], + ) + + writer.writeheader() + writer.writerows(samples) + + print("\n" + "=" * 60) + print("BACKPRESSURE RESULTS") + print("=" * 60) + + print(f"Total jobs: {TOTAL}") + print(f"Ready: {completed}") + print(f"Failed: {failed}") + print(f"Peak queued: {peak_ready}") + print(f"Peak unacked: {peak_unacked}") + print(f"Final queued: {final_ready}") + print(f"Final unacked: {final_unacked}") + + backlog_observed = peak_ready > 0 + prefetch_respected = peak_unacked <= 1 + recovered = ( + completed == TOTAL + and failed == 0 + and final_ready == 0 + and final_unacked == 0 + ) + + print( + "Backlog observed:", + "YES" if backlog_observed else "NO", + ) + + print( + "Prefetch limit respected:", + "YES" if prefetch_respected else "NO", + ) + + print( + "Recovery:", + "PASSED" if recovered else "FAILED", + ) + + print(f"CSV: {csv_path}") + print("=" * 60) + + +def main(): + try: + run_test() + + finally: + print("\nВосстанавливаем обычный Worker...") + + remove_test_worker() + + result = command( + ["docker", "compose", "up", "-d", "worker"], + check=False, + ) + + if result.returncode != 0: + print(result.stderr) + + +if __name__ == "__main__": + main() diff --git a/benchmark_07ebc3e5.csv b/benchmark_07ebc3e5.csv new file mode 100644 index 00000000..5ec6e602 --- /dev/null +++ b/benchmark_07ebc3e5.csv @@ -0,0 +1,16 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +fa479f2c-54ac-44cf-baf0-e14d84b3fff2,512,ready,2.288,2.393 +8da11c17-a222-4532-888a-92cdf1bbab94,512,ready,4.895,5.104 +1441ad9e-fcc3-47c3-84e6-490559ad289a,512,ready,2.704,2.81 +a01f4d3e-8eb5-46d5-b778-cd956d9db214,512,ready,4.376,4.584 +53019d9f-949e-4aef-ab42-636d3fd330d2,512,ready,4.272,4.376 +a91a25c2-c464-4166-9c69-ec382a729bb8,1024,ready,4.584,4.895 +648e14f9-3596-44d3-870b-2c7e8357181a,1024,ready,5.419,5.835 +e127fe4e-b18d-4914-8945-3cb0aa9ef2ab,1024,ready,5.104,5.419 +4db57861-f63d-4971-8ccc-7fc1b5490f84,1024,ready,3.855,4.272 +eb1b8add-1ad5-41a9-a745-e8ee5fdaf779,1024,ready,0.831,1.249 +32dc271d-676c-4834-8575-8a273c13722b,2048,ready,0.831,2.288 +ffd1408c-cf40-4805-bf30-5a515bebd116,2048,ready,2.81,4.272 +a981c934-34b2-44fb-a09e-987c8ee66ac6,2048,ready,2.393,3.855 +5e69aead-9343-4ca7-bee9-27ae9dcbd785,2048,ready,1.249,2.704 +c0d08a45-7aa1-4afd-bd40-1d6864a4801e,2048,ready,4.272,5.627 diff --git a/benchmark_413d66b3.csv b/benchmark_413d66b3.csv new file mode 100644 index 00000000..725ee0a6 --- /dev/null +++ b/benchmark_413d66b3.csv @@ -0,0 +1,16 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +e2b24b97-ad9b-46d1-8790-2041f58914ca,512,ready,1.235,1.337 +971a55e8-3fd0-4215-b752-fff9b31ab126,512,ready,6.981,7.082 +f2e85596-fbcc-4e43-a534-d1d27cd75fc9,512,ready,7.082,7.183 +0ad8283c-968d-4525-a80f-5860f1da4b8b,512,ready,2.681,2.785 +bcdaec76-41ae-454e-b45f-174a26326b19,512,ready,1.337,1.544 +1eaccbad-b7a5-49d1-afdf-a6a137137034,1024,ready,0.821,1.235 +b78d8044-cb8f-4501-9462-7ac880cc1bc4,1024,ready,1.544,1.857 +86200ea3-a17e-4074-b1ae-90641804c48c,1024,ready,0.208,0.821 +383bbc63-331c-4510-b74b-f316c91ad902,1024,ready,1.857,2.277 +9782bfbf-9fbe-4e22-9a4b-18fa02924ccc,1024,ready,2.277,2.681 +d5594ebc-56bd-4007-b9cf-855db2f59e7f,2048,ready,4.218,5.553 +1061bf47-bde5-49a1-9971-ef41eefe499e,2048,ready,7.183,8.618 +70f8f306-73e6-4794-83fe-93c29bfd2c72,2048,ready,2.785,4.218 +c505d22f-01f2-4396-bb27-ebf2d1ff473b,2048,ready,8.618,9.958 +793328a6-cf5e-4078-814e-64d4c4e1e66e,2048,ready,5.553,6.981 diff --git a/benchmark_8742d471.csv b/benchmark_8742d471.csv new file mode 100644 index 00000000..333ca6fe --- /dev/null +++ b/benchmark_8742d471.csv @@ -0,0 +1,16 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +a940e027-a230-4343-a771-96ef998b207c,512,ready,1.155,1.364 +3a603917-37dd-4ce7-91fc-04771c87e7aa,512,ready,1.568,1.774 +1d20bc66-466e-4bd3-a754-57500694cb51,512,ready,1.467,1.568 +efc77f3e-f5d0-40da-9f33-155a0eb4db43,512,ready,1.155,1.364 +32a3e8e9-bf48-4db0-9333-5cf0cc183acc,512,ready,1.364,1.467 +ecf65392-5165-4990-bcb4-42d80628cd86,1024,ready,1.774,2.19 +b8bcc085-d1d9-411a-ac1c-08688140d4c8,1024,ready,1.155,1.568 +7eeed358-a38b-4d39-a4ac-55476bfcdcfe,1024,ready,1.364,1.774 +1e0fb527-7023-4cc2-8292-56b7bbef10c8,1024,ready,1.155,1.568 +f29de97b-d1c1-4200-b80a-4b7babdfc2d1,1024,ready,1.774,2.19 +8e2bb5c9-e613-4343-8fc1-fe6d3eda869d,2048,ready,3.647,5.102 +2038744a-177e-4a3b-bc22-85d6b82fc879,2048,ready,1.568,3.544 +748dbf01-4049-4297-837f-cb3457875e10,2048,ready,2.19,3.859 +f578c650-71b8-472b-8a0f-c6c09bcd00e3,2048,ready,2.19,4.171 +3ed1e2a1-43d5-4f9f-a475-01fa3cac932e,2048,ready,1.568,3.544 diff --git a/benchmark_results/backpressure_queue.csv b/benchmark_results/backpressure_queue.csv new file mode 100644 index 00000000..518b91a8 --- /dev/null +++ b/benchmark_results/backpressure_queue.csv @@ -0,0 +1,64 @@ +elapsed_s,queue_ready,unacked,total_pending +0.37,0,0,0 +1.74,0,0,0 +3.15,119,1,120 +4.5,118,1,119 +5.81,115,1,116 +7.16,114,1,115 +8.45,112,1,113 +9.75,111,1,112 +11.06,108,1,109 +12.37,107,1,108 +13.67,104,1,105 +14.98,101,1,102 +16.26,99,1,100 +17.55,96,1,97 +18.85,93,1,94 +20.14,92,1,93 +21.44,91,1,92 +22.74,86,1,87 +24.05,84,1,85 +25.35,82,1,83 +26.66,80,1,81 +27.98,75,1,76 +29.27,71,1,72 +30.57,68,1,69 +31.87,66,1,67 +33.18,65,1,66 +34.48,64,1,65 +35.77,63,1,64 +37.06,60,1,61 +38.43,58,1,59 +39.76,57,1,58 +41.05,57,1,58 +42.37,52,1,53 +43.67,51,1,52 +44.98,50,1,51 +46.27,49,1,50 +47.58,48,1,49 +48.87,46,1,47 +50.17,45,1,46 +51.47,44,1,45 +52.77,43,1,44 +54.08,42,1,43 +55.37,39,1,40 +56.68,38,1,39 +57.97,37,1,38 +59.26,35,1,36 +60.55,33,1,34 +61.84,31,1,32 +63.15,30,1,31 +64.44,29,1,30 +65.73,23,1,24 +67.03,22,1,23 +68.34,18,1,19 +69.63,17,1,18 +70.93,14,1,15 +72.23,12,1,13 +73.52,11,1,12 +74.82,9,1,10 +76.11,6,1,7 +77.4,5,1,6 +78.69,2,1,3 +79.98,1,1,2 +81.28,0,1,1 diff --git a/benchmark_results/benchmark_16f64efa.csv b/benchmark_results/benchmark_16f64efa.csv new file mode 100644 index 00000000..300ed0a2 --- /dev/null +++ b/benchmark_results/benchmark_16f64efa.csv @@ -0,0 +1,61 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +28bebbfc-799e-4217-8c30-4f47f1e46c7e,512,ready,5.683,5.893 +96b8fb1e-3ca4-497e-a4b0-0d1f73522c44,512,ready,10.223,10.328 +ea372b62-a051-4cc1-afc5-dde4fdfdb85f,512,ready,5.893,6.098 +6d4cae25-8487-401b-b89f-16fd3e012079,512,ready,6.098,6.201 +42bcd40d-5de7-42d9-9cac-511bc000160f,512,ready,4.137,4.241 +f6c50369-fcfa-48c4-b04d-8599c2c20711,512,ready,5.376,5.478 +f55f9964-9adc-4adc-b48e-9c93d4019889,512,ready,5.893,6.098 +b3ea2331-706e-4dd1-81ae-8972c922075f,512,ready,4.137,4.241 +2f0d8af7-ee6f-479a-96b9-c24c311b1790,512,ready,9.709,9.812 +9fca7598-037f-4f6a-81dd-038bc8f5c9d3,512,ready,3.93,4.137 +3c1673ca-f03b-41d4-8706-f8a65a1951af,512,ready,2.901,3.003 +0eaad040-3f62-446d-842e-fc4a21cb74cc,512,ready,5.893,6.098 +ee0404fd-8dd6-49dd-9a77-e849a0382ff2,512,ready,0.843,1.048 +20f6197b-e3fb-4ce8-9985-13aa260a3e9f,512,ready,5.478,5.683 +9d401a3c-d5d5-45b9-9153-8a392132c74a,512,ready,5.683,5.893 +b2de90a9-4354-4c66-b905-c2cd501e94b2,512,ready,3.829,4.034 +ed1ded18-31cc-495e-9e1e-7a15cb77fb67,512,ready,5.683,5.893 +a8e31c1d-0ef1-4fea-a538-3fd9e36a594c,512,ready,6.201,6.406 +5451263a-8d6b-4940-b44b-87c1bbb66b97,512,ready,8.671,8.773 +0af8a964-e248-4153-936a-7bd5f7ec9c82,512,ready,8.049,8.259 +c6ab01d9-cb1e-4154-8678-77453b549a82,1024,ready,9.812,10.223 +b3f6ab04-bfe9-45c3-a409-d8aed2efba92,1024,ready,5.893,6.303 +de9ed0d6-8d6d-41a4-8eb4-16eb6793eca3,1024,ready,9.297,9.709 +452d6dce-5947-4d08-8983-37b9b517ea0a,1024,ready,8.362,8.773 +91f662ce-6cec-4586-b2a0-1a5cc101340b,1024,ready,1.048,1.46 +b41f3d80-1b54-4402-9928-a83e67e2aa2f,1024,ready,2.491,2.901 +4b7cfdbf-8bdc-4569-96dc-8410b0fb965d,1024,ready,3.003,3.412 +19df5c43-84d5-4fc6-8469-daf469f8d096,1024,ready,1.46,1.975 +c5211831-85d2-4c44-a871-9acd71a17337,1024,ready,8.259,8.671 +369d13b1-ead7-4aed-bcd4-9f12e918d038,1024,ready,4.034,4.452 +7e9c5099-2f5e-4ea9-a6e2-d4acc540901f,1024,ready,7.946,8.362 +927b907a-fbc4-4383-a6e3-0fd02206f79c,1024,ready,6.098,6.51 +d96167a6-cda0-46a8-adad-df52ccc789bd,1024,ready,9.709,10.121 +e9577c3a-8d88-46c9-9ad5-1979469b38be,1024,ready,0.843,1.356 +52f8e432-ff40-4419-8968-17ef1089a361,1024,ready,4.966,5.376 +99a1e17e-94b0-4ca1-810f-fa5e3ff9691f,1024,ready,3.31,3.726 +33eb7872-0980-4336-a560-cf6f14a302f9,1024,ready,9.297,9.709 +4a3cacba-350f-4878-85f7-ead76d7cacbe,1024,ready,3.726,4.137 +fc3cb344-8f7c-4421-bae0-ce5e282d6143,1024,ready,1.975,2.39 +22228d29-952b-4ec0-aad0-b8762967dee0,1024,ready,1.356,1.768 +1d29e426-37fa-41e4-b5cc-55eedbf2af38,2048,ready,8.773,10.223 +d62cc431-925c-47c6-9e0a-bec4c2d561dc,2048,ready,0.843,2.491 +4d3d5437-fded-4c5a-9ab5-50460df9fade,2048,ready,4.452,5.893 +78668df5-05d3-4ca1-b309-fcb16181d973,2048,ready,4.241,5.683 +86af36aa-2193-4da2-a786-1787fb6229d5,2048,ready,2.39,3.829 +211fbe0b-a085-4549-812a-07a2c326a9b2,2048,ready,6.51,8.049 +f8e04547-8aaa-4b87-9e9d-383e387ade00,2048,ready,0.843,2.491 +9b065c78-8c00-4fe5-a890-bfa698c8bfea,2048,ready,4.241,5.683 +3906bf23-8afc-47d1-85c5-b2d0eb207236,2048,ready,10.223,11.673 +4a2f285d-327c-44be-ba2c-da0d627109cc,2048,ready,6.406,7.946 +f1f72a39-92e6-4284-a373-30214f1141bc,2048,ready,10.223,11.673 +9eee27d0-b3fb-4be4-ab0f-3f99b0d024f1,2048,ready,6.098,7.741 +cb946763-c80a-4284-85b3-3aea16d2eced,2048,ready,10.223,11.673 +f0aed629-1d01-4f7a-a699-eff85d4dd239,2048,ready,7.844,9.297 +b7957cd4-0942-40cd-a1f9-a4a614d4c5f3,2048,ready,8.773,10.223 +ea48ce0d-a3e8-44f3-bdbc-37d1ddfc9eb0,2048,ready,7.844,9.297 +e6a26bb8-4c70-4074-b130-357a742c6b5f,2048,ready,1.768,3.31 +1530b7a5-227c-4bb0-ad09-d975105a757c,2048,ready,3.412,4.864 +e9bcf42c-2a3b-43ee-8802-3d1432bfba5a,2048,ready,6.303,7.844 +390b5116-460d-45f6-976b-240c2b6066bf,2048,ready,2.491,3.93 diff --git a/benchmark_results/benchmark_7499bd42.csv b/benchmark_results/benchmark_7499bd42.csv new file mode 100644 index 00000000..50151b88 --- /dev/null +++ b/benchmark_results/benchmark_7499bd42.csv @@ -0,0 +1,61 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +00eb6bd3-3102-44e1-9be4-a5194b56cbd5,512,ready,2.814,3.021 +cde0e2c2-02c2-4c05-bf05-6a98d0acd71a,512,ready,17.363,17.464 +62fe77fe-bdf2-4dbb-85c7-9ff6981bc5f5,512,ready,16.524,16.626 +36b35f4e-5497-4165-b0f6-bb3724e9cd00,512,ready,11.442,11.544 +7f4286a2-d91e-40fb-8231-bd7674cfa8bc,512,ready,7.172,7.274 +f34c5f28-536f-45b6-abf3-88a8229dd945,512,ready,3.021,3.123 +1caa346a-80e8-44a7-9dfb-1e83bf71534e,512,ready,11.337,11.442 +cf10641f-fbb8-43ea-88e0-22daa1627188,512,ready,2.095,2.198 +98099d7b-724d-4e70-a4cf-9a6ab019333b,512,ready,12.906,13.111 +b26e4907-1919-4dce-a7da-6909890b2d5f,512,ready,3.854,3.958 +e3468d1b-5637-4e5a-aa70-0249559b4cfd,512,ready,2.712,2.814 +2f9f2e4a-d4e9-47d6-8c87-25043e5199ae,512,ready,1.89,2.095 +de8fbd2e-e392-4663-98a1-96d93542280f,512,ready,11.442,11.544 +9d3f5e09-c41f-4018-bdab-175b00a452e6,512,ready,7.274,7.484 +e56365eb-2d10-4d3c-ad32-931f8a31d030,512,ready,2.608,2.712 +4bc57fb6-32c7-47ca-8899-ea40df7da799,512,ready,11.134,11.337 +d2522db6-f17a-4e18-9d79-386a195fd633,512,ready,3.539,3.641 +6d1cc5f9-d992-44cf-9ed0-530467774337,512,ready,4.368,4.572 +985e8927-4f0b-44fa-bd09-fb19d6ea29fe,512,ready,11.235,11.442 +50c58946-727b-48d2-9809-3f7a390ea741,512,ready,3.641,3.854 +39ee9ce0-1494-4f3a-ab1f-7c03dcc546b2,1024,ready,1.058,1.578 +9778715e-f4c6-4fa0-8c3c-64e621611fda,1024,ready,3.641,4.059 +72ba59cb-3e56-483b-b65d-c53e0cf7c302,1024,ready,11.544,12.065 +cb97fe45-a6b2-43c8-bd1b-b51824014994,1024,ready,4.572,4.885 +df00bd9f-2470-4390-ace0-920e3f206b97,1024,ready,10.723,11.134 +9694c476-cb8a-46b8-9e8d-57cba2152a17,1024,ready,14.336,14.745 +42169eb9-3324-4c2a-99ca-36148faaa150,1024,ready,13.111,13.519 +6c26553f-30ba-4f18-9c3c-d08472afc7f0,1024,ready,18.499,18.913 +8315304d-7550-4eb9-82d8-0db4db41aa04,1024,ready,3.123,3.539 +d8ee601e-6b75-417c-affa-1b24b1c1a02a,1024,ready,6.76,7.172 +e4536000-5159-486f-97a3-8e0727854ab2,1024,ready,6.341,6.76 +77809b9b-ae97-41b3-9463-cbe70acf2a0c,1024,ready,13.519,13.926 +757b621c-fb42-499d-b4dd-721d9a9e876a,1024,ready,1.578,1.89 +18ba2c34-56eb-46ec-a54b-20b2d2752ac5,1024,ready,12.065,12.485 +75ebd991-3525-459a-a1dd-c71c6e78663b,1024,ready,13.926,14.336 +5b5a607f-76b6-48f7-ba9e-09d9aa732cf4,1024,ready,3.958,4.368 +813b7fe0-f445-4420-a8ab-757046d266b6,1024,ready,12.485,12.906 +19fb3f65-760e-4e53-bd98-dc51dcbf237f,1024,ready,16.733,17.056 +1ff822e0-69ae-461e-9df3-c27261f9009a,1024,ready,7.484,7.901 +b24dbee4-e1c5-4a18-8f62-3f5012559cd4,1024,ready,16.103,16.524 +baa5cea0-5554-4a32-90a6-e3ad6c204228,2048,ready,8.428,9.891 +c58ddcd7-ffa3-46f6-b3dc-ea8f0fe9c1f4,2048,ready,18.913,20.363 +c4636200-eae1-49d4-8e59-f5dfa5b86c36,2048,ready,9.891,11.235 +c5d2e767-ce88-4682-82f5-80b9ab209580,2048,ready,4.059,5.504 +bf65fcbd-3ec1-4851-ab11-7f9239cdf1e5,2048,ready,7.901,9.268 +04533086-f63d-4aa8-9926-fab91e5ebf52,2048,ready,15.894,17.363 +7d88df73-ae6b-4e4a-821e-3de54784e87c,2048,ready,2.198,3.641 +06bc64bf-f949-48a1-a0d4-61d8e2053523,2048,ready,14.745,16.103 +7751ba6d-1bd3-48a1-91dd-6f9d837e37f2,2048,ready,4.885,6.341 +d3a36f16-3a76-43a4-9bc3-b806fb264ae1,2048,ready,14.541,15.894 +e9daef68-894e-4433-9004-8e515f539203,2048,ready,9.268,10.723 +f1afb046-e7d2-46e3-a639-bb9abd841039,2048,ready,20.363,21.721 +29d51f63-9f4f-4f80-b493-7c55823c6015,2048,ready,13.111,14.541 +0770bd79-18dc-4c61-8daf-5eddebac520f,2048,ready,1.058,2.608 +eb6b54b5-c1ab-4f7a-a527-78dcb2c5cbac,2048,ready,5.504,6.964 +09f1c7ae-0050-437e-ad85-4a416b813010,2048,ready,6.964,8.428 +939d499e-3d5e-4e86-8e08-19dc70d73b18,2048,ready,11.544,13.111 +64942f86-b40b-4aa5-99f3-b64e2617ddfc,2048,ready,18.913,20.363 +37210ed4-847c-48f1-be14-924231466055,2048,ready,17.464,18.913 +e1506996-5a46-42af-8083-dd815d6c2050,2048,ready,17.056,18.499 diff --git a/benchmark_results/benchmark_be804c9e.csv b/benchmark_results/benchmark_be804c9e.csv new file mode 100644 index 00000000..c24e1bb3 --- /dev/null +++ b/benchmark_results/benchmark_be804c9e.csv @@ -0,0 +1,61 @@ +attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s +6d0948d1-018f-4649-bc93-3839a19d960b,512,ready,32.346,32.448 +821f8778-92d6-4570-a1f3-f2d0bc20a4bc,512,ready,7.948,8.055 +2c44d62f-bbaf-4443-95ae-9025bd84f48d,512,ready,11.024,11.132 +a5bbe073-3027-42ae-98a4-2e93451a7895,512,ready,10.819,11.024 +c6314a8d-0a91-4a9d-a466-ae1959418c44,512,ready,20.047,20.151 +93fe6aee-0a67-4186-b5de-a7c8911c9c92,512,ready,4.143,4.347 +0ca9a2cc-d750-4f9f-96ac-e1c26b14c751,512,ready,30.364,30.573 +58b114fd-2289-405a-b1b1-aeed97690be3,512,ready,14.81,15.014 +38d4dc82-c0af-4a2e-ba4d-5e83c7d39bb6,512,ready,15.014,15.118 +f4b6d8b6-0abf-4de9-92bc-79427df63c1b,512,ready,15.731,15.937 +53992b21-20c3-434d-b4c3-4375f50a2d5a,512,ready,20.668,20.873 +65e871d7-7b80-43e2-84a2-01c77dc4b0ef,512,ready,15.527,15.629 +0284edbb-fbde-481f-971f-ee483f41da53,512,ready,36.524,36.738 +8244a394-feef-493a-acad-784ee2f63985,512,ready,18.098,18.2 +289ecaf9-2ea5-42ed-9757-aaf20eace246,512,ready,38.816,39.023 +10863feb-0bdf-426c-8827-e779e397a2c9,512,ready,15.629,15.731 +9fa71750-b870-47e3-b95f-f814fcc2ec49,512,ready,33.289,33.391 +abac43bc-c866-4c99-bd32-7e7b91ee54ef,512,ready,20.563,20.668 +d6072440-3839-420d-b74d-98683f8ed80a,512,ready,3.628,3.732 +5c0b5e43-e8df-4e6c-8774-df2698340e00,512,ready,14.708,14.81 +e4cb92cf-7f15-4598-b8db-e2bb195cf897,1024,ready,30.573,30.99 +09390dfc-b29a-4746-8353-7355b93cd34e,1024,ready,39.023,39.336 +65464b2b-1e23-464b-86c5-f506248392a9,1024,ready,28.19,28.608 +6326723a-8739-4afc-91b1-c5f86dcc883d,1024,ready,24.988,25.399 +c0528510-a4e8-4b6b-a5fd-a8510730a91c,1024,ready,18.2,18.614 +353e643e-0d25-4ee5-a18d-9101a32049f2,1024,ready,20.151,20.563 +9d6d2329-d9bc-4299-a1ab-d6b27881457c,1024,ready,32.872,33.289 +b68909e6-f9f2-466f-9938-9e8fd125758f,1024,ready,7.539,7.948 +df4a4574-25ae-4eca-b967-0bd8f5a85aea,1024,ready,17.268,17.689 +2e704f53-ae9c-4948-bd09-b8110f935baf,1024,ready,33.391,33.806 +1a17e811-2645-49aa-8ad9-11e7d499b999,1024,ready,4.347,4.655 +b1f5e98c-6587-4fdb-aee9-ccb8c6cd6f99,1024,ready,28.608,29.021 +b7abcbbc-3ed9-4e85-b5e1-5d90b132034d,1024,ready,15.118,15.527 +3e3a19a7-5485-45f4-a8a4-387f9002e54c,1024,ready,3.732,4.143 +73aad940-7807-4f21-a83e-8ba1cd15863e,1024,ready,36.738,37.048 +015511fd-3538-47d9-b7a0-f14969501967,1024,ready,38.509,38.816 +cd33c6d5-6913-4e21-a36c-9f13ca3c7bce,1024,ready,12.46,12.869 +2a2915a9-a211-4ac3-bdbf-a06b6bccbb04,1024,ready,17.689,18.098 +9084e41a-d7cb-450b-8efb-0110b39d20a8,1024,ready,32.448,32.872 +47d491a2-8f43-4b09-86be-939ed73d3216,1024,ready,12.869,13.277 +ac499ba7-af23-4626-8386-18dd34b7d0cb,2048,ready,15.937,17.268 +361ac439-f549-41ce-ba31-f0fdcc3e5a9d,2048,ready,6.097,7.539 +f4831394-fdbe-421e-99b1-3ebb9d635f7a,2048,ready,8.055,9.487 +e7c14f44-5b76-4950-a77c-a2d8edcd14a1,2048,ready,18.614,20.047 +d40b2c16-d32c-4244-94f5-255d77d0d189,2048,ready,37.048,38.509 +c61f7efe-e5e4-49fd-ab9d-b4d813d42697,2048,ready,35.169,36.524 +3b1bfd22-01e8-4874-8ea2-7716785f88e1,2048,ready,4.655,6.097 +21fa4d34-4dcc-487c-9205-ab1e48d05d09,2048,ready,22.221,23.658 +b169f1bf-5ede-447e-aa4d-c7886b80e6a8,2048,ready,33.806,35.169 +2c9a01c7-e488-4ca6-bc70-19e7b20d91a6,2048,ready,13.277,14.708 +8f439608-d82f-4050-b939-5c35d9938eb2,2048,ready,0.735,2.179 +fe635dd1-5fc7-4c82-a63a-0819dcac246c,2048,ready,23.658,24.988 +1be24fbc-bf40-4883-9029-6075d2f4c557,2048,ready,30.99,32.346 +ba19044d-d21c-4706-a263-240fdafb5749,2048,ready,26.843,28.19 +52e7cc24-f8a3-443b-b62d-93c143ce75a7,2048,ready,2.179,3.628 +97f34f85-f51f-4c8c-8f27-ae573690f1c8,2048,ready,29.021,30.364 +4228d8dd-5da4-4532-a132-9b27308cc1e7,2048,ready,25.399,26.843 +d58514f2-17f8-481e-b5c5-ae054324f80e,2048,ready,20.873,22.221 +51f3190e-f63b-4b0e-880c-947089cdfef9,2048,ready,9.487,10.819 +e5de7654-15d1-4f47-9f22-d2cab4c76737,2048,ready,11.132,12.46 diff --git a/benchmark_results/rss_1_1.csv b/benchmark_results/rss_1_1.csv new file mode 100644 index 00000000..6302a952 --- /dev/null +++ b/benchmark_results/rss_1_1.csv @@ -0,0 +1,74 @@ +elapsed_s,rss_mb +0.0018562499899417162,63.9609375 +0.5586906669195741,63.9609375 +1.1478328749071807,63.9609375 +1.7124463329091668,63.9609375 +2.3201119999866933,154.03125 +2.886150083038956,181.53125 +3.454367541940883,208.90625 +4.0352518749423325,172.203125 +4.596314958063886,199.328125 +5.158594999928027,94.390625 +5.731980707962066,90.484375 +6.294250082923099,160.44921875 +6.866898791864514,187.32421875 +7.449180916883051,214.32421875 +8.02075345790945,175.76953125 +8.586141041945666,203.89453125 +9.151447457959875,111.85546875 +9.7154242079705,164.8046875 +10.274698958033696,192.0546875 +10.821836207993329,214.4296875 +11.391208708053455,178.421875 +11.957092249998823,206.921875 +12.530426000012085,95.53515625 +13.091126542072743,180.48828125 +13.658847207902,208.73828125 +14.223303207894787,115.96875 +14.79374916688539,122.56640625 +15.352951207896695,183.80078125 +15.913661292055622,212.42578125 +16.488913499983028,94.2734375 +17.04698033304885,89.0625 +17.61788404197432,166.3125 +18.182629582937807,194.5625 +18.746452332939953,73.2265625 +19.316664916928858,110.76953125 +19.88158891699277,112.84765625 +20.447920500067994,173.1171875 +21.019828042015433,200.3671875 +21.583593416959047,95.69921875 +22.148048375034705,68.84375 +22.701521083014086,174.2734375 +23.266354499850422,202.7734375 +23.836700625019148,161.3046875 +24.401869332883507,188.8046875 +24.96454008296132,214.8046875 +25.525170125067234,175.265625 +26.084088041912764,203.515625 +26.649612499866635,111.0 +27.214416875038296,170.390625 +27.77859991695732,198.640625 +28.34497074992396,122.6796875 +28.900013708043844,184.03515625 +29.459034791914746,212.28515625 +30.019986792001873,106.3125 +30.584227999905124,160.7734375 +31.149346332997084,187.8984375 +31.712559124920517,214.6484375 +32.281295999884605,116.64453125 +32.84523691702634,176.890625 +33.407698791939765,204.765625 +33.96293962490745,104.8046875 +34.52196699986234,113.38671875 +35.08372745802626,116.0078125 +35.665793374879286,176.3828125 +36.23321974999271,204.6328125 +36.79392512491904,162.4609375 +37.351651957957074,190.7109375 +37.91607549996115,214.7109375 +38.483901750063524,118.4921875 +39.04901108285412,178.9453125 +39.61435758299194,207.6953125 +40.18525333306752,116.02734375 +40.75472962507047,117.7734375 diff --git a/benchmark_results/rss_2_2.csv b/benchmark_results/rss_2_2.csv new file mode 100644 index 00000000..f5f2755b --- /dev/null +++ b/benchmark_results/rss_2_2.csv @@ -0,0 +1,43 @@ +elapsed_s,rss_mb +0.006153792142868042,63.90234375 +0.5778734171763062,63.90234375 +1.147869042120874,63.90234375 +1.7034579580649734,63.90234375 +2.276205125031993,63.90234375 +2.8752599169965833,194.703125 +3.4416670829523355,230.83203125 +3.9975552500691265,302.11328125 +4.574312333017588,213.53515625 +5.14147137501277,263.47265625 +5.7091500831302255,110.90625 +6.2797843751031905,220.01171875 +6.835572292096913,304.51953125 +7.403642917051911,289.5234375 +7.963550708023831,196.44921875 +8.53717741696164,258.8515625 +9.096077542053536,186.86328125 +9.66255066706799,292.37890625 +10.221317625138909,277.51171875 +10.789583917008713,333.26171875 +11.363086167024449,317.81640625 +11.92844095802866,303.1171875 +12.479885042179376,243.38671875 +13.050684583140537,94.1015625 +13.61427737516351,182.80078125 +14.231398458126932,240.96484375 +14.796835124958307,197.81640625 +15.373798333108425,231.953125 +15.93250362505205,221.671875 +16.498262125067413,263.9453125 +17.066447000019252,319.6953125 +17.630022125085816,269.69140625 +18.18958625011146,206.35546875 +18.755617167102173,245.66015625 +19.313573125051335,277.03125 +19.88116762507707,332.53125 +20.441875083139166,269.94140625 +21.01258633309044,281.08984375 +21.573833167087287,337.21484375 +22.150734583148733,166.046875 +22.70690429210663,193.421875 +23.275886042043567,218.296875 diff --git a/benchmark_results/rss_4_4.csv b/benchmark_results/rss_4_4.csv new file mode 100644 index 00000000..eb6695d1 --- /dev/null +++ b/benchmark_results/rss_4_4.csv @@ -0,0 +1,25 @@ +elapsed_s,rss_mb +0.0017751660197973251,63.90234375 +0.5725909580942243,63.90234375 +1.146317790960893,63.90234375 +1.712353958049789,63.90234375 +2.2800673749297857,63.90234375 +2.886254500132054,342.91015625 +3.4468102499376982,361.43359375 +4.018739791121334,518.78515625 +4.595743082929403,414.22265625 +5.1572650000452995,385.99609375 +5.722510583000258,270.40625 +6.291970041114837,495.10546875 +6.863853875081986,489.14453125 +7.440310999983922,251.58203125 +7.994805958122015,232.55078125 +8.561554083134979,507.96484375 +9.145290333079174,536.33984375 +9.71233612508513,399.1328125 +10.283997416030616,374.50390625 +10.863315333146602,531.59375 +11.457262207986787,337.00390625 +12.02924433303997,245.5234375 +12.59735562512651,383.62109375 +13.175934208091348,467.24609375 diff --git a/benchmark_results/summary.csv b/benchmark_results/summary.csv new file mode 100644 index 00000000..da907315 --- /dev/null +++ b/benchmark_results/summary.csv @@ -0,0 +1,4 @@ +config,total,ready,failed,duration_s,throughput,peak_processing,peak_rss_mb,mean_rss_mb +1/1,60,60,0,39.34,1.53,1.0,214.8,155.83 +2/2,60,60,0,21.72,2.76,2.0,337.21,224.02 +4/4,60,60,0,11.67,5.14,4.0,536.34,327.69 diff --git a/benchmark_rss.py b/benchmark_rss.py new file mode 100644 index 00000000..1422d809 --- /dev/null +++ b/benchmark_rss.py @@ -0,0 +1,297 @@ +import csv +import re +import subprocess +import time +from pathlib import Path + +CONFIGS = [(1, 1), (2, 2), (4, 4)] +COUNT_PER_SIZE = 20 +RESULTS_DIR = Path("benchmark_results") +RESULTS_DIR.mkdir(exist_ok=True) + +RSS_CODE = """ +import os + +total_kb = 0 +current_pid = os.getpid() + +for name in os.listdir("/proc"): + if not name.isdigit(): + continue + + if int(name) == current_pid: + continue + + try: + with open(f"/proc/{name}/status") as file: + for line in file: + if line.startswith("VmRSS:"): + total_kb += int(line.split()[1]) + break + except (OSError, ValueError): + pass + +print(total_kb / 1024) +""" + + +def command(args, check=True): + return subprocess.run( + args, + text=True, + capture_output=True, + check=check, + ) + + +def remove_test_worker(): + command( + ["docker", "rm", "-f", "benchmark-worker"], + check=False, + ) + + +def start_worker(concurrency, prefetch): + remove_test_worker() + + result = command([ + "docker", "compose", "run", + "-d", + "--name", "benchmark-worker", + "--no-deps", + "-e", f"WORKER_CONCURRENCY={concurrency}", + "-e", f"WORKER_PREFETCH={prefetch}", + "worker", + ]) + + expected = ( + f"[Worker] CONCURRENCY={concurrency}, " + f"PREFETCH={prefetch}" + ) + + for _ in range(30): + logs = command( + ["docker", "logs", "benchmark-worker"], + check=False, + ) + + if expected in logs.stdout + logs.stderr: + print(expected) + return + + time.sleep(1) + + raise RuntimeError( + f"Worker {concurrency}/{prefetch} не запустился.\n" + + command( + ["docker", "logs", "benchmark-worker"], + check=False, + ).stderr + ) + + +def measure_rss(): + result = command( + [ + "docker", "exec", + "benchmark-worker", + "python", "-c", RSS_CODE, + ], + check=False, + ) + + if result.returncode != 0: + return None + + try: + return float(result.stdout.strip()) + except ValueError: + return None + + +def extract_number(text, label): + match = re.search( + rf"^{re.escape(label)}:\s*([0-9.]+)", + text, + re.MULTILINE, + ) + return float(match.group(1)) if match else None + + +def run_benchmark(concurrency, prefetch): + config = f"{concurrency}_{prefetch}" + + print(f"\n{'=' * 55}") + print(f"ЭКСПЕРИМЕНТ: {concurrency}/{prefetch}") + print(f"{'=' * 55}") + + start_worker(concurrency, prefetch) + + log_path = RESULTS_DIR / f"load_{config}.log" + rss_samples = [] + + load_command = [ + "docker", "compose", "exec", + "-T", + "-w", "/tmp", + "-e", "PYTHONPATH=/app", + "app1", + "python", "-m", "app.load_test", + "--count", str(COUNT_PER_SIZE), + "--timeout", "300", + ] + + print("Запускаем 60 фотографий...") + start_time = time.monotonic() + + with log_path.open("w") as log_file: + process = subprocess.Popen( + load_command, + stdout=log_file, + stderr=subprocess.STDOUT, + ) + + while process.poll() is None: + elapsed = time.monotonic() - start_time + rss = measure_rss() + + if rss is not None: + rss_samples.append((elapsed, rss)) + + time.sleep(0.5) + + return_code = process.wait() + + output = log_path.read_text() + + if return_code != 0: + print(output) + raise RuntimeError( + f"Нагрузочный тест {concurrency}/{prefetch} завершился с ошибкой" + ) + + print(output) + + if not rss_samples: + raise RuntimeError("Не удалось измерить RSS") + + rss_values = [value for _, value in rss_samples] + peak_rss = max(rss_values) + mean_rss = sum(rss_values) / len(rss_values) + + samples_path = RESULTS_DIR / f"rss_{config}.csv" + + with samples_path.open("w", newline="") as file: + writer = csv.writer(file) + writer.writerow(["elapsed_s", "rss_mb"]) + writer.writerows(rss_samples) + + csv_match = re.search( + r"CSV:\s*(benchmark_[a-zA-Z0-9]+\.csv)", + output, + ) + + if csv_match: + filename = csv_match.group(1) + + copy_result = command( + [ + "docker", "compose", "cp", + f"app1:/tmp/{filename}", + str(RESULTS_DIR / filename), + ], + check=False, + ) + + if copy_result.returncode != 0: + print("Не удалось скопировать CSV:", copy_result.stderr) + + result = { + "config": f"{concurrency}/{prefetch}", + "total": int(extract_number(output, "Total jobs") or 0), + "ready": int(extract_number(output, "Ready") or 0), + "failed": int(extract_number(output, "Failed") or 0), + "duration_s": extract_number(output, "Duration"), + "throughput": extract_number(output, "Throughput"), + "peak_processing": extract_number( + output, "Peak observed processing" + ), + "peak_rss_mb": round(peak_rss, 2), + "mean_rss_mb": round(mean_rss, 2), + } + + print(f"\n[RSS] Пиковое значение: {peak_rss:.2f} MB") + print(f"[RSS] Среднее значение: {mean_rss:.2f} MB") + print(f"[RSS] Количество замеров: {len(rss_samples)}") + + return result + + +def save_summary(results): + output = RESULTS_DIR / "summary.csv" + + with output.open("w", newline="") as file: + writer = csv.DictWriter( + file, + fieldnames=[ + "config", + "total", + "ready", + "failed", + "duration_s", + "throughput", + "peak_processing", + "peak_rss_mb", + "mean_rss_mb", + ], + ) + + writer.writeheader() + writer.writerows(results) + + print("\n" + "=" * 70) + print("ИТОГОВАЯ ТАБЛИЦА") + print("=" * 70) + + for item in results: + print( + f"{item['config']:5} | " + f"Ready: {item['ready']:3} | " + f"Time: {item['duration_s']} s | " + f"Throughput: {item['throughput']} img/s | " + f"Peak RSS: {item['peak_rss_mb']} MB" + ) + + print(f"\nРезультаты сохранены в {RESULTS_DIR}/") + + +def main(): + results = [] + + try: + print("Останавливаем обычный Worker...") + command(["docker", "compose", "stop", "worker"]) + + for concurrency, prefetch in CONFIGS: + result = run_benchmark(concurrency, prefetch) + results.append(result) + + finally: + print("\nВосстанавливаем обычный Worker...") + remove_test_worker() + + restore = command( + ["docker", "compose", "up", "-d", "worker"], + check=False, + ) + + if restore.returncode != 0: + print("Ошибка восстановления Worker:") + print(restore.stderr) + + if results: + save_summary(results) + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..feb9eb9e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,197 @@ +services: + db: + image: postgres:16-alpine + container_name: mtusi_messenger_db + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: postgres -c synchronous_commit=off + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: mtusi_messenger_redis + command: redis-server --appendonly yes + volumes: + - redis_data:/data + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + + app1: + build: . + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS} + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + volumes: + - ./app:/app/app + - ./alembic:/app/alembic + - ./alembic.ini:/app/alembic.ini + + app2: + build: . + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS} + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + volumes: + - ./app:/app/app + - ./alembic:/app/alembic + - ./alembic.ini:/app/alembic.ini + + rabbitmq: + image: rabbitmq:3.12-management-alpine + container_name: mtusi_messenger_rabbitmq + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - "127.0.0.1:5672:5672" + - "127.0.0.1:15672:15672" + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: [ "CMD", "rabbitmq-diagnostics", "-q", "ping" ] + interval: 10s + timeout: 5s + retries: 5 + + worker: + build: . + mem_limit: 512m + command: python -m app.worker + environment: + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET} + RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/ + depends_on: + db: + condition: service_healthy + minio: + condition: service_healthy + rabbitmq: + condition: service_healthy + volumes: + - ./app:/app/app + - ./scripts:/app/scripts + + outbox: + build: . + command: python -m app.outbox_dispatcher + restart: unless-stopped + environment: + DATABASE_URL: ${DATABASE_URL} + RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/ + depends_on: + db: + condition: service_healthy + rabbitmq: + condition: service_healthy + volumes: + - ./app:/app/app + + nginx: + image: nginx:alpine + container_name: mtusi_messenger_nginx + ports: + - "127.0.0.1:8080:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - app1 + - app2 + + minio: + image: pgsty/silo:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio_data:/data + ports: + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ] + interval: 5s + timeout: 5s + retries: 5 + + minio-init: + image: pgsty/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb --ignore-existing myminio/${S3_BUCKET}; + /usr/bin/mc anonymous set none myminio/${S3_BUCKET}; + /usr/bin/mc admin user add myminio api-user api-user-secret-2026 || true; + /usr/bin/mc admin policy attach myminio readwrite --user api-user || true; + exit 0; + " + +volumes: + postgres_data: + redis_data: + minio_data: + rabbitmq_data: \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..048bde26 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,27 @@ +events { worker_connections 1024; } + +http { + upstream api_backend { + least_conn; + server app1:8080 max_fails=2 fail_timeout=5s; + server app2:8080 max_fails=2 fail_timeout=5s; + } + + server { + listen 80; + + client_max_body_size 10M; + + location / { + proxy_pass http://api_backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_next_upstream error timeout http_502 http_503 http_504; + + proxy_connect_timeout 2s; + proxy_send_timeout 5s; + proxy_read_timeout 5s; + } + } +} diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..9d511cbc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,19 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +pytest==8.3.4 +fastapi==0.115.0 +uvicorn[standard]==0.52.4 +pydantic==2.13.5 +asyncpg==0.31.0 +sqlalchemy[asyncio]==2.1.0 +redis==6.2.0 +passlib==1.7.4 +bcrypt==4.1.2 +alembic==1.14.0 +psycopg2-binary +filetype==1.2.0 +Pillow==10.4.0 +minio==7.2.9 +python-multipart==0.0.9 +aio-pika==9.4.0 +Pillow==10.4.0 diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/schemas/chat.py b/schemas/chat.py new file mode 100644 index 00000000..97786751 --- /dev/null +++ b/schemas/chat.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, Field, field_validator, ConfigDict + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v): + if len(set(v)) != len(v): + raise ValueError("member_ids должны быть уникальными") + return v \ No newline at end of file diff --git a/schemas/message.py b/schemas/message.py new file mode 100644 index 00000000..6e8830b5 --- /dev/null +++ b/schemas/message.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel, Field, ConfigDict + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=1, max_length=4000) \ No newline at end of file diff --git a/schemas/user.py b/schemas/user.py new file mode 100644 index 00000000..81888ef1 --- /dev/null +++ b/schemas/user.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel, Field, ConfigDict + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) \ No newline at end of file diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/failover-test.sh b/scripts/failover-test.sh new file mode 100755 index 00000000..e7b7b199 --- /dev/null +++ b/scripts/failover-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +REPLICA="${1:-app1}" + +echo "Останавливаю $REPLICA" +date +"%H:%M:%S.%3N" +./scripts/stop-one.sh "$REPLICA" + +echo "" +echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id" +for i in {1..20}; do + TS=$(date +"%H:%M:%S.%3N") + CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live) + INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r') + echo "$TS code=$CODE $INST" + sleep 0.5 +done + +echo "" +echo "Поднимаю $REPLICA обратно" +docker compose start "$REPLICA" \ No newline at end of file diff --git a/scripts/load-50.py b/scripts/load-50.py new file mode 100644 index 00000000..3dac0cf8 --- /dev/null +++ b/scripts/load-50.py @@ -0,0 +1,76 @@ +import io +import os +import time +import uuid +import requests +from PIL import Image + +BASE_URL = "http://localhost:8080" +COUNT = 50 + + +def make_png(width, height): + img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + buf = io.BytesIO() + img.save(buf, format="PNG", optimize=False) + return buf.getvalue() + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Cost Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def upload(token, chat_id, content, filename): + headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers=headers, data=content, timeout=60, + ) + return resp.status_code + + +def main(): + print(f"Загружаем {COUNT} файлов (по ~1 МБ)...") + token = register_and_login("cost_tester") + chat_id = create_chat(token) + print(f"Chat ID: {chat_id}") + + content = make_png(200, 200) + print(f"Размер файла: {len(content) / 1024:.1f} КБ") + + success = 0 + errors = 0 + start = time.monotonic() + + for i in range(COUNT): + status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png") + if status == 201: + success += 1 + else: + errors += 1 + if (i + 1) % 10 == 0: + print(f" Загружено: {i + 1}/{COUNT}") + + elapsed = time.monotonic() - start + print(f"\n=== Результаты ===") + print(f"Успешно: {success}") + print(f"Ошибок: {errors}") + print(f"Время: {elapsed:.2f}s") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/plot.py b/scripts/plot.py new file mode 100644 index 00000000..45fad166 --- /dev/null +++ b/scripts/plot.py @@ -0,0 +1,23 @@ +import matplotlib.pyplot as plt +import os + +replicas = [1, 2, 4, 6] +rps = [250, 255, 260, 257] + +plt.figure(figsize=(8, 5)) +plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8) + +plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14) +plt.xlabel('Количество API-реплик', fontsize=12) +plt.ylabel('RPS (запросов в секунду)', fontsize=12) + +plt.grid(True, linestyle='--', alpha=0.7) +plt.xticks(replicas) + +plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252), + arrowprops=dict(facecolor='red', shrink=0.05), + fontsize=10, color='red') + +os.makedirs('evidence', exist_ok=True) +plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight') +print("График сохранен в evidence/scaling.png") \ No newline at end of file diff --git a/scripts/ram-experiment.py b/scripts/ram-experiment.py new file mode 100644 index 00000000..ea6525c9 --- /dev/null +++ b/scripts/ram-experiment.py @@ -0,0 +1,90 @@ +import io +import os +import time +import uuid +import requests +from concurrent.futures import ThreadPoolExecutor +from PIL import Image + +BASE_URL = "http://localhost:8080" +CONCURRENCY = 10 + + +def make_png(width, height): + img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3)) + buf = io.BytesIO() + img.save(buf, format='PNG', optimize=False) + return buf.getvalue() + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "RAM Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def upload(token, chat_id, content, filename): + headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + start = time.monotonic() + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers=headers, data=content, timeout=60, + ) + elapsed = time.monotonic() - start + return resp.status_code, elapsed, len(content) + + +def main(): + print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each") + token = register_and_login("ram_tester") + chat_id = create_chat(token) + print(f"Chat ID: {chat_id}") + + print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...") + contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)] + sizes = [len(c) / (1024 * 1024) for c in contents] + print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ") + print(f"Суммарно: {sum(sizes):.2f} МБ") + + print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...") + start = time.monotonic() + results, errors = [], [] + + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + futures = [ + pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png") + for i in range(CONCURRENCY) + ] + for f in futures: + try: + results.append(f.result()) + except Exception as e: + errors.append(str(e)) + + total = time.monotonic() - start + print(f"\n=== Результаты ===") + print(f"Всего: {len(results) + len(errors)}") + print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}") + print(f"Ошибок: {len(errors)}") + if results: + lat = [r[1] for r in results] + print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s") + print(f"Общее время: {total:.2f}s") + if errors: + print(f"Ошибки: {errors[:5]}") + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..8655c5ce --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,146 @@ +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +from minio import Minio +from sqlalchemy import select, cast, Text +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.models.message import Message +from app.database import async_session_maker +from app.models.attachment import Attachment +from app.config import ( + S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET +) + +GRACE_PERIOD = timedelta(seconds=1) + + +endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "") +minio_client = Minio( + endpoint, + access_key=S3_ACCESS_KEY_ID, + secret_key=S3_SECRET_ACCESS_KEY, + secure=False, +) +BUCKET = S3_BUCKET + + +async def find_orphan_s3_objects(dry_run: bool) -> int: + print("Поиск orphan объектов в S3, файлы без записи в БД") + + async with async_session_maker() as session: + result = await session.execute(select(Attachment.object_key)) + db_keys = {row[0] for row in result.all()} + + print(f"Файлов в БД: {len(db_keys)}") + + objects = list(minio_client.list_objects(BUCKET, recursive=True)) + print(f"Объектов в MinIO: {len(objects)}") + + now = datetime.now(timezone.utc) + orphans = [] + + for obj in objects: + if obj.object_name not in db_keys: + age = now - obj.last_modified if obj.last_modified else timedelta(0) + if age > GRACE_PERIOD: + orphans.append(obj.object_name) + print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)") + else: + print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)") + + if not orphans: + print("Orphan-объектов не найдено.") + return 0 + + if dry_run: + print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.") + else: + for key in orphans: + minio_client.remove_object(BUCKET, key) + print(f" [DELETED] {key}") + print(f"Удалено {len(orphans)} orphan-объектов.") + + return len(orphans) + + +async def find_missing_s3_objects(dry_run: bool) -> int: + print("\nПоиск missing объектов, записи в БД без файла в MinIO") + + async with async_session_maker() as session: + result = await session.execute(select(Attachment.id, Attachment.object_key)) + attachments = result.all() + + print(f"Записей в БД: {len(attachments)}") + + missing = [] + used_missing = [] + + for att_id, key in attachments: + try: + minio_client.stat_object(BUCKET, key) + except Exception: + + async with async_session_maker() as session: + result_msg = await session.execute( + select(Message).where( + cast(Message.attachment_ids, Text).contains(f'"{att_id}"') + ) + ) + is_used = result_msg.scalar_one_or_none() is not None + + if is_used: + used_missing.append((att_id, key)) + print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем") + else: + missing.append((att_id, key)) + print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить") + + total = len(missing) + len(used_missing) + if total == 0: + print("Missing-записей не найдено.") + return 0 + + if dry_run: + print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.") + else: + if missing: + async with async_session_maker() as session: + for att_id, _ in missing: + att = await session.get(Attachment, att_id) + if att: + await session.delete(att) + await session.commit() + print(f"Удалено {len(missing)} неиспользуемых missing из БД.") + if used_missing: + print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).") + + return len(missing) + + +async def main(): + parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД") + parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять") + parser.add_argument("--execute", action="store_true", help="Удалить найденное") + args = parser.parse_args() + + if not args.dry_run and not args.execute: + print("Укажите --dry-run или --execute") + return + + dry_run = args.dry_run and not args.execute + + orphans = await find_orphan_s3_objects(dry_run) + missing = await find_missing_s3_objects(dry_run) + + print("\nИтог") + print(f"Orphan S3 без БД: {orphans}") + print(f"Missing БД без S3 (удалено): {missing}") + if dry_run: + print("Режим: DRY-RUN (ничего не удалено)") + else: + print("Режим: EXECUTE (изменения применены)") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/scripts/rolling-load.sh b/scripts/rolling-load.sh new file mode 100755 index 00000000..5b4a862c --- /dev/null +++ b/scripts/rolling-load.sh @@ -0,0 +1,12 @@ + +echo "Нагрузка: 30 запросов с интервалом 0.5 сек" +for i in {1..30}; do + TS=$(date +"%H:%M:%S") + + RESP=$(curl -s -i http://localhost:8080/health/live) + CODE=$(echo "$RESP" | head -1 | awk '{print $2}') + INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}') + + echo "$TS code=$CODE instance=${INST:-<нет>}" + sleep 0.5 +done \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..89b1888e --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,20 @@ + +set -e + +echo "Rolling update: app1" +docker compose stop app1 +docker compose up -d --build app1 +echo "Жду, пока app1 станет healthy..." +sleep 5 + +echo "" +echo "Rolling update: app2" +docker compose stop app2 +docker compose up -d --build app2 +echo "Жду, пока app2 станет healthy..." +sleep 5 + +echo "" +echo "Проверка: обе реплики живы" +curl -i http://localhost:8080/health/live +curl -i http://localhost:8080/health/live \ No newline at end of file diff --git a/scripts/run-grace-period-test.sh b/scripts/run-grace-period-test.sh new file mode 100755 index 00000000..e6fcaed9 --- /dev/null +++ b/scripts/run-grace-period-test.sh @@ -0,0 +1,78 @@ +set -e +mkdir -p evidence + +{ +echo "=== GRACE PERIOD TEST — $(date) ===" +echo "" + +curl -s -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null + +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "grace_tester:secret1234567" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") +echo "Токен получен: ${TOKEN:0:20}..." + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") + +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Attachment ID: $ATT_ID" +echo "Object key: $OBJECT_KEY" + +docker compose exec -T db psql -U messenger -d messenger_db -c \ + "DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null +echo "Запись удалена из БД — ORPHAN" +echo "" + +echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов" +echo "" + +echo "=== Пауза 11 секунд ===" +sleep 11 +echo "" + +echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов" +echo "" + +echo "=== EXECUTE (удаляем созревший orphan) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов" +echo "" + +echo "=== Проверка: orphan удалён из MinIO ===" +STAT=$(docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1) +echo "Orphan в MinIO: $STAT" +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется" +echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»" +echo "3. Execute → [DELETED] — удалён из MinIO" +echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются" + +} 2>&1 | tee evidence/track2-grace-period.txt + +echo "" +echo "Лог сохранён в evidence/track2-grace-period.txt" diff --git a/scripts/run-reconciler-test.sh b/scripts/run-reconciler-test.sh new file mode 100755 index 00000000..bf501473 --- /dev/null +++ b/scripts/run-reconciler-test.sh @@ -0,0 +1,153 @@ +set -e +mkdir -p evidence + +{ +echo "================================================================" +echo " RECONCILER TEST — $(date)" +echo "================================================================" +echo "" + +TEST_USER="reconcile_tester_$(date +%s)" +TEST_PASS="secret1234567" + +echo "=== 1. Создание пользователя ($TEST_USER) ===" +REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}") +REG_BODY=$(echo "$REG_RESP" | sed '$d') +REG_CODE=$(echo "$REG_RESP" | tail -n 1) +echo "HTTP $REG_CODE" +echo "Body: $REG_BODY" + +if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then + echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)" + exit 1 +fi +echo "" + +echo "=== 2. Логин ===" +LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "$TEST_USER:$TEST_PASS") +LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d') +LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1) +echo "HTTP $LOGIN_CODE" + +if [ "$LOGIN_CODE" != "201" ]; then + echo "FAIL: логин не удался (HTTP $LOGIN_CODE)" + echo "Body: $LOGIN_BODY" + exit 1 +fi + +TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "") +if [ -z "$TOKEN" ]; then + echo "FAIL: токен не получен из ответа" + echo "Body: $LOGIN_BODY" + exit 1 +fi +echo "Токен получен: ${TOKEN:0:20}..." +echo "" + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "User ID: $USER_ID" +echo "" + +echo "=== 3. Создание чата ===" +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +echo "=== 4. Создаём ORPHAN ===" +RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n') +echo "Orphan attachment_id: $ORPHAN_ATT_ID" +echo "Orphan object_key: $ORPHAN_KEY" + +docker compose exec -T db psql -U messenger -d messenger_db -c \ + "DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null +echo "Запись удалена из БД — ORPHAN" +echo "" + +echo "=== 5. Создаём MISSING ===" +RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n') +echo "Missing attachment_id: $MISSING_ATT_ID" +echo "Missing object_key: $MISSING_KEY" + +docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY" +echo "Файл удалён из MinIO — MISSING" +echo "" + +echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ===" +sleep 11 +echo "" + +echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "=== 7. EXECUTE ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --execute +echo "" + +echo "=== 8. Проверка: orphan удалён из MinIO ===" +ORPHAN_STAT=$(docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1) +echo "Orphan в MinIO: $ORPHAN_STAT" +echo "" + +echo "=== 9. Проверка: missing удалён из БД ===" +MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n') +echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT" +echo "" + +echo "=== 10. Идемпотентность: повторный DRY-RUN ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +ORPHAN_GONE=false +if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then + ORPHAN_GONE=true +fi + +if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then + echo "PASS:" + echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен" + echo " - ORPHAN удалён из MinIO" + echo " - MISSING удалён из БД" + echo " - Повторный dry-run: 0/0 (идемпотентность)" +else + echo "FAIL:" + echo " - Orphan в MinIO: $ORPHAN_STAT" + echo " - Missing count в БД: $MISSING_COUNT" +fi + +} 2>&1 | tee evidence/reconciler-full.txt + +echo "" +echo "Лог сохранён в evidence/reconciler-full.txt" diff --git a/scripts/run-used-attachment-test.sh b/scripts/run-used-attachment-test.sh new file mode 100755 index 00000000..711b4e22 --- /dev/null +++ b/scripts/run-used-attachment-test.sh @@ -0,0 +1,76 @@ +set -e +mkdir -p evidence + +{ +echo "=== USED ATTACHMENT TEST — $(date) ===" +echo "" + +curl -s -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null + +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "used_tester:secret1234567" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") + +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Attachment ID: $ATT_ID" +echo "Object key: $KEY" + +# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ +IDEM_KEY="used-test-$(date +%s)" +curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Idempotency-Key: $IDEM_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null +echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ" +echo "" + +docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY" +echo "Файл удалён из MinIO — missing (используется)" +echo "" + +echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей" +echo "" + +echo "=== EXECUTE (используемый не удаляется) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено" +echo "" + +COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Записей с id=$ATT_ID в БД: $COUNT" +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +if [ "$COUNT" = "1" ]; then + echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])" +else + echo "FAIL: используемый missing удалён (count=$COUNT)" +fi + +} 2>&1 | tee evidence/track2-used-attachment.txt diff --git a/scripts/test-stream-limit.py b/scripts/test-stream-limit.py new file mode 100644 index 00000000..081a4aee --- /dev/null +++ b/scripts/test-stream-limit.py @@ -0,0 +1,61 @@ +import time +import requests + +BASE_URL = "http://localhost:8080" + + +def get_token(username="stream_tester"): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "T", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def get_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Stream Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def main(): + print("=" * 60) + print(" STREAM LIMIT TEST") + print("=" * 60) + token = get_token() + chat_id = get_chat(token) + print(f"Chat ID: {chat_id}") + print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===") + + def gen(): + for _ in range(100): + yield b"0" * (1024 * 1024) + + start = time.monotonic() + try: + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png", + headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"}, + data=gen(), timeout=60, + ) + status = resp.status_code + except Exception as e: + print(f"Ошибка: {e}") + status = None + elapsed = time.monotonic() - start + print(f"Status: {status} (ожидаем 413)") + print(f"Время: {elapsed:.2f}s") + print("\n" + "=" * 60) + if status == 413: + print("PASS: потоковый лимит работает, 413") + else: + print(f"FAIL: ожидался 413, получен {status}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/tests/check_attachment_status b/tests/check_attachment_status new file mode 100644 index 00000000..756e0e61 --- /dev/null +++ b/tests/check_attachment_status @@ -0,0 +1,29 @@ + +import asyncio +import sys +from uuid import UUID + +from app.worker import async_session_maker, engine +from app.models.attachment import Attachment + + +async def main(): + attachment_id = UUID(sys.argv[1]) + + try: + async with async_session_maker() as session: + att = await session.get(Attachment, attachment_id) + + if att is None: + print("Attachment not found") + return + + print("Attachment ID:", att.id) + print("Status:", att.status) + print("Error code:", att.error_code) + finally: + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/check_attachment_status.py b/tests/check_attachment_status.py new file mode 100644 index 00000000..756e0e61 --- /dev/null +++ b/tests/check_attachment_status.py @@ -0,0 +1,29 @@ + +import asyncio +import sys +from uuid import UUID + +from app.worker import async_session_maker, engine +from app.models.attachment import Attachment + + +async def main(): + attachment_id = UUID(sys.argv[1]) + + try: + async with async_session_maker() as session: + att = await session.get(Attachment, attachment_id) + + if att is None: + print("Attachment not found") + return + + print("Attachment ID:", att.id) + print("Status:", att.status) + print("Error code:", att.error_code) + finally: + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_worker_retry.py b/tests/test_worker_retry.py new file mode 100644 index 00000000..2fef3b05 --- /dev/null +++ b/tests/test_worker_retry.py @@ -0,0 +1,154 @@ + +import asyncio +import json +import subprocess +import sys +import uuid +from pathlib import Path + + +def test_worker_retry(): + result = subprocess.run( + [ + "docker", "compose", "exec", + "-T", "-w", "/app", + "worker", "python", "-" + ], + input=Path(__file__).read_text(encoding="utf-8"), + text=True, + capture_output=True, + timeout=60, + ) + + print(result.stdout, end="") + + if result.stderr: + print(result.stderr, file=sys.stderr) + + assert result.returncode == 0, "Проверка Worker не прошла" + + +async def main(): + import aio_pika + from sqlalchemy import select + + from app.worker import ( + async_session_maker, + engine, + RABBITMQ_URL, + QUEUE_NAME, + ) + from app.models.attachment import Attachment + + try: + async with async_session_maker() as session: + example = ( + await session.execute( + select(Attachment).limit(1) + ) + ).scalars().first() + + if example is None: + print( + "FAIL: нет вложений. Сначала запусти make test." + ) + return False + + test_id = uuid.uuid4() + + test_att = Attachment( + id=test_id, + object_key=f"retry-test-{test_id}.jpg", + filename="retry-test.jpg", + mime_type="image/jpeg", + size_bytes=1, + sha256="0" * 64, + chat_id=example.chat_id, + user_id=example.user_id, + status="queued", + ) + + session.add(test_att) + await session.commit() + + print(f"Тестовое вложение: {test_id}", flush=True) + + connection = await aio_pika.connect_robust( + RABBITMQ_URL + ) + + async with connection: + channel = await connection.channel( + publisher_confirms=True + ) + + await channel.declare_queue( + QUEUE_NAME, + durable=True + ) + + await channel.default_exchange.publish( + aio_pika.Message( + body=json.dumps({ + "attachment_id": str(test_id) + }).encode(), + delivery_mode=( + aio_pika.DeliveryMode.PERSISTENT + ), + ), + routing_key=QUEUE_NAME, + ) + + success = False + + for _ in range(60): + await asyncio.sleep(0.5) + + async with async_session_maker() as session: + att = await session.get( + Attachment, + test_id + ) + + if att and att.status == "failed": + print("Финальный статус:", att.status) + print( + "Безопасный error_code:", + att.error_code + ) + + success = ( + att.error_code == "s3_download_error" + ) + break + + if success: + async with async_session_maker() as session: + att = await session.get( + Attachment, + test_id + ) + + if att: + await session.delete(att) + await session.commit() + + print("Тестовая запись удалена.") + print("PASS: failed и безопасный error_code") + else: + print("FAIL: ожидаемый результат не получен.") + print( + "Тестовая запись оставлена для диагностики." + ) + + print(f"ID для проверки логов: {test_id}") + + return success + + finally: + await engine.dispose() + + +if __name__ == "__main__": + success = asyncio.run(main()) + sys.exit(0 if success else 1)