Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
35753ff11d |
+10
-16
@@ -1,25 +1,19 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=messenger_password
|
||||
POSTGRES_DB=messenger_db
|
||||
DATABASE_URL=postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
SESSION_STORE=redis
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=api-user
|
||||
S3_SECRET_ACCESS_KEY=api-user-secret-2026
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
@@ -1,18 +1,30 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
**Автор / группа:** Долда Кирилл, БПИ2301
|
||||
**Целевая оценка:** 5
|
||||
**Трек 5:** Трек 1 — «Очередь под перегрузкой».
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
**Что реализовано:** асинхронная обработка изображений через RabbitMQ, Worker, Transactional Outbox, retry, восстановление после сбоев, backpressure и ограничение очереди до 100 активных заданий. Из лабораторной 4 сохранены API, PostgreSQL, Redis, MinIO и система вложений.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
**Запуск из чистого clone:**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
docker compose up -d --build
|
||||
docker compose exec app1 alembic upgrade head
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
**Результаты проверок:** обработано 120/120 изображений без ошибок. Throughput вырос с 1,53 до 5,14 изображений/сек. Проверены восстановление после сбоя, очистка staging и ограничение приёма заданий. Подробности — [REPORT.md](REPORT.md).
|
||||
|
||||
**Ограничения:** увеличение параллелизма повышает потребление RAM; queue lag измеряется приблизительно.
|
||||
|
||||
**Источники и ИИ:** FastAPI, SQLAlchemy, RabbitMQ, MinIO, Pillow, Docker;
|
||||
|
||||
**Чек-лист:**
|
||||
- [x] Сохранена функциональность предыдущих лабораторных.
|
||||
- [x] Проверены сценарии отказов и перегрузки.
|
||||
- [x] Реализован трек 5.
|
||||
- [x] Финально выполнить `make check` и `make test`, сохранить вывод.
|
||||
- [x] Проверить отсутствие секретов в Git.
|
||||
- [x] Подготовлен `REPORT.md`.
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
FROM python:3.13-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements-dev.txt .
|
||||
RUN pip install --no-cache-dir -r requirements-dev.txt
|
||||
|
||||
RUN useradd --create-home --shell /bin/bash appuser
|
||||
|
||||
COPY app/ app/
|
||||
|
||||
COPY alembic.ini .
|
||||
COPY alembic/ alembic/
|
||||
RUN chown -R appuser:appuser /app/alembic
|
||||
COPY scripts/ /app/scripts/
|
||||
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/live')" || exit 1
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10", "--reload"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -25,6 +25,148 @@ test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
explain-10k:
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222' ORDER BY created_at, id LIMIT 50 OFFSET 0;"
|
||||
|
||||
explain-composite:
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_composite.txt
|
||||
|
||||
explain-simple:
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX IF EXISTS ix_messages_chat_created_id;"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_only ON messages (chat_id); ANALYZE messages;"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_simple.txt
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX ix_messages_chat_only;"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_created_id ON messages (chat_id, created_at, id);"
|
||||
|
||||
backup-restore:
|
||||
@echo "Делаем бекап pg_dump"
|
||||
time docker exec -it mtusi_messenger_db pg_dump -U messenger -d messenger_db > evidence/backup.sql
|
||||
@echo "Создаем новую базу messenger_restore"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS messenger_restore;"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "CREATE DATABASE messenger_restore;"
|
||||
@echo "Копируем бекап в контейнер и восстанавливаем"
|
||||
docker cp evidence/backup.sql mtusi_messenger_db:/tmp/
|
||||
time docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -f /tmp/backup.sql > /dev/null
|
||||
@echo "В основной базе messenger_db:"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -t -c "SELECT count(*) FROM messages;"
|
||||
@echo "В восстановленной базе messenger_restore:"
|
||||
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -t -c "SELECT count(*) FROM messages;"
|
||||
|
||||
test-race:
|
||||
@echo "Запускаем два одновременных запроса на регистрацию"
|
||||
@RACE_USER="race_$$(date +%s)"; \
|
||||
curl -s -o /dev/null -w "Запрос 1: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \
|
||||
curl -s -o /dev/null -w "Запрос 2: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \
|
||||
wait
|
||||
|
||||
test-redis:
|
||||
@echo "Ставим Redis на паузу"
|
||||
docker pause mtusi_messenger_redis
|
||||
@echo "Запрос"
|
||||
time curl -s -w "\nHTTP Статус: %{http_code}\n" http://localhost:8080/api/v1/users/me -H "Authorization: Bearer test_token"
|
||||
@echo "Снимаем паузу"
|
||||
docker unpause mtusi_messenger_redis
|
||||
|
||||
test-health-db:
|
||||
@echo "Останавливаем БД"
|
||||
docker compose stop db
|
||||
@echo "Проверяем health ready"
|
||||
curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready
|
||||
docker compose start db
|
||||
|
||||
test-health-redis:
|
||||
@echo "Останавливаем"
|
||||
docker compose stop redis
|
||||
@echo "Проверяем health ожидая 503 "
|
||||
curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready
|
||||
docker compose start redis
|
||||
|
||||
test-dependencies:
|
||||
$(PYTHON) -m pytest tests/test_resilience.py -v
|
||||
|
||||
failover:
|
||||
./scripts/failover-test.sh app1
|
||||
|
||||
failover-app2:
|
||||
./scripts/failover-test.sh app2
|
||||
|
||||
rolling-load:
|
||||
./scripts/rolling-load.sh
|
||||
|
||||
rolling-update:
|
||||
./scripts/rolling-update.sh
|
||||
|
||||
test-lab3:
|
||||
$(PYTHON) -m pytest tests/test_lab3.py -v
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) -m pytest tests/test_attachments.py -v
|
||||
|
||||
reconcile-dry:
|
||||
docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run
|
||||
|
||||
reconcile-execute:
|
||||
docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute
|
||||
|
||||
reconcile-test:
|
||||
./scripts/run-reconciler-test.sh
|
||||
|
||||
ram-experiment:
|
||||
@echo "В другом терминале для определения нагрузки: docker stats --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}'"
|
||||
mkdir -p evidence
|
||||
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" > evidence/ram-experiment.txt
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/ram-experiment.py 2>&1 | tee -a evidence/ram-experiment.txt
|
||||
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/ram-experiment.txt
|
||||
|
||||
stream-limit:
|
||||
mkdir -p evidence
|
||||
@echo "=== ДО ===" > evidence/stream-limit.txt
|
||||
@docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt
|
||||
@echo "" >> evidence/stream-limit.txt
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/test-stream-limit.py 2>&1 | tee -a evidence/stream-limit.txt
|
||||
@echo "" >> evidence/stream-limit.txt
|
||||
@echo "=== ПОСЛЕ ===" >> evidence/stream-limit.txt
|
||||
@docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt
|
||||
|
||||
private-bucket:
|
||||
mkdir -p evidence
|
||||
@OBJECT_KEY=$$(docker compose exec -T db psql -U messenger -d messenger_db -t -c "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1;" | tr -d ' \n'); \
|
||||
echo "Object key: $$OBJECT_KEY"; \
|
||||
{ \
|
||||
echo "=== Проверка private bucket ==="; \
|
||||
echo "Object key: $$OBJECT_KEY"; \
|
||||
echo ""; \
|
||||
echo "=== Прямой GET без авторизации (ожидаем 403) ==="; \
|
||||
curl -i "http://localhost:9000/messenger-attachments/$$OBJECT_KEY"; \
|
||||
} > evidence/private-bucket.txt 2>&1; \
|
||||
cat evidence/private-bucket.txt
|
||||
|
||||
s3-user-check:
|
||||
@echo "=== 1. api-user пытается создать пользователя (admin) ==="
|
||||
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user add myminio hacker hacker123 2>&1' | head -3
|
||||
@echo ""
|
||||
@echo "=== 2. api-user пытается изменить политику (admin) ==="
|
||||
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin policy attach myminio readwrite --user hacker 2>&1' | head -3
|
||||
@echo ""
|
||||
@echo "=== 3. api-user пытается удалить пользователя (admin) ==="
|
||||
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user remove myminio minioadmin 2>&1' | head -3
|
||||
|
||||
reconcile-cost:
|
||||
@echo "=== Стоимость обхода reconciler ==="
|
||||
time docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run 2>&1 | tee evidence/track2-cost-50.txt
|
||||
|
||||
grace-period-test:
|
||||
./scripts/run-grace-period-test.sh
|
||||
|
||||
used-attachment-test:
|
||||
./scripts/run-used-attachment-test.sh
|
||||
|
||||
test-retry:
|
||||
$(PYTHON) -m pytest tests/test_worker_retry.py -v -s
|
||||
|
||||
test-queue-upload:
|
||||
$(PYTHON) -c 'import sys; sys.path.insert(0, "tests"); from client import Client; c = Client(); alice, bob, eve, chat = c.bootstrap(); att, content = c.attachment(alice, chat); print("Attachment ID:", att["id"]); print("Status:", att["status"])'
|
||||
|
||||
test-queue-status:
|
||||
@test -n "$(ATTACHMENT_ID)" || (echo "Укажи ATTACHMENT_ID"; exit 1)
|
||||
docker compose exec -T worker python - "$(ATTACHMENT_ID)" < tests/check_attachment_status.py
|
||||
@@ -1,37 +1,84 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Долда Кирилл, БПИ2301
|
||||
- **Целевая оценка:** 5, трек 1 «Очередь под перегрузкой»
|
||||
- **Commit/PR:** TODO после публикации
|
||||
- **Источники, библиотеки и помощь ИИ:** README лабораторной, FastAPI, SQLAlchemy, asyncpg, Alembic, RabbitMQ (aio-pika), PostgreSQL, Redis, MinIO, Pillow, Docker Compose.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Требования:** Docker Desktop, Docker Compose, Git, make, curl. Для локальной разработки — Python 3.13.
|
||||
|
||||
**Основные образы:** `python:3.13-slim`, `postgres:16-alpine`, `redis:7-alpine`, `rabbitmq:3.12-management-alpine`, `nginx:alpine`, `pgsty/silo:latest`, `pgsty/mc:latest`.
|
||||
|
||||
**Подготовка, сборка, миграции и проверка:**
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
docker compose up -d --build
|
||||
docker compose exec -T app1 alembic upgrade head
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
**Схема:** клиент → Nginx → FastAPI (app1/app2) → PostgreSQL -> MinIO (исходники и варианты) → Outbox dispatcher → RabbitMQ → Worker → subprocess Pillow → MinIO + PostgreSQL. Redis используется для сессий.
|
||||
|
||||
**Источник истины:** PostgreSQL хранит состояние задания (`queued`, `processing`, `ready`, `failed`), ошибки и lease. MinIO хранит файлы; RabbitMQ доставляет задания, но не определяет их итоговый статус.
|
||||
|
||||
**Транзакции и надёжность:**
|
||||
|
||||
- API сохраняет исходник в MinIO, затем создаёт `Attachment` и `OutboxEvent` в одной транзакции БД; возвращает HTTP `202`.
|
||||
- Диспетчер публикует события в RabbitMQ с publisher confirm. Если публикация прервалась, неопубликованное событие остаётся в outbox.
|
||||
- Worker подтверждает сообщение (`ack`) после фиксации результата. Повторная доставка уже готового задания безопасна (идемпотентность).
|
||||
- Для зависших задач используется lease; для временных ошибок — ограниченные повторы (до 5) с backoff и jitter, затем `failed` с безопасным `error_code`.
|
||||
|
||||
**Обработка:** отдельный subprocess с таймаутом и ограничением памяти создаёт thumbnail JPEG 256×256, crop JPEG 128×128 и WebP.
|
||||
|
||||
**Лимиты:** загрузка до 10 МБ и 40 млн пикселей; `WORKER_CONCURRENCY` и `WORKER_PREFETCH` управляют параллелизмом. API принимает не более **100 активных** (`queued` + `processing`) задач; при переполнении — HTTP `429`.
|
||||
|
||||
**Из lab4 перенесено:** FastAPI, Nginx, PostgreSQL, Redis, MinIO, авторизация, ограничения загрузки, миграции, reconciler. **Добавлено в lab5:** RabbitMQ, worker, outbox, варианты изображений, retry, lease, prefetch и нагрузочные эксперименты.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Команда / сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- |--------------------------------------------------| --- |
|
||||
| Публичный smoke | `make test` | PASS | `evidence/smoke-lab5.txt` |
|
||||
| Асинхронная обработка | Upload → Outbox → RabbitMQ → Worker | Задания переходят в `ready` | `app/worker.py`, `app/outbox_dispatcher.py` |
|
||||
| Retry и `failed` | Ошибка обработки, до 5 попыток | Реализовано; отдельный финальный лог не сохранён | `app/worker.py` |
|
||||
| Варианты и изоляция | Обработка PNG/JPEG | Thumbnail, crop, WebP; subprocess с лимитами | `app/image_processor.py` |
|
||||
| Outbox и идемпотентность | Crash после COMMIT до ack | `REDELIVERED: True`, `CRASH TEST: PASSED` | Вывод ручного теста |
|
||||
| Трек 1: три режима | `benchmark_rss.py`, 60 задач на режим | 60/60 `ready` в каждом, 0 `failed` | `benchmark_results/summary.csv`, `rss_*.csv` |
|
||||
| Трек 1: перегрузка | `backpressure_test.py`, 120 задач | `Recovery: PASSED`, потерь нет | `benchmark_results/backpressure_queue.csv` |
|
||||
| Лимит приёма | Имитация 100/99 активных задач | HTTP `429` / разрешённая загрузка; `PASSED` | Вывод ручного теста |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Трек 1: очередь под перегрузкой
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
**Гипотеза:** большее число параллельных обработчиков повышает throughput, но увеличивает расход RAM. При перегрузке RabbitMQ должен накопить задания, а затем разобрать очередь без потерь.
|
||||
|
||||
**Методика:** `app/load_test.py` создаёт JPEG трёх размеров: 512×512 (~172 КБ), 1024×1024 (~688 КБ), 2048×2048 (~2751 КБ). На каждый режим — 60 задач (20 каждого размера). Время до `ready` и queue lag оценивались опросом БД; RSS — по процессам worker-контейнера.
|
||||
|
||||
| Concurrency / Prefetch | Ready / Failed | Время | Throughput | Пик RSS | p95 ready: 512 / 1024 / 2048, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 / 1 | 60 / 0 | 39.34 с | 1.53 img/s | 214.80 МБ | 36.85 / 38.84 / 36.62 |
|
||||
| 2 / 2 | 60 / 0 | 21.72 с | 2.76 img/s | 337.21 МБ | 16.67 / 17.15 / 20.43 |
|
||||
| 4 / 4 | 60 / 0 | 11.67 с | 5.14 img/s | 536.34 МБ | 9.84 / 10.13 / 11.67 |
|
||||
|
||||
**Средний наблюдаемый queue lag (512 / 1024 / 2048), с:** 1/1 — 18.98 / 22.74 / 18.76; 2/2 — 7.42 / 9.58 / 11.14; 4/4 — 5.72 / 5.14 / 5.70.
|
||||
|
||||
**Вывод:** переход с 1/1 на 4/4 повысил throughput в 3.36 раза и сократил время обработки пакета примерно на 70%, но увеличил пиковый RSS в 2.5 раза. Режим 2/2 — компромисс.
|
||||
|
||||
**Перегрузка (120 задач, 1/1):** 120/120 `ready`, 0 ошибок; 79.06 с, 1.52 img/s. Максимум в очереди — **119**, максимум `unacked` — **1**. После обработки `queued=0`, `unacked=0`, `Recovery: PASSED`. Это подтверждает backpressure по prefetch и восстановление после всплеска нагрузки.
|
||||
|
||||
| Воздействие | Что наблюдали | Результат |
|
||||
| --- | --- | --- |
|
||||
| Перегрузка 120 изображениями | Очередь выросла до 119, затем опустела | 120/120 готовы, потерь нет |
|
||||
| Истечение lease | Зависшая задача возвращена в обработку | Ручная проверка пройдена |
|
||||
| 100 активных задач | Новый upload отклонён | HTTP 429, тест `PASSED` |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**Подтверждено:** три режима worker обработали все задания; при перегрузке очередь восстановилась без потерь; повторная доставка после COMMIT безопасна; ветки лимита 100/99 работают.
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
# Use forward slashes (/) also on windows to provide an os agnostic path
|
||||
script_location = alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to alembic/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
|
||||
|
||||
# version path separator; As mentioned above, this is the character used to split
|
||||
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||
# Valid values for version_path_separator are:
|
||||
#
|
||||
# version_path_separator = :
|
||||
# version_path_separator = ;
|
||||
# version_path_separator = space
|
||||
# version_path_separator = newline
|
||||
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = %(here)s/.venv/bin/ruff
|
||||
# ruff.options = --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
@@ -0,0 +1,92 @@
|
||||
from app.models.base import Base
|
||||
from app.models.user import User
|
||||
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.message import Message
|
||||
from app.config import DATABASE_URL
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.outbox import OutboxEvent
|
||||
|
||||
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", DATABASE_URL.replace("+asyncpg", "+psycopg2"))
|
||||
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,30 @@
|
||||
"""create tables
|
||||
|
||||
Revision ID: 08fe661df060
|
||||
Revises: bc35c1b6b748
|
||||
Create Date: 2026-10-31 11:52:26.342048
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '08fe661df060'
|
||||
down_revision: Union[str, None] = 'bc35c1b6b748'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
pass
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
pass
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,35 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "12043b148948"
|
||||
down_revision = "ffdfa8869ddc"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"outbox_events",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("attachment_id", sa.Uuid(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"published_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=True
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["attachment_id"],
|
||||
["attachments.id"]
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("outbox_events")
|
||||
@@ -0,0 +1,33 @@
|
||||
|
||||
"""add attachment lease
|
||||
|
||||
Revision ID: 157ed8ce0e22
|
||||
Revises: 12043b148948
|
||||
Create Date: 2026-11-20 13:07:19.055182
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "157ed8ce0e22"
|
||||
down_revision: Union[str, None] = "12043b148948"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"attachments",
|
||||
sa.Column(
|
||||
"lease_expires_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("attachments", "lease_expires_at")
|
||||
@@ -0,0 +1,28 @@
|
||||
"""add status to attachments
|
||||
|
||||
Revision ID: 1c0bb32fbe88
|
||||
Revises: 8203fbed8a46
|
||||
Create Date: 2026-11-17 15:38:59.622410
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '1c0bb32fbe88'
|
||||
down_revision: Union[str, None] = '8203fbed8a46'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('attachments', sa.Column('status', sa.String(), server_default='queued', nullable=False))
|
||||
op.add_column('attachments', sa.Column('error_code', sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('attachments', 'error_code')
|
||||
op.drop_column('attachments', 'status')
|
||||
@@ -0,0 +1,23 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "2a1b3c4d5e6f"
|
||||
down_revision = "1c0bb32fbe88"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_sha256", sa.String(), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_size_bytes", sa.Integer(), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_width", sa.Integer(), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_height", sa.Integer(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("attachments", "thumbnail_height")
|
||||
op.drop_column("attachments", "thumbnail_width")
|
||||
op.drop_column("attachments", "thumbnail_size_bytes")
|
||||
op.drop_column("attachments", "thumbnail_sha256")
|
||||
op.drop_column("attachments", "thumbnail_key")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add timestamps to attachments
|
||||
|
||||
Revision ID: 8203fbed8a46
|
||||
Revises: b0f31212df2c
|
||||
Create Date: 2026-11-11 15:59:23.045039
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '8203fbed8a46'
|
||||
down_revision: Union[str, None] = 'b0f31212df2c'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Добавляем колонки created_at и updated_at
|
||||
op.add_column('attachments', sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False))
|
||||
op.add_column('attachments', sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False))
|
||||
# УДАЛЕНО: op.alter_column('attachments', 'user_id', ...) - чтобы не упасть на старых записях
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('attachments', 'updated_at')
|
||||
op.drop_column('attachments', 'created_at')
|
||||
# УДАЛЕНО: op.alter_column('attachments', 'user_id', nullable=True)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""add owner and sha256 to attachments
|
||||
|
||||
Revision ID: b0f31212df2c
|
||||
Revises: d78a7aaedac6
|
||||
Create Date: 2026-11-11 15:39:07.391800
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'b0f31212df2c'
|
||||
down_revision: Union[str, None] = 'd78a7aaedac6'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. Добавляем user_id с nullable=True, чтобы не упасть на старых данных
|
||||
op.add_column('attachments', sa.Column('user_id', sa.UUID(), nullable=True))
|
||||
|
||||
# 2. Добавляем sha256
|
||||
op.add_column('attachments', sa.Column('sha256', sa.String(), nullable=True))
|
||||
|
||||
# 3. Создаем внешний ключ
|
||||
op.create_foreign_key(None, 'attachments', 'users', ['user_id'], ['id'], ondelete='CASCADE')
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(None, 'attachments', type_='foreignkey')
|
||||
op.drop_column('attachments', 'sha256')
|
||||
op.drop_column('attachments', 'user_id')
|
||||
@@ -0,0 +1,67 @@
|
||||
"""create tables
|
||||
|
||||
Revision ID: b24373e3b620
|
||||
Revises:
|
||||
Create Date: 2026-10-24 06:54:22.471827
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'b24373e3b620'
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('chats',
|
||||
sa.Column('id', sa.UUID(), nullable=False),
|
||||
sa.Column('title', sa.String(length=100), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('users',
|
||||
sa.Column('id', sa.UUID(), nullable=False),
|
||||
sa.Column('username', sa.String(length=32), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=100), nullable=False),
|
||||
sa.Column('password_hash', sa.String(length=255), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True)
|
||||
op.create_table('chat_members',
|
||||
sa.Column('chat_id', sa.UUID(), nullable=False),
|
||||
sa.Column('user_id', sa.UUID(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('chat_id', 'user_id')
|
||||
)
|
||||
op.create_table('messages',
|
||||
sa.Column('id', sa.UUID(), nullable=False),
|
||||
sa.Column('chat_id', sa.UUID(), nullable=False),
|
||||
sa.Column('sender_id', sa.UUID(), nullable=False),
|
||||
sa.Column('text', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index('ix_messages_chat_created_id', table_name='messages')
|
||||
op.drop_table('messages')
|
||||
op.drop_table('chat_members')
|
||||
op.drop_index(op.f('ix_users_username'), table_name='users')
|
||||
op.drop_table('users')
|
||||
op.drop_table('chats')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,43 @@
|
||||
"""seed explain test 10k
|
||||
|
||||
Revision ID: bc35c1b6b748
|
||||
Revises: f9b85be2b710
|
||||
Create Date: 2026-10-24 07:04:09.294404
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'bc35c1b6b748'
|
||||
down_revision: Union[str, None] = 'f9b85be2b710'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ('11111111-1111-1111-1111-111111111111', 'explain_test', 'Test', 'hash', NOW())
|
||||
ON CONFLICT DO NOTHING;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
VALUES ('22222222-2222-2222-2222-222222222222', 'Explain Chat', NOW())
|
||||
ON CONFLICT DO NOTHING;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '22222222-2222-2222-2222-222222222222', '11111111-1111-1111-1111-111111111111', 'Message ' || i, NOW() - (i || ' minutes')::interval
|
||||
FROM generate_series(1, 10000) AS i;
|
||||
""")
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DELETE FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222';")
|
||||
op.execute("DELETE FROM chats WHERE id = '22222222-2222-2222-2222-222222222222';")
|
||||
op.execute("DELETE FROM users WHERE id = '11111111-1111-1111-1111-111111111111';")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""add attachment_ids to messages
|
||||
|
||||
Revision ID: d78a7aaedac6
|
||||
Revises: ea4f73899364
|
||||
Create Date: 2026-11-11 15:16:50.773392
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'd78a7aaedac6'
|
||||
down_revision: Union[str, None] = 'ea4f73899364'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('messages', sa.Column('attachment_ids', sa.JSON(), nullable=True))
|
||||
|
||||
op.execute("UPDATE messages SET attachment_ids = '[]'::json WHERE attachment_ids IS NULL")
|
||||
|
||||
op.alter_column('messages', 'attachment_ids', nullable=False, server_default='[]')
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('messages', 'attachment_ids')
|
||||
@@ -0,0 +1,20 @@
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "e090e7a3e071"
|
||||
down_revision = "2a1b3c4d5e6f"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"attachments",
|
||||
sa.Column("crop_key", sa.String(), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("attachments", "crop_key")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""add attachments table
|
||||
|
||||
Revision ID: ea4f73899364
|
||||
Revises: 08fe661df060
|
||||
Create Date: 2026-11-11 14:37:24.096041
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'ea4f73899364'
|
||||
down_revision: Union[str, None] = '08fe661df060'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('attachments',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('object_key', sa.String(), nullable=False),
|
||||
sa.Column('filename', sa.String(), nullable=False),
|
||||
sa.Column('mime_type', sa.String(), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('chat_id', sa.UUID(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('object_key')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('attachments')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,69 @@
|
||||
"""seed load test data
|
||||
|
||||
Revision ID: f9b85be2b710
|
||||
Revises: b24373e3b620
|
||||
Create Date: ...
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'f9b85be2b710'
|
||||
down_revision: Union[str, None] = 'b24373e3b620'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ('55555555-5555-4555-8555-555555555555', 'load_test', 'Load', 'hash', NOW())
|
||||
ON CONFLICT DO NOTHING;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO chats (id, title, created_at) VALUES
|
||||
('66666666-6666-4666-8666-666666666666', 'Big Chat', NOW()),
|
||||
('77777777-7777-4777-8777-777777777777', 'Small Chat 1', NOW()),
|
||||
('88888888-8888-4888-8888-888888888888', 'Small Chat 2', NOW())
|
||||
ON CONFLICT DO NOTHING;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '66666666-6666-4666-8666-666666666666', '55555555-5555-4555-8555-555555555555', 'Big ' || i, NOW() - (i || ' minutes')::interval
|
||||
FROM generate_series(1, 80000) AS i;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '77777777-7777-4777-8777-777777777777', '55555555-5555-4555-8555-555555555555', 'Small1 ' || i, NOW() - (i || ' minutes')::interval
|
||||
FROM generate_series(1, 10000) AS i;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '88888888-8888-4888-8888-888888888888', '55555555-5555-4555-8555-555555555555', 'Small2 ' || i, NOW() - (i || ' minutes')::interval
|
||||
FROM generate_series(1, 10000) AS i;
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("""
|
||||
DELETE FROM messages WHERE chat_id IN (
|
||||
'66666666-6666-4666-8666-666666666666',
|
||||
'77777777-7777-4777-8777-777777777777',
|
||||
'88888888-8888-4888-8888-888888888888'
|
||||
);
|
||||
""")
|
||||
op.execute("""
|
||||
DELETE FROM chats WHERE id IN (
|
||||
'66666666-6666-4666-8666-666666666666',
|
||||
'77777777-7777-4777-8777-777777777777',
|
||||
'88888888-8888-4888-8888-888888888888'
|
||||
);
|
||||
""")
|
||||
op.execute("DELETE FROM users WHERE id = '55555555-5555-4555-8555-555555555555';")
|
||||
@@ -0,0 +1,20 @@
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "ffdfa8869ddc"
|
||||
down_revision = "e090e7a3e071"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"attachments",
|
||||
sa.Column("webp_key", sa.String(), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("attachments", "webp_key")
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db",
|
||||
)
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
CODE_MAP = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "minioadmin")
|
||||
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadmin")
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
@@ -0,0 +1,13 @@
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
||||
from app.config import DATABASE_URL
|
||||
|
||||
engine = create_async_engine(
|
||||
DATABASE_URL,
|
||||
pool_size=5,
|
||||
max_overflow=10,
|
||||
pool_timeout=30,
|
||||
pool_recycle=1800,
|
||||
echo=False,
|
||||
)
|
||||
|
||||
async_session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
||||
@@ -0,0 +1,6 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.database import async_session_maker
|
||||
|
||||
async def get_session():
|
||||
async with async_session_maker() as session:
|
||||
yield session
|
||||
@@ -0,0 +1,37 @@
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from app.config import CODE_MAP
|
||||
|
||||
|
||||
def error_response(request: Request, status_code: int, code: str, message: str, extra_headers: dict = None):
|
||||
request_id = getattr(request.state, "request_id", "unknown")
|
||||
|
||||
|
||||
headers = {"X-Request-Id": request_id}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
code = CODE_MAP.get(exc.status_code, "internal_error")
|
||||
return error_response(request, exc.status_code, code, str(exc.detail), extra_headers=exc.headers)
|
||||
|
||||
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Ошибка валидации данных")
|
||||
|
||||
async def internal_error_handler(request: Request, exc: Exception):
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
@@ -0,0 +1,62 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
|
||||
MAX_INPUT_PIXELS = 20_000_000
|
||||
|
||||
|
||||
def main():
|
||||
input_path = Path(sys.argv[1])
|
||||
output_dir = Path(sys.argv[2])
|
||||
|
||||
try:
|
||||
with Image.open(input_path) as source:
|
||||
if source.width * source.height > MAX_INPUT_PIXELS:
|
||||
return 20
|
||||
|
||||
source.load()
|
||||
img = source.convert("RGB")
|
||||
|
||||
crop = ImageOps.fit(
|
||||
img,
|
||||
(128, 128),
|
||||
method=Image.Resampling.LANCZOS,
|
||||
centering=(0.5, 0.5)
|
||||
)
|
||||
|
||||
crop.save(
|
||||
output_dir / "crop.jpg",
|
||||
format="JPEG",
|
||||
quality=85
|
||||
)
|
||||
|
||||
img.save(
|
||||
output_dir / "optimized.webp",
|
||||
format="WEBP",
|
||||
quality=80,
|
||||
method=6
|
||||
)
|
||||
|
||||
img.thumbnail((256, 256))
|
||||
|
||||
img.save(
|
||||
output_dir / "thumbnail.jpg",
|
||||
format="JPEG",
|
||||
quality=85
|
||||
)
|
||||
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
Image.DecompressionBombError,
|
||||
):
|
||||
return 21
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,336 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import statistics
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import S3_BUCKET
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.outbox_dispatcher import session_maker
|
||||
from app.worker import minio_client
|
||||
|
||||
|
||||
IMAGE_SIZES = [512, 1024, 2048]
|
||||
|
||||
|
||||
def make_image(size):
|
||||
image = Image.effect_noise((size, size), 80).convert("RGB")
|
||||
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85)
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def percentile(values, percent):
|
||||
values = sorted(values)
|
||||
|
||||
if not values:
|
||||
return None
|
||||
|
||||
position = (len(values) - 1) * percent / 100
|
||||
lower = int(position)
|
||||
upper = min(lower + 1, len(values) - 1)
|
||||
|
||||
return values[lower] + (
|
||||
values[upper] - values[lower]
|
||||
) * (position - lower)
|
||||
|
||||
|
||||
async def get_source():
|
||||
async with session_maker() as session:
|
||||
result = await session.execute(
|
||||
select(Attachment)
|
||||
.where(Attachment.status == "ready")
|
||||
.order_by(Attachment.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
source = result.scalar_one_or_none()
|
||||
|
||||
if source is None:
|
||||
raise RuntimeError(
|
||||
"Нет готовой фотографии для создания тестовых записей"
|
||||
)
|
||||
|
||||
return source.chat_id, source.user_id
|
||||
|
||||
|
||||
async def create_jobs(count_per_size, run_id):
|
||||
chat_id, user_id = await get_source()
|
||||
|
||||
jobs = []
|
||||
|
||||
for size in IMAGE_SIZES:
|
||||
data = make_image(size)
|
||||
|
||||
print(
|
||||
f"[LOAD] Изображение {size}x{size}: "
|
||||
f"{len(data) / 1024:.1f} KB",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
for index in range(count_per_size):
|
||||
key = f"benchmark/{run_id}/{size}_{index}.jpg"
|
||||
|
||||
minio_client.put_object(
|
||||
S3_BUCKET,
|
||||
key,
|
||||
io.BytesIO(data),
|
||||
len(data),
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
|
||||
jobs.append({
|
||||
"size": size,
|
||||
"key": key,
|
||||
"data": data,
|
||||
})
|
||||
|
||||
async with session_maker() as session:
|
||||
async with session.begin():
|
||||
for job in jobs:
|
||||
data = job["data"]
|
||||
|
||||
attachment = Attachment(
|
||||
object_key=job["key"],
|
||||
filename=f"benchmark_{job['size']}.jpg",
|
||||
mime_type="image/jpeg",
|
||||
size_bytes=len(data),
|
||||
chat_id=chat_id,
|
||||
user_id=user_id,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
status="queued",
|
||||
)
|
||||
|
||||
session.add(attachment)
|
||||
await session.flush()
|
||||
|
||||
session.add(
|
||||
OutboxEvent(attachment_id=attachment.id)
|
||||
)
|
||||
|
||||
job["id"] = attachment.id
|
||||
del job["data"]
|
||||
|
||||
return jobs
|
||||
|
||||
|
||||
async def monitor_jobs(jobs, timeout):
|
||||
ids = [job["id"] for job in jobs]
|
||||
|
||||
started = time.perf_counter()
|
||||
last_report = started
|
||||
|
||||
first_processing = {}
|
||||
ready_times = {}
|
||||
failed = set()
|
||||
|
||||
peak_processing = 0
|
||||
|
||||
while True:
|
||||
now = time.perf_counter()
|
||||
elapsed = now - started
|
||||
|
||||
async with session_maker() as session:
|
||||
result = await session.execute(
|
||||
select(Attachment.id, Attachment.status)
|
||||
.where(Attachment.id.in_(ids))
|
||||
)
|
||||
|
||||
statuses = result.all()
|
||||
|
||||
processing_count = 0
|
||||
|
||||
for attachment_id, status in statuses:
|
||||
if status == "processing":
|
||||
processing_count += 1
|
||||
|
||||
if attachment_id not in first_processing:
|
||||
first_processing[attachment_id] = elapsed
|
||||
|
||||
elif status == "ready":
|
||||
if attachment_id not in ready_times:
|
||||
ready_times[attachment_id] = elapsed
|
||||
|
||||
elif status == "failed":
|
||||
failed.add(attachment_id)
|
||||
|
||||
peak_processing = max(
|
||||
peak_processing,
|
||||
processing_count,
|
||||
)
|
||||
|
||||
completed = len(ready_times) + len(failed)
|
||||
|
||||
if now - last_report >= 5:
|
||||
print(
|
||||
f"[LOAD] Готово: {len(ready_times)}/{len(jobs)}, "
|
||||
f"failed={len(failed)}, "
|
||||
f"processing={processing_count}, "
|
||||
f"elapsed={elapsed:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
last_report = now
|
||||
|
||||
if completed == len(jobs):
|
||||
break
|
||||
|
||||
if elapsed >= timeout:
|
||||
print("[LOAD] Достигнут таймаут", flush=True)
|
||||
break
|
||||
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
return {
|
||||
"ready_times": ready_times,
|
||||
"first_processing": first_processing,
|
||||
"failed": failed,
|
||||
"peak_processing": peak_processing,
|
||||
"elapsed": time.perf_counter() - started,
|
||||
}
|
||||
|
||||
|
||||
def save_results(jobs, results, run_id):
|
||||
output = Path(f"benchmark_{run_id}.csv")
|
||||
|
||||
with output.open("w", newline="") as file:
|
||||
writer = csv.writer(file)
|
||||
|
||||
writer.writerow([
|
||||
"attachment_id",
|
||||
"image_size",
|
||||
"status",
|
||||
"queue_lag_estimate_s",
|
||||
"time_to_ready_s",
|
||||
])
|
||||
|
||||
for job in jobs:
|
||||
attachment_id = job["id"]
|
||||
|
||||
ready = results["ready_times"].get(attachment_id)
|
||||
lag = results["first_processing"].get(attachment_id)
|
||||
|
||||
status = (
|
||||
"ready" if ready is not None
|
||||
else "failed" if attachment_id in results["failed"]
|
||||
else "unfinished"
|
||||
)
|
||||
|
||||
writer.writerow([
|
||||
str(attachment_id),
|
||||
job["size"],
|
||||
status,
|
||||
round(lag, 3) if lag is not None else "",
|
||||
round(ready, 3) if ready is not None else "",
|
||||
])
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def print_results(jobs, results):
|
||||
ready_times = results["ready_times"]
|
||||
first_processing = results["first_processing"]
|
||||
|
||||
total = len(jobs)
|
||||
completed = len(ready_times)
|
||||
|
||||
throughput = (
|
||||
completed / results["elapsed"]
|
||||
if results["elapsed"] > 0
|
||||
else 0
|
||||
)
|
||||
|
||||
print("\n========== BENCHMARK RESULTS ==========")
|
||||
print(f"Total jobs: {total}")
|
||||
print(f"Ready: {completed}")
|
||||
print(f"Failed: {len(results['failed'])}")
|
||||
print(f"Duration: {results['elapsed']:.2f} s")
|
||||
print(f"Throughput: {throughput:.2f} images/s")
|
||||
print(f"Peak observed processing: {results['peak_processing']}")
|
||||
|
||||
for size in IMAGE_SIZES:
|
||||
size_jobs = [
|
||||
job for job in jobs
|
||||
if job["size"] == size
|
||||
]
|
||||
|
||||
times = [
|
||||
ready_times[job["id"]]
|
||||
for job in size_jobs
|
||||
if job["id"] in ready_times
|
||||
]
|
||||
|
||||
lags = [
|
||||
first_processing[job["id"]]
|
||||
for job in size_jobs
|
||||
if job["id"] in first_processing
|
||||
]
|
||||
|
||||
print(f"\nImage size: {size}x{size}")
|
||||
print(f"Completed: {len(times)}/{len(size_jobs)}")
|
||||
|
||||
if times:
|
||||
print(f"Mean ready: {statistics.mean(times):.2f} s")
|
||||
print(f"P95 ready: {percentile(times, 95):.2f} s")
|
||||
|
||||
if lags:
|
||||
print(f"Mean observed queue lag: {statistics.mean(lags):.2f} s")
|
||||
print(f"Queue lag samples: {len(lags)}/{len(size_jobs)}")
|
||||
|
||||
print("\n=======================================")
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"--count",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Количество изображений каждого размера",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=300,
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.count < 1 or args.timeout < 1:
|
||||
parser.error("--count и --timeout должны быть положительными")
|
||||
|
||||
run_id = uuid.uuid4().hex[:8]
|
||||
|
||||
print(f"[LOAD] Run ID: {run_id}")
|
||||
print("[LOAD] Подготовка изображений...")
|
||||
|
||||
jobs = await create_jobs(args.count, run_id)
|
||||
|
||||
print(f"[LOAD] Создано заданий: {len(jobs)}")
|
||||
print("[LOAD] Ожидание обработки...")
|
||||
|
||||
results = await monitor_jobs(jobs, args.timeout)
|
||||
|
||||
print_results(jobs, results)
|
||||
|
||||
output = save_results(jobs, results, run_id)
|
||||
|
||||
print(f"\nCSV: {output}")
|
||||
|
||||
if len(results["ready_times"]) != len(jobs):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi import HTTPException
|
||||
import uuid
|
||||
from app.middleware import RequestIdMiddleware
|
||||
from app.exceptions import http_exception_handler, validation_exception_handler
|
||||
from app.routers import health, users, chats, messages, auth, attachments
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger",
|
||||
version="1.1.0",
|
||||
)
|
||||
|
||||
INSTANCE_ID = str(uuid.uuid4())
|
||||
@app.middleware("http")
|
||||
async def add_instance_id_header(request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
app.add_exception_handler(HTTPException, http_exception_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_exception_handler)
|
||||
app.include_router(health.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(chats.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(attachments.router_upload)
|
||||
app.include_router(attachments.router_download)
|
||||
@@ -0,0 +1,11 @@
|
||||
import uuid as uuid_lib
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request, call_next):
|
||||
request_id = str(uuid_lib.uuid4())
|
||||
request.state.request_id = request_id
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
return response
|
||||
@@ -0,0 +1,48 @@
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String, Integer, ForeignKey, DateTime, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
object_key: Mapped[str] = mapped_column(String, unique=True)
|
||||
filename: Mapped[str] = mapped_column(String)
|
||||
mime_type: Mapped[str] = mapped_column(String)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
sha256: Mapped[str] = mapped_column(String, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now()
|
||||
)
|
||||
|
||||
status: Mapped[str] = mapped_column(String, default="queued")
|
||||
error_code: Mapped[str] = mapped_column(String, nullable=True)
|
||||
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=True
|
||||
)
|
||||
|
||||
thumbnail_key: Mapped[str] = mapped_column(String, nullable=True)
|
||||
thumbnail_sha256: Mapped[str] = mapped_column(String, nullable=True)
|
||||
thumbnail_size_bytes: Mapped[int] = mapped_column(Integer, nullable=True)
|
||||
thumbnail_width: Mapped[int] = mapped_column(Integer, nullable=True)
|
||||
thumbnail_height: Mapped[int] = mapped_column(Integer, nullable=True)
|
||||
|
||||
crop_key: Mapped[str] = mapped_column(String, nullable=True)
|
||||
webp_key: Mapped[str] = mapped_column(String, nullable=True)
|
||||
@@ -0,0 +1,4 @@
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
@@ -0,0 +1,35 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String, DateTime, ForeignKey, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
title: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("chats.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Text, DateTime, ForeignKey, Index, func, JSON
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
|
||||
attachment_ids: Mapped[list] = mapped_column(JSON, default=list)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("chats.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
text: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_messages_chat_created_id", "chat_id", "created_at", "id"),
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
import uuid
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.attachment import Base
|
||||
|
||||
|
||||
class OutboxEvent(Base):
|
||||
__tablename__ = "outbox_events"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
primary_key=True,
|
||||
default=uuid.uuid4
|
||||
)
|
||||
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"),
|
||||
nullable=False
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
nullable=False
|
||||
)
|
||||
|
||||
published_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=True
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String, DateTime, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
username: Mapped[str] = mapped_column(
|
||||
String(32), unique=True, nullable=False, index=True
|
||||
)
|
||||
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import aio_pika
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.config import DATABASE_URL
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.user import User
|
||||
|
||||
RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/")
|
||||
QUEUE_NAME = "image_tasks"
|
||||
POLL_INTERVAL_SECONDS = 1
|
||||
|
||||
engine = create_async_engine(DATABASE_URL)
|
||||
session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def recover_expired_lease():
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
async with session_maker() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
select(Attachment)
|
||||
.where(
|
||||
Attachment.status == "processing",
|
||||
Attachment.lease_expires_at.is_not(None),
|
||||
Attachment.lease_expires_at <= now,
|
||||
)
|
||||
.order_by(Attachment.lease_expires_at)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
|
||||
att = result.scalar_one_or_none()
|
||||
|
||||
if att is None:
|
||||
return False
|
||||
|
||||
att.status = "queued"
|
||||
att.lease_expires_at = None
|
||||
|
||||
session.add(
|
||||
OutboxEvent(attachment_id=att.id)
|
||||
)
|
||||
|
||||
attachment_id = att.id
|
||||
|
||||
print(f"[Outbox] Recovered expired lease: {attachment_id}", flush=True)
|
||||
return True
|
||||
|
||||
|
||||
async def dispatch_one(channel):
|
||||
async with session_maker() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
select(OutboxEvent)
|
||||
.where(OutboxEvent.published_at.is_(None))
|
||||
.order_by(OutboxEvent.created_at, OutboxEvent.id)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
|
||||
event = result.scalar_one_or_none()
|
||||
|
||||
if event is None:
|
||||
return False
|
||||
|
||||
confirmation = await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=json.dumps(
|
||||
{"attachment_id": str(event.attachment_id)}
|
||||
).encode(),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=str(event.id),
|
||||
content_type="application/json",
|
||||
),
|
||||
routing_key=QUEUE_NAME,
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
if confirmation is False:
|
||||
raise RuntimeError("RabbitMQ did not confirm publication")
|
||||
|
||||
event.published_at = datetime.now(timezone.utc)
|
||||
|
||||
print(f"[Outbox] Published {event.attachment_id}", flush=True)
|
||||
return True
|
||||
|
||||
|
||||
async def main():
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
connection = await aio_pika.connect_robust(RABBITMQ_URL)
|
||||
|
||||
async with connection:
|
||||
channel = await connection.channel(
|
||||
publisher_confirms=True,
|
||||
on_return_raises=True,
|
||||
)
|
||||
|
||||
await channel.declare_queue(
|
||||
QUEUE_NAME,
|
||||
durable=True,
|
||||
)
|
||||
|
||||
while True:
|
||||
recovered = await recover_expired_lease()
|
||||
published = await dispatch_one(channel)
|
||||
|
||||
if not recovered and not published:
|
||||
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"[Outbox] Retry after error: {type(exc).__name__}",
|
||||
flush=True,
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
def paginate(items: list, limit: int, cursor: str | None):
|
||||
start = 0
|
||||
if cursor is not None:
|
||||
try:
|
||||
start = int(cursor)
|
||||
except ValueError:
|
||||
start = 0
|
||||
|
||||
if start < 0:
|
||||
start = 0
|
||||
|
||||
page = items[start : start + limit]
|
||||
next_index = start + limit
|
||||
|
||||
if next_index < len(items):
|
||||
next_cursor = str(next_index)
|
||||
else:
|
||||
next_cursor = None
|
||||
|
||||
return page, next_cursor
|
||||
@@ -0,0 +1,9 @@
|
||||
import redis.asyncio as redis
|
||||
from app.config import REDIS_URL
|
||||
|
||||
redis_client = redis.from_url(
|
||||
REDIS_URL,
|
||||
decode_responses=True,
|
||||
socket_timeout=3,
|
||||
socket_connect_timeout=3,
|
||||
)
|
||||
@@ -0,0 +1,91 @@
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.config import DATABASE_URL
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.user import User
|
||||
from app.models.chat import Chat, ChatMember
|
||||
|
||||
|
||||
RETRYABLE_ERROR_CODES = {
|
||||
"s3_download_error",
|
||||
"s3_upload_error",
|
||||
"image_processing_timeout",
|
||||
"image_processor_start_error",
|
||||
"image_processor_error",
|
||||
"image_processor_output_error",
|
||||
"worker_error",
|
||||
}
|
||||
|
||||
|
||||
async def replay_failed(attachment_id: uuid.UUID):
|
||||
engine = create_async_engine(DATABASE_URL)
|
||||
session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
try:
|
||||
async with session_maker() as session:
|
||||
async with session.begin():
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
attachment_id,
|
||||
with_for_update=True,
|
||||
)
|
||||
|
||||
if att is None:
|
||||
print("Файл не найден")
|
||||
return
|
||||
|
||||
if att.status != "failed":
|
||||
print(f"Replay запрещён: status={att.status}")
|
||||
return
|
||||
|
||||
if att.error_code not in RETRYABLE_ERROR_CODES:
|
||||
print(
|
||||
f"Replay запрещён: error_code={att.error_code}"
|
||||
)
|
||||
return
|
||||
|
||||
pending = await session.scalar(
|
||||
select(OutboxEvent.id)
|
||||
.where(
|
||||
OutboxEvent.attachment_id == attachment_id,
|
||||
OutboxEvent.published_at.is_(None),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
if pending is not None:
|
||||
print("Задание уже ожидает публикации")
|
||||
return
|
||||
|
||||
att.status = "queued"
|
||||
att.error_code = None
|
||||
|
||||
session.add(
|
||||
OutboxEvent(attachment_id=attachment_id)
|
||||
)
|
||||
|
||||
print(f"Задание {attachment_id} отправлено на повторную обработку")
|
||||
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Использование: python -m app.replay_failed UUID")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
attachment_id = uuid.UUID(sys.argv[1])
|
||||
except ValueError:
|
||||
print("Некорректный UUID")
|
||||
sys.exit(1)
|
||||
|
||||
asyncio.run(replay_failed(attachment_id))
|
||||
@@ -0,0 +1,286 @@
|
||||
import uuid
|
||||
import filetype
|
||||
import io
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from PIL import Image
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func, text
|
||||
|
||||
from app.dependencies import get_session
|
||||
from app.security import get_current_user
|
||||
from app.models.user import User
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.chat import ChatMember
|
||||
from app.storage import minio_client, BUCKET
|
||||
|
||||
|
||||
router_upload = APIRouter(
|
||||
prefix="/api/v1/chats/{chat_id}/attachments",
|
||||
tags=["attachments"]
|
||||
)
|
||||
|
||||
router_download = APIRouter(
|
||||
prefix="/api/v1/attachments",
|
||||
tags=["attachments"]
|
||||
)
|
||||
|
||||
MAX_SIZE = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
MAX_ACTIVE_ATTACHMENTS = 100
|
||||
ADMISSION_LOCK_KEY = 823164001
|
||||
|
||||
def format_utc_timestamp(value: datetime | None) -> str:
|
||||
if value is None:
|
||||
value = datetime.now(timezone.utc)
|
||||
|
||||
elif value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
|
||||
return (
|
||||
value.astimezone(timezone.utc)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def build_attachment_response(att: Attachment) -> dict:
|
||||
variants = []
|
||||
|
||||
if (
|
||||
att.status == "ready"
|
||||
and att.thumbnail_key
|
||||
and att.thumbnail_sha256
|
||||
and att.thumbnail_size_bytes is not None
|
||||
and att.thumbnail_width is not None
|
||||
and att.thumbnail_height is not None
|
||||
):
|
||||
variants.append({
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": att.thumbnail_size_bytes,
|
||||
"width": att.thumbnail_width,
|
||||
"height": att.thumbnail_height,
|
||||
"sha256": att.thumbnail_sha256,
|
||||
})
|
||||
|
||||
return {
|
||||
"id": str(att.id),
|
||||
"chat_id": str(att.chat_id),
|
||||
"owner_id": str(att.user_id),
|
||||
"filename": att.filename,
|
||||
"content_type": att.mime_type,
|
||||
"size_bytes": att.size_bytes,
|
||||
"sha256": att.sha256,
|
||||
"status": att.status,
|
||||
"variants": variants,
|
||||
"error_code": att.error_code,
|
||||
"created_at": format_utc_timestamp(att.created_at),
|
||||
"updated_at": format_utc_timestamp(att.updated_at),
|
||||
}
|
||||
|
||||
|
||||
@router_upload.post("", status_code=202)
|
||||
async def upload_attachment(
|
||||
chat_id: uuid.UUID,
|
||||
filename: str = Query(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=255,
|
||||
pattern=r"^[^/\\\x00]+$"
|
||||
),
|
||||
request: Request = None,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
):
|
||||
result = await session.execute(
|
||||
select(ChatMember).where(
|
||||
ChatMember.chat_id == chat_id,
|
||||
ChatMember.user_id == user.id,
|
||||
)
|
||||
)
|
||||
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(404, "Чат не найден")
|
||||
|
||||
sha_hasher = hashlib.sha256()
|
||||
buffer = io.BytesIO()
|
||||
total = 0
|
||||
first_bytes = b""
|
||||
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
|
||||
total += len(chunk)
|
||||
|
||||
if total > MAX_SIZE:
|
||||
raise HTTPException(413, "File too large")
|
||||
|
||||
sha_hasher.update(chunk)
|
||||
buffer.write(chunk)
|
||||
|
||||
if len(first_bytes) < 262:
|
||||
need = 262 - len(first_bytes)
|
||||
first_bytes += chunk[:need]
|
||||
|
||||
kind = filetype.guess(first_bytes)
|
||||
|
||||
if not kind or kind.mime not in ("image/png", "image/jpeg"):
|
||||
raise HTTPException(400, "Only PNG/JPEG allowed")
|
||||
|
||||
contents = buffer.getvalue()
|
||||
|
||||
try:
|
||||
img = Image.open(io.BytesIO(contents))
|
||||
|
||||
if img.width * img.height > MAX_PIXELS:
|
||||
raise HTTPException(400, "Image too large (pixels)")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
raise HTTPException(400, "Corrupted image")
|
||||
|
||||
# Serialize admission checks across all API instances.
|
||||
# This transaction-level PostgreSQL lock is released on commit/rollback.
|
||||
await session.execute(
|
||||
text("SELECT pg_advisory_xact_lock(:lock_key)"),
|
||||
{"lock_key": ADMISSION_LOCK_KEY},
|
||||
)
|
||||
|
||||
active_count = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Attachment)
|
||||
.where(Attachment.status.in_(("queued", "processing")))
|
||||
)
|
||||
|
||||
if active_count >= MAX_ACTIVE_ATTACHMENTS:
|
||||
await session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Image processing queue is full; try again later",
|
||||
headers={"Retry-After": "30"},
|
||||
)
|
||||
|
||||
object_key = f"{uuid.uuid4()}.{kind.extension}"
|
||||
|
||||
minio_client.put_object(
|
||||
BUCKET,
|
||||
object_key,
|
||||
io.BytesIO(contents),
|
||||
len(contents),
|
||||
content_type=kind.mime,
|
||||
)
|
||||
|
||||
sha_hash = sha_hasher.hexdigest()
|
||||
|
||||
att = Attachment(
|
||||
object_key=object_key,
|
||||
filename=filename,
|
||||
mime_type=kind.mime,
|
||||
size_bytes=len(contents),
|
||||
chat_id=chat_id,
|
||||
user_id=user.id,
|
||||
sha256=sha_hash,
|
||||
status="queued"
|
||||
)
|
||||
|
||||
session.add(att)
|
||||
await session.flush()
|
||||
|
||||
event = OutboxEvent(attachment_id=att.id)
|
||||
session.add(event)
|
||||
|
||||
await session.commit()
|
||||
await session.refresh(att)
|
||||
|
||||
return build_attachment_response(att)
|
||||
|
||||
|
||||
@router_download.get("/{attachment_id}/content")
|
||||
async def download_attachment_content(
|
||||
attachment_id: uuid.UUID,
|
||||
variant: str = "original",
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
):
|
||||
att = await session.get(Attachment, attachment_id)
|
||||
|
||||
if not att:
|
||||
raise HTTPException(404, "Not found")
|
||||
|
||||
result = await session.execute(
|
||||
select(ChatMember).where(
|
||||
ChatMember.chat_id == att.chat_id,
|
||||
ChatMember.user_id == user.id,
|
||||
)
|
||||
)
|
||||
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(404, "Not found")
|
||||
|
||||
if att.status != "ready":
|
||||
raise HTTPException(409, "File is not ready yet")
|
||||
|
||||
if variant == "thumbnail":
|
||||
if not att.thumbnail_key:
|
||||
raise HTTPException(404, "Thumbnail not found")
|
||||
|
||||
key_to_download = att.thumbnail_key
|
||||
mime = "image/jpeg"
|
||||
|
||||
elif variant == "original":
|
||||
key_to_download = att.object_key
|
||||
mime = att.mime_type
|
||||
|
||||
else:
|
||||
raise HTTPException(404, "Variant not found")
|
||||
|
||||
try:
|
||||
response = minio_client.get_object(
|
||||
BUCKET,
|
||||
key_to_download
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
response,
|
||||
media_type=mime,
|
||||
headers={
|
||||
"Content-Disposition":
|
||||
f'attachment; filename="{att.filename}"'
|
||||
},
|
||||
)
|
||||
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
404,
|
||||
"File not found in storage"
|
||||
)
|
||||
|
||||
|
||||
@router_download.get("/{attachment_id}")
|
||||
async def get_attachment_metadata(
|
||||
attachment_id: uuid.UUID,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
):
|
||||
att = await session.get(Attachment, attachment_id)
|
||||
|
||||
if not att:
|
||||
raise HTTPException(404, "Not found")
|
||||
|
||||
result = await session.execute(
|
||||
select(ChatMember).where(
|
||||
ChatMember.chat_id == att.chat_id,
|
||||
ChatMember.user_id == user.id,
|
||||
)
|
||||
)
|
||||
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(404, "Not found")
|
||||
|
||||
return build_attachment_response(att)
|
||||
@@ -0,0 +1,83 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPAuthorizationCredentials
|
||||
from pydantic import BaseModel, ConfigDict, field_serializer
|
||||
from app.redis_client import redis_client
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from app.database import async_session_maker
|
||||
from app.models.user import User
|
||||
from app.security import verify_password, create_session, delete_session, bearer_scheme
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
|
||||
security = HTTPBasic(realm="messenger")
|
||||
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
id: uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
@field_serializer('created_at')
|
||||
def serialize_dt(self, dt: datetime):
|
||||
return dt.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
class SessionResponse(BaseModel):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserResponse
|
||||
|
||||
@field_serializer('expires_at')
|
||||
def serialize_dt(self, dt_str: str):
|
||||
if dt_str and dt_str.endswith("+00:00"):
|
||||
return dt_str.replace("+00:00", "Z")
|
||||
return dt_str
|
||||
|
||||
|
||||
# Basic Auth в Bearer Token
|
||||
@router.post("/sessions", status_code=201, response_model=SessionResponse)
|
||||
async def login(credentials: HTTPBasicCredentials = Depends(security)):
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(User).where(User.username == credentials.username))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user or not verify_password(credentials.password, user.password_hash):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Incorrect username or password",
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
|
||||
session_data = await create_session(user.id)
|
||||
|
||||
session_data["user"] = user
|
||||
|
||||
return session_data
|
||||
|
||||
@router.delete("/sessions/current", status_code=204)
|
||||
async def logout(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)):
|
||||
if not credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
token = credentials.credentials
|
||||
|
||||
user_id = await redis_client.get(f"session:{token}")
|
||||
if not user_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
await delete_session(token)
|
||||
return None
|
||||
@@ -0,0 +1,139 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from pydantic import BaseModel, Field, field_validator, ConfigDict, field_serializer
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.user import User
|
||||
from app.security import get_current_user
|
||||
from app.dependencies import get_session
|
||||
from app.pagination import paginate
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_members(cls, v):
|
||||
if len(set(v)) != len(v):
|
||||
raise ValueError("member_ids должны быть уникальными")
|
||||
return v
|
||||
|
||||
class ChatResponse(BaseModel):
|
||||
id: uuid.UUID
|
||||
title: str
|
||||
member_ids: list[uuid.UUID]
|
||||
created_at: datetime
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
@field_serializer('created_at')
|
||||
def serialize_dt(self, dt: datetime):
|
||||
return dt.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_chat(
|
||||
payload: CreateChat,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session)
|
||||
):
|
||||
if str(user.id) not in payload.member_ids:
|
||||
raise HTTPException(status_code=400, detail="Пользователь должен быть участником чата")
|
||||
|
||||
for member_id in payload.member_ids:
|
||||
result = await session.execute(select(User).where(User.id == member_id))
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail=f"несуществующий пользователь: {member_id}")
|
||||
|
||||
chat = Chat(title=payload.title)
|
||||
session.add(chat)
|
||||
await session.flush()
|
||||
|
||||
for member_id in payload.member_ids:
|
||||
session.add(ChatMember(chat_id=chat.id, user_id=member_id))
|
||||
|
||||
await session.commit()
|
||||
await session.refresh(chat)
|
||||
|
||||
return {
|
||||
"id": chat.id,
|
||||
"title": chat.title,
|
||||
"member_ids": payload.member_ids,
|
||||
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{chat_id}")
|
||||
async def get_chat(
|
||||
chat_id: str,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session)
|
||||
):
|
||||
result = await session.execute(select(Chat).where(Chat.id == chat_id))
|
||||
chat = result.scalar_one_or_none()
|
||||
|
||||
if not chat:
|
||||
raise HTTPException(status_code=404, detail="чат не найден")
|
||||
|
||||
result = await session.execute(
|
||||
select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user.id)
|
||||
)
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=404, detail="чат не найден")
|
||||
|
||||
result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id))
|
||||
member_ids = []
|
||||
for row in result.all():
|
||||
user_id = row[0]
|
||||
member_ids.append(str(user_id))
|
||||
|
||||
return {
|
||||
"id": chat.id,
|
||||
"title": chat.title,
|
||||
"member_ids": member_ids,
|
||||
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_chats(
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session)
|
||||
):
|
||||
if limit < 1 or limit > 100:
|
||||
raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100")
|
||||
|
||||
result = await session.execute(
|
||||
select(Chat)
|
||||
.join(ChatMember, Chat.id == ChatMember.chat_id)
|
||||
.where(ChatMember.user_id == user.id)
|
||||
.order_by(Chat.created_at, Chat.id)
|
||||
)
|
||||
chats = result.scalars().all()
|
||||
|
||||
my_chats = []
|
||||
for chat in chats:
|
||||
result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id))
|
||||
member_ids = []
|
||||
for row in result.all():
|
||||
user_id = row[0]
|
||||
member_ids.append(str(user_id))
|
||||
|
||||
my_chats.append({
|
||||
"id": chat.id,
|
||||
"title": chat.title,
|
||||
"member_ids": member_ids,
|
||||
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
|
||||
})
|
||||
|
||||
page, next_cursor = paginate(my_chats, limit, cursor)
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
@@ -0,0 +1,34 @@
|
||||
from fastapi import APIRouter
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import text
|
||||
from app.database import async_session_maker
|
||||
from app.redis_client import redis_client
|
||||
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/ready")
|
||||
async def ready():
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
await session.execute(text("SELECT 1"))
|
||||
|
||||
await redis_client.ping()
|
||||
|
||||
return {"status": "ok"}
|
||||
except Exception:
|
||||
raise HTTPException(status_code=503, detail="unavailable")
|
||||
|
||||
@router.get("/")
|
||||
def home():
|
||||
return {
|
||||
"service": "MTUSI Messenger",
|
||||
"version": "1.1.0",
|
||||
"docs": "/docs",
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, HTTPException, Depends, Header
|
||||
from pydantic import BaseModel, Field, ConfigDict, field_serializer, model_validator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from app.redis_client import redis_client
|
||||
import json
|
||||
|
||||
from app.models.message import Message
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.user import User
|
||||
from app.models.attachment import Attachment
|
||||
from app.security import get_current_user
|
||||
from app.dependencies import get_session
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str = Field(min_length=0, max_length=4000)
|
||||
attachment_ids: list[uuid.UUID] = []
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_text_or_attachment(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Сообщение должно содержать текст или вложение")
|
||||
return self
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
sender_id: uuid.UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
attachment_ids: list[uuid.UUID] = []
|
||||
|
||||
@field_serializer('created_at')
|
||||
def serialize_dt(self, dt: datetime):
|
||||
return dt.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
async def check_chat_access(chat_id: str, user_id: uuid.UUID, session: AsyncSession):
|
||||
chat = await session.get(Chat, chat_id)
|
||||
if not chat:
|
||||
raise HTTPException(status_code=404, detail="чат не найден")
|
||||
|
||||
result = await session.execute(
|
||||
select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user_id)
|
||||
)
|
||||
if not result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=404, detail="чат не найден")
|
||||
return chat
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=MessageResponse)
|
||||
async def create_message(
|
||||
chat_id: str,
|
||||
payload: CreateMessage,
|
||||
idempotency_key: str = Header(
|
||||
...,
|
||||
alias="Idempotency-Key",
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9._:-]+$",
|
||||
),
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session)
|
||||
):
|
||||
await check_chat_access(chat_id, user.id, session)
|
||||
|
||||
if payload.attachment_ids:
|
||||
result = await session.execute(
|
||||
select(Attachment.id).where(
|
||||
Attachment.id.in_(payload.attachment_ids),
|
||||
Attachment.chat_id == chat_id,
|
||||
)
|
||||
)
|
||||
found_ids = {str(row[0]) for row in result.all()}
|
||||
requested_ids = {str(a) for a in payload.attachment_ids}
|
||||
if found_ids != requested_ids:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="attachment_ids содержат файлы из другого чата или несуществующие"
|
||||
)
|
||||
|
||||
redis_key = f"idem:{user.id}:{chat_id}:{idempotency_key}"
|
||||
body = payload.model_dump(mode="json")
|
||||
|
||||
raw = await redis_client.get(redis_key)
|
||||
if raw:
|
||||
stored = json.loads(raw)
|
||||
if stored["state"] == "done":
|
||||
if stored["request_body"] == body:
|
||||
return stored["response"]
|
||||
raise HTTPException(status_code=409, detail="Idempotency-Key переиспользован с другим телом запроса")
|
||||
raise HTTPException(status_code=409, detail="Запрос с этим Idempotency-Key ещё обрабатывается")
|
||||
|
||||
acquired = await redis_client.set(
|
||||
redis_key,
|
||||
json.dumps({"state": "processing", "request_body": body}),
|
||||
nx=True,
|
||||
ex=86400,
|
||||
)
|
||||
if not acquired:
|
||||
raise HTTPException(409, "Одновременный запрос с тем же самым Idempotency-Key")
|
||||
|
||||
message = Message(
|
||||
chat_id=chat_id,
|
||||
sender_id=user.id,
|
||||
text=payload.text,
|
||||
attachment_ids=[str(a) for a in payload.attachment_ids]
|
||||
)
|
||||
session.add(message)
|
||||
await session.commit()
|
||||
await session.refresh(message)
|
||||
|
||||
response_data = {
|
||||
"id": str(message.id),
|
||||
"chat_id": str(message.chat_id),
|
||||
"sender_id": str(message.sender_id),
|
||||
"text": message.text,
|
||||
"created_at": message.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"attachment_ids": message.attachment_ids,
|
||||
}
|
||||
|
||||
await redis_client.set(
|
||||
redis_key,
|
||||
json.dumps({"state": "done",
|
||||
"request_body": body,
|
||||
"response": response_data}),
|
||||
ex=86400,
|
||||
)
|
||||
|
||||
return response_data
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_messages(
|
||||
chat_id: str,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
user: User = Depends(get_current_user),
|
||||
session: AsyncSession = Depends(get_session)
|
||||
):
|
||||
if limit < 1 or limit > 100:
|
||||
raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100")
|
||||
|
||||
await check_chat_access(chat_id, user.id, session)
|
||||
|
||||
offset = int(cursor) if cursor else 0
|
||||
|
||||
result = await session.execute(
|
||||
select(Message)
|
||||
.where(Message.chat_id == chat_id)
|
||||
.order_by(Message.created_at, Message.id)
|
||||
.limit(limit + 1)
|
||||
.offset(offset)
|
||||
)
|
||||
msgs = result.scalars().all()
|
||||
|
||||
has_next = len(msgs) > limit
|
||||
if has_next:
|
||||
msgs = msgs[:limit]
|
||||
|
||||
my_messages = [
|
||||
{
|
||||
"id": m.id,
|
||||
"chat_id": m.chat_id,
|
||||
"sender_id": m.sender_id,
|
||||
"text": m.text,
|
||||
"created_at": m.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"attachment_ids": m.attachment_ids,
|
||||
} for m in msgs
|
||||
]
|
||||
|
||||
next_cursor = str(offset + limit) if has_next else None
|
||||
return {"items": my_messages, "next_cursor": next_cursor}
|
||||
@@ -0,0 +1,55 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from pydantic import BaseModel, Field, ConfigDict, field_serializer
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from app.models.user import User
|
||||
from app.security import get_password_hash, get_current_user
|
||||
from app.dependencies import get_session
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=5, max_length=128)
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
id: uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
@field_serializer('created_at')
|
||||
def serialize_dt(self, dt: datetime):
|
||||
return dt.isoformat().replace("+00:00", "Z")
|
||||
|
||||
@router.post("", status_code=201, response_model=UserResponse)
|
||||
async def register_user(payload: CreateUser, session: AsyncSession = Depends(get_session)):
|
||||
result = await session.execute(select(User).where(User.username == payload.username))
|
||||
if result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="Имя пользователя уже существует")
|
||||
|
||||
user = User(
|
||||
username=payload.username,
|
||||
display_name=payload.display_name,
|
||||
password_hash=get_password_hash(payload.password)
|
||||
)
|
||||
session.add(user)
|
||||
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
raise HTTPException(status_code=409, detail="Имя пользователя уже существует")
|
||||
|
||||
await session.refresh(user)
|
||||
return user
|
||||
|
||||
@router.get("/me", response_model=UserResponse)
|
||||
async def current_user(user: User = Depends(get_current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,81 @@
|
||||
import secrets
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from passlib.context import CryptContext
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from sqlalchemy import select
|
||||
from app.database import async_session_maker
|
||||
from app.models.user import User
|
||||
from app.redis_client import redis_client
|
||||
from app.config import SESSION_TTL_SECONDS
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
bearer_scheme = HTTPBearer(auto_error=False)
|
||||
|
||||
def get_password_hash(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
async def create_session(user_id: str) -> dict:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
|
||||
await redis_client.setex(
|
||||
name=f"session:{token}",
|
||||
time=SESSION_TTL_SECONDS,
|
||||
value=str(user_id),
|
||||
)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at.isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
async def delete_session(token: str) -> None:
|
||||
await redis_client.delete(f"session:{token}")
|
||||
|
||||
|
||||
async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)):
|
||||
if not credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
token = credentials.credentials
|
||||
|
||||
|
||||
try:
|
||||
user_id = await redis_client.get(f"session:{token}")
|
||||
except Exception:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Внутренняя ошибка сервера"
|
||||
)
|
||||
|
||||
if not user_id:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or expired token",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="User not found",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
return user
|
||||
@@ -0,0 +1,22 @@
|
||||
import threading
|
||||
from minio import Minio
|
||||
from app.config import S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
|
||||
|
||||
users_lock = threading.Lock()
|
||||
messages_lock = threading.Lock()
|
||||
|
||||
users_by_id: dict[str, dict] = {}
|
||||
users_by_username: dict[str, dict] = {}
|
||||
chats_by_id: dict[str, dict] = {}
|
||||
messages_by_chat: dict[str, list[dict]] = {}
|
||||
|
||||
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
|
||||
|
||||
minio_client = Minio(
|
||||
endpoint,
|
||||
access_key=S3_ACCESS_KEY_ID,
|
||||
secret_key=S3_SECRET_ACCESS_KEY,
|
||||
secure=False,
|
||||
)
|
||||
|
||||
BUCKET = S3_BUCKET
|
||||
+639
@@ -0,0 +1,639 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import aio_pika
|
||||
from minio import Minio
|
||||
from PIL import Image
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
||||
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.user import User
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.config import (
|
||||
DATABASE_URL,
|
||||
S3_ENDPOINT,
|
||||
S3_ACCESS_KEY_ID,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
S3_BUCKET,
|
||||
)
|
||||
|
||||
RABBITMQ_URL = os.getenv(
|
||||
"RABBITMQ_URL",
|
||||
"amqp://guest:guest@rabbitmq:5672/",
|
||||
)
|
||||
|
||||
QUEUE_NAME = "image_tasks"
|
||||
|
||||
MAX_ATTEMPTS = max(
|
||||
1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5"))
|
||||
)
|
||||
|
||||
CONCURRENCY = max(
|
||||
1, int(os.getenv("WORKER_CONCURRENCY", "1"))
|
||||
)
|
||||
|
||||
PREFETCH = max(
|
||||
1, int(os.getenv("WORKER_PREFETCH", "1"))
|
||||
)
|
||||
|
||||
MAX_IMAGE_BYTES = 10 * 1024 * 1024
|
||||
PROCESS_TIMEOUT_SECONDS = 30
|
||||
LEASE_SECONDS = 300
|
||||
|
||||
endpoint = S3_ENDPOINT.replace(
|
||||
"http://", ""
|
||||
).replace(
|
||||
"https://", ""
|
||||
)
|
||||
|
||||
minio_client = Minio(
|
||||
endpoint,
|
||||
access_key=S3_ACCESS_KEY_ID,
|
||||
secret_key=S3_SECRET_ACCESS_KEY,
|
||||
secure=False,
|
||||
)
|
||||
|
||||
engine = create_async_engine(DATABASE_URL)
|
||||
|
||||
async_session_maker = async_sessionmaker(
|
||||
engine,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
|
||||
class RetryableProcessingError(Exception):
|
||||
def __init__(self, error_code: str):
|
||||
self.error_code = error_code
|
||||
super().__init__(error_code)
|
||||
|
||||
|
||||
class PermanentProcessingError(Exception):
|
||||
def __init__(self, error_code: str):
|
||||
self.error_code = error_code
|
||||
super().__init__(error_code)
|
||||
|
||||
|
||||
class LeaseBusyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LeaseLostError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_rss_mb():
|
||||
try:
|
||||
with open("/proc/self/status") as file:
|
||||
for line in file:
|
||||
if line.startswith("VmRSS:"):
|
||||
kb = int(line.split()[1])
|
||||
return round(kb / 1024, 2)
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def claim_attachment(attachment_id: str):
|
||||
async with async_session_maker() as session:
|
||||
async with session.begin():
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
uuid.UUID(attachment_id),
|
||||
with_for_update=True,
|
||||
)
|
||||
|
||||
if att is None:
|
||||
print(
|
||||
f"[Worker] Файл {attachment_id} не найден",
|
||||
flush=True,
|
||||
)
|
||||
return None
|
||||
|
||||
if att.status == "ready" and att.thumbnail_key:
|
||||
print(
|
||||
f"[Worker] Файл {attachment_id} уже обработан",
|
||||
flush=True,
|
||||
)
|
||||
return None
|
||||
|
||||
if att.status == "failed":
|
||||
print(
|
||||
f"[Worker] Файл {attachment_id} уже failed",
|
||||
flush=True,
|
||||
)
|
||||
return None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if (
|
||||
att.status == "processing"
|
||||
and att.lease_expires_at is not None
|
||||
and att.lease_expires_at > now
|
||||
):
|
||||
raise LeaseBusyError()
|
||||
|
||||
lease_expires_at = now + timedelta(
|
||||
seconds=LEASE_SECONDS
|
||||
)
|
||||
|
||||
att.status = "processing"
|
||||
att.error_code = None
|
||||
att.lease_expires_at = lease_expires_at
|
||||
|
||||
object_key = att.object_key
|
||||
|
||||
return object_key, lease_expires_at
|
||||
|
||||
|
||||
async def release_lease(
|
||||
attachment_id: str,
|
||||
lease_expires_at: datetime,
|
||||
):
|
||||
async with async_session_maker() as session:
|
||||
async with session.begin():
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
uuid.UUID(attachment_id),
|
||||
with_for_update=True,
|
||||
)
|
||||
|
||||
if (
|
||||
att is not None
|
||||
and att.status == "processing"
|
||||
and att.lease_expires_at == lease_expires_at
|
||||
):
|
||||
att.status = "queued"
|
||||
att.lease_expires_at = None
|
||||
|
||||
|
||||
async def mark_failed(
|
||||
attachment_id: str,
|
||||
error_code: str,
|
||||
):
|
||||
async with async_session_maker() as session:
|
||||
async with session.begin():
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
uuid.UUID(attachment_id),
|
||||
with_for_update=True,
|
||||
)
|
||||
|
||||
if att is None or att.status == "ready":
|
||||
return
|
||||
|
||||
if att.status == "processing":
|
||||
raise LeaseBusyError()
|
||||
|
||||
att.status = "failed"
|
||||
att.error_code = error_code
|
||||
att.lease_expires_at = None
|
||||
|
||||
print(
|
||||
f"[Worker] Файл {attachment_id}: "
|
||||
f"failed, error_code={error_code}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
async def process_image(image_data: bytes):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
directory = Path(temp_dir)
|
||||
|
||||
input_path = directory / "original"
|
||||
input_path.write_bytes(image_data)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"app.image_processor",
|
||||
str(input_path),
|
||||
str(directory),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise RetryableProcessingError(
|
||||
"image_processor_start_error"
|
||||
) from exc
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=PROCESS_TIMEOUT_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError as exc:
|
||||
if process.returncode is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
await process.communicate()
|
||||
|
||||
raise RetryableProcessingError(
|
||||
"image_processing_timeout"
|
||||
) from exc
|
||||
|
||||
if process.returncode == 20:
|
||||
raise PermanentProcessingError(
|
||||
"image_too_many_pixels"
|
||||
)
|
||||
|
||||
if process.returncode == 21:
|
||||
raise PermanentProcessingError(
|
||||
"image_processing_error"
|
||||
)
|
||||
|
||||
if process.returncode != 0:
|
||||
raise RetryableProcessingError(
|
||||
"image_processor_error"
|
||||
)
|
||||
|
||||
try:
|
||||
thumb_bytes = (
|
||||
directory / "thumbnail.jpg"
|
||||
).read_bytes()
|
||||
|
||||
crop_bytes = (
|
||||
directory / "crop.jpg"
|
||||
).read_bytes()
|
||||
|
||||
webp_bytes = (
|
||||
directory / "optimized.webp"
|
||||
).read_bytes()
|
||||
|
||||
with Image.open(
|
||||
io.BytesIO(thumb_bytes)
|
||||
) as thumbnail:
|
||||
thumb_width, thumb_height = thumbnail.size
|
||||
|
||||
except OSError as exc:
|
||||
raise RetryableProcessingError(
|
||||
"image_processor_output_error"
|
||||
) from exc
|
||||
|
||||
return (
|
||||
thumb_bytes,
|
||||
crop_bytes,
|
||||
webp_bytes,
|
||||
thumb_width,
|
||||
thumb_height,
|
||||
)
|
||||
|
||||
|
||||
async def process_attachment(attachment_id: str):
|
||||
print(
|
||||
f"[Worker] Обработка файла: {attachment_id}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
claim = await claim_attachment(attachment_id)
|
||||
|
||||
if claim is None:
|
||||
return
|
||||
|
||||
object_key, lease_expires_at = claim
|
||||
|
||||
try:
|
||||
try:
|
||||
response = minio_client.get_object(
|
||||
S3_BUCKET,
|
||||
object_key,
|
||||
)
|
||||
|
||||
try:
|
||||
image_data = response.read(
|
||||
MAX_IMAGE_BYTES + 1
|
||||
)
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
except Exception as exc:
|
||||
raise RetryableProcessingError(
|
||||
"s3_download_error"
|
||||
) from exc
|
||||
|
||||
if len(image_data) > MAX_IMAGE_BYTES:
|
||||
raise PermanentProcessingError(
|
||||
"image_too_large"
|
||||
)
|
||||
|
||||
(
|
||||
thumb_bytes,
|
||||
crop_bytes,
|
||||
webp_bytes,
|
||||
thumb_width,
|
||||
thumb_height,
|
||||
) = await process_image(image_data)
|
||||
|
||||
thumb_key = f"thumbnails/{object_key}"
|
||||
crop_key = f"crops/{attachment_id}.jpg"
|
||||
webp_key = f"webp/{attachment_id}.webp"
|
||||
|
||||
try:
|
||||
minio_client.put_object(
|
||||
S3_BUCKET,
|
||||
thumb_key,
|
||||
io.BytesIO(thumb_bytes),
|
||||
len(thumb_bytes),
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
|
||||
minio_client.put_object(
|
||||
S3_BUCKET,
|
||||
crop_key,
|
||||
io.BytesIO(crop_bytes),
|
||||
len(crop_bytes),
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
|
||||
minio_client.put_object(
|
||||
S3_BUCKET,
|
||||
webp_key,
|
||||
io.BytesIO(webp_bytes),
|
||||
len(webp_bytes),
|
||||
content_type="image/webp",
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
raise RetryableProcessingError(
|
||||
"s3_upload_error"
|
||||
) from exc
|
||||
|
||||
async with async_session_maker() as session:
|
||||
async with session.begin():
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
uuid.UUID(attachment_id),
|
||||
with_for_update=True,
|
||||
)
|
||||
|
||||
if (
|
||||
att is None
|
||||
or att.status != "processing"
|
||||
or att.lease_expires_at != lease_expires_at
|
||||
):
|
||||
raise LeaseLostError()
|
||||
|
||||
att.thumbnail_key = thumb_key
|
||||
att.thumbnail_sha256 = hashlib.sha256(
|
||||
thumb_bytes
|
||||
).hexdigest()
|
||||
att.thumbnail_size_bytes = len(thumb_bytes)
|
||||
att.thumbnail_width = thumb_width
|
||||
att.thumbnail_height = thumb_height
|
||||
att.crop_key = crop_key
|
||||
att.webp_key = webp_key
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
att.lease_expires_at = None
|
||||
|
||||
print(
|
||||
f"[Worker] Успешно обработан {attachment_id}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
await release_lease(
|
||||
attachment_id,
|
||||
lease_expires_at,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
async def main():
|
||||
connection = await aio_pika.connect_robust(
|
||||
RABBITMQ_URL
|
||||
)
|
||||
|
||||
async with connection:
|
||||
channel = await connection.channel(
|
||||
publisher_confirms=True
|
||||
)
|
||||
|
||||
await channel.set_qos(
|
||||
prefetch_count=PREFETCH
|
||||
)
|
||||
|
||||
queue = await channel.declare_queue(
|
||||
QUEUE_NAME,
|
||||
durable=True,
|
||||
)
|
||||
|
||||
semaphore = asyncio.Semaphore(CONCURRENCY)
|
||||
|
||||
print(
|
||||
f"[Worker] CONCURRENCY={CONCURRENCY}, "
|
||||
f"PREFETCH={PREFETCH}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
async def on_message(
|
||||
message: aio_pika.IncomingMessage
|
||||
):
|
||||
async with semaphore:
|
||||
started = time.perf_counter()
|
||||
|
||||
try:
|
||||
data = json.loads(
|
||||
message.body.decode()
|
||||
)
|
||||
|
||||
attachment_id = str(
|
||||
uuid.UUID(data["attachment_id"])
|
||||
)
|
||||
|
||||
except (
|
||||
ValueError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
):
|
||||
print(
|
||||
"[Worker] Некорректное сообщение",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await message.reject(
|
||||
requeue=False
|
||||
)
|
||||
return
|
||||
|
||||
headers = dict(message.headers or {})
|
||||
|
||||
try:
|
||||
attempt = max(
|
||||
1,
|
||||
int(headers.get("x-attempt", 1)),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
attempt = 1
|
||||
|
||||
print(
|
||||
f"[Worker] Попытка "
|
||||
f"{attempt}/{MAX_ATTEMPTS} "
|
||||
f"для {attachment_id}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
try:
|
||||
await process_attachment(
|
||||
attachment_id
|
||||
)
|
||||
|
||||
except LeaseBusyError:
|
||||
print(
|
||||
"[Worker] Задание уже "
|
||||
"обрабатывается, ждём Lease",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await asyncio.sleep(5)
|
||||
|
||||
await message.nack(
|
||||
requeue=True
|
||||
)
|
||||
return
|
||||
|
||||
except LeaseLostError:
|
||||
print(
|
||||
"[Worker] Lease перешёл "
|
||||
"другому Worker",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
except PermanentProcessingError as exc:
|
||||
try:
|
||||
await mark_failed(
|
||||
attachment_id,
|
||||
exc.error_code,
|
||||
)
|
||||
except Exception:
|
||||
await message.nack(
|
||||
requeue=True
|
||||
)
|
||||
return
|
||||
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
except Exception as exc:
|
||||
error_code = (
|
||||
exc.error_code
|
||||
if isinstance(
|
||||
exc,
|
||||
RetryableProcessingError,
|
||||
)
|
||||
else "worker_error"
|
||||
)
|
||||
|
||||
if attempt < MAX_ATTEMPTS:
|
||||
next_attempt = attempt + 1
|
||||
|
||||
print(
|
||||
f"[Worker] Ошибка {error_code}. "
|
||||
f"Повтор "
|
||||
f"{next_attempt}/{MAX_ATTEMPTS}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
headers["x-attempt"] = next_attempt
|
||||
|
||||
base_delay = min(
|
||||
30,
|
||||
2 ** (attempt - 1),
|
||||
)
|
||||
|
||||
delay = base_delay + random.uniform(
|
||||
0,
|
||||
base_delay / 2,
|
||||
)
|
||||
|
||||
print(
|
||||
f"[Worker] Повтор через "
|
||||
f"{delay:.1f} сек.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
try:
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.body,
|
||||
headers=headers,
|
||||
delivery_mode=(
|
||||
aio_pika.DeliveryMode.PERSISTENT
|
||||
),
|
||||
content_type="application/json",
|
||||
),
|
||||
routing_key=QUEUE_NAME,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
await message.nack(
|
||||
requeue=True
|
||||
)
|
||||
return
|
||||
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
print(
|
||||
f"[Worker] Попытки исчерпаны: "
|
||||
f"{attachment_id}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
try:
|
||||
await mark_failed(
|
||||
attachment_id,
|
||||
error_code,
|
||||
)
|
||||
except Exception:
|
||||
await message.nack(
|
||||
requeue=True
|
||||
)
|
||||
return
|
||||
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
await message.ack()
|
||||
|
||||
duration = time.perf_counter() - started
|
||||
rss_mb = get_rss_mb()
|
||||
|
||||
print(
|
||||
f"[Metrics] attachment_id={attachment_id} "
|
||||
f"duration_s={duration:.3f} "
|
||||
f"rss_mb={rss_mb}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print(
|
||||
"[Worker] Ждем сообщения из RabbitMQ...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
await queue.consume(on_message)
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,259 @@
|
||||
import csv
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from benchmark_rss import command, start_worker, remove_test_worker
|
||||
|
||||
RESULTS_DIR = Path("benchmark_results")
|
||||
RESULTS_DIR.mkdir(exist_ok=True)
|
||||
|
||||
QUEUE_NAME = "image_tasks"
|
||||
COUNT_PER_SIZE = 40
|
||||
TOTAL = COUNT_PER_SIZE * 3
|
||||
|
||||
|
||||
def queue_stats():
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "exec",
|
||||
"mtusi_messenger_rabbitmq",
|
||||
"rabbitmqctl", "-q", "list_queues",
|
||||
"name",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise RuntimeError(
|
||||
"RabbitMQ не ответил на запрос статистики"
|
||||
) from exc
|
||||
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"Не удалось получить статистику RabbitMQ:\n"
|
||||
+ result.stderr
|
||||
)
|
||||
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split()
|
||||
|
||||
if len(parts) >= 3 and parts[0] == QUEUE_NAME:
|
||||
return int(parts[1]), int(parts[2])
|
||||
|
||||
raise RuntimeError(
|
||||
f"Очередь {QUEUE_NAME} не найдена:\n"
|
||||
+ result.stdout
|
||||
)
|
||||
|
||||
|
||||
def run_test():
|
||||
print("Останавливаем обычный Worker...")
|
||||
command(["docker", "compose", "stop", "worker"])
|
||||
|
||||
print("Запускаем Worker 1/1...")
|
||||
start_worker(1, 1)
|
||||
|
||||
initial_ready, initial_unacked = queue_stats()
|
||||
|
||||
print(
|
||||
f"Начальная очередь: ready={initial_ready}, "
|
||||
f"unacked={initial_unacked}"
|
||||
)
|
||||
|
||||
if initial_ready or initial_unacked:
|
||||
raise RuntimeError(
|
||||
"Очередь не пустая. Для чистого эксперимента "
|
||||
"сначала нужно завершить предыдущие задания."
|
||||
)
|
||||
|
||||
print(f"\nЗапускаем нагрузку: {TOTAL} фотографий")
|
||||
|
||||
log_path = RESULTS_DIR / "backpressure_load.log"
|
||||
samples = []
|
||||
|
||||
with log_path.open("w") as log_file:
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
"docker", "compose", "exec",
|
||||
"-T",
|
||||
"-w", "/tmp",
|
||||
"-e", "PYTHONPATH=/app",
|
||||
"app1",
|
||||
"python", "-m", "app.load_test",
|
||||
"--count", str(COUNT_PER_SIZE),
|
||||
"--timeout", "600",
|
||||
],
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
try:
|
||||
while process.poll() is None:
|
||||
ready, unacked = queue_stats()
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
samples.append({
|
||||
"elapsed_s": round(elapsed, 2),
|
||||
"queue_ready": ready,
|
||||
"unacked": unacked,
|
||||
"total_pending": ready + unacked,
|
||||
})
|
||||
|
||||
print(
|
||||
f"[QUEUE] {elapsed:6.1f}s | "
|
||||
f"ready={ready:3} | "
|
||||
f"unacked={unacked} | "
|
||||
f"pending={ready + unacked:3}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
return_code = process.wait()
|
||||
|
||||
except BaseException:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise
|
||||
|
||||
output = log_path.read_text()
|
||||
|
||||
if return_code != 0:
|
||||
print(output)
|
||||
raise RuntimeError("Нагрузочный тест завершился с ошибкой")
|
||||
|
||||
print("\n" + output)
|
||||
|
||||
ready_match = re.search(
|
||||
r"^Ready:\s*(\d+)",
|
||||
output,
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
failed_match = re.search(
|
||||
r"^Failed:\s*(\d+)",
|
||||
output,
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
completed = (
|
||||
int(ready_match.group(1))
|
||||
if ready_match else -1
|
||||
)
|
||||
|
||||
failed = (
|
||||
int(failed_match.group(1))
|
||||
if failed_match else -1
|
||||
)
|
||||
|
||||
print("Проверяем окончательное состояние очереди...")
|
||||
|
||||
final_ready = None
|
||||
final_unacked = None
|
||||
|
||||
for _ in range(15):
|
||||
final_ready, final_unacked = queue_stats()
|
||||
|
||||
if final_ready == 0 and final_unacked == 0:
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
peak_ready = max(
|
||||
(row["queue_ready"] for row in samples),
|
||||
default=0,
|
||||
)
|
||||
|
||||
peak_unacked = max(
|
||||
(row["unacked"] for row in samples),
|
||||
default=0,
|
||||
)
|
||||
|
||||
csv_path = RESULTS_DIR / "backpressure_queue.csv"
|
||||
|
||||
with csv_path.open("w", newline="") as file:
|
||||
writer = csv.DictWriter(
|
||||
file,
|
||||
fieldnames=[
|
||||
"elapsed_s",
|
||||
"queue_ready",
|
||||
"unacked",
|
||||
"total_pending",
|
||||
],
|
||||
)
|
||||
|
||||
writer.writeheader()
|
||||
writer.writerows(samples)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BACKPRESSURE RESULTS")
|
||||
print("=" * 60)
|
||||
|
||||
print(f"Total jobs: {TOTAL}")
|
||||
print(f"Ready: {completed}")
|
||||
print(f"Failed: {failed}")
|
||||
print(f"Peak queued: {peak_ready}")
|
||||
print(f"Peak unacked: {peak_unacked}")
|
||||
print(f"Final queued: {final_ready}")
|
||||
print(f"Final unacked: {final_unacked}")
|
||||
|
||||
backlog_observed = peak_ready > 0
|
||||
prefetch_respected = peak_unacked <= 1
|
||||
recovered = (
|
||||
completed == TOTAL
|
||||
and failed == 0
|
||||
and final_ready == 0
|
||||
and final_unacked == 0
|
||||
)
|
||||
|
||||
print(
|
||||
"Backlog observed:",
|
||||
"YES" if backlog_observed else "NO",
|
||||
)
|
||||
|
||||
print(
|
||||
"Prefetch limit respected:",
|
||||
"YES" if prefetch_respected else "NO",
|
||||
)
|
||||
|
||||
print(
|
||||
"Recovery:",
|
||||
"PASSED" if recovered else "FAILED",
|
||||
)
|
||||
|
||||
print(f"CSV: {csv_path}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
run_test()
|
||||
|
||||
finally:
|
||||
print("\nВосстанавливаем обычный Worker...")
|
||||
|
||||
remove_test_worker()
|
||||
|
||||
result = command(
|
||||
["docker", "compose", "up", "-d", "worker"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
fa479f2c-54ac-44cf-baf0-e14d84b3fff2,512,ready,2.288,2.393
|
||||
8da11c17-a222-4532-888a-92cdf1bbab94,512,ready,4.895,5.104
|
||||
1441ad9e-fcc3-47c3-84e6-490559ad289a,512,ready,2.704,2.81
|
||||
a01f4d3e-8eb5-46d5-b778-cd956d9db214,512,ready,4.376,4.584
|
||||
53019d9f-949e-4aef-ab42-636d3fd330d2,512,ready,4.272,4.376
|
||||
a91a25c2-c464-4166-9c69-ec382a729bb8,1024,ready,4.584,4.895
|
||||
648e14f9-3596-44d3-870b-2c7e8357181a,1024,ready,5.419,5.835
|
||||
e127fe4e-b18d-4914-8945-3cb0aa9ef2ab,1024,ready,5.104,5.419
|
||||
4db57861-f63d-4971-8ccc-7fc1b5490f84,1024,ready,3.855,4.272
|
||||
eb1b8add-1ad5-41a9-a745-e8ee5fdaf779,1024,ready,0.831,1.249
|
||||
32dc271d-676c-4834-8575-8a273c13722b,2048,ready,0.831,2.288
|
||||
ffd1408c-cf40-4805-bf30-5a515bebd116,2048,ready,2.81,4.272
|
||||
a981c934-34b2-44fb-a09e-987c8ee66ac6,2048,ready,2.393,3.855
|
||||
5e69aead-9343-4ca7-bee9-27ae9dcbd785,2048,ready,1.249,2.704
|
||||
c0d08a45-7aa1-4afd-bd40-1d6864a4801e,2048,ready,4.272,5.627
|
||||
|
@@ -0,0 +1,16 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
e2b24b97-ad9b-46d1-8790-2041f58914ca,512,ready,1.235,1.337
|
||||
971a55e8-3fd0-4215-b752-fff9b31ab126,512,ready,6.981,7.082
|
||||
f2e85596-fbcc-4e43-a534-d1d27cd75fc9,512,ready,7.082,7.183
|
||||
0ad8283c-968d-4525-a80f-5860f1da4b8b,512,ready,2.681,2.785
|
||||
bcdaec76-41ae-454e-b45f-174a26326b19,512,ready,1.337,1.544
|
||||
1eaccbad-b7a5-49d1-afdf-a6a137137034,1024,ready,0.821,1.235
|
||||
b78d8044-cb8f-4501-9462-7ac880cc1bc4,1024,ready,1.544,1.857
|
||||
86200ea3-a17e-4074-b1ae-90641804c48c,1024,ready,0.208,0.821
|
||||
383bbc63-331c-4510-b74b-f316c91ad902,1024,ready,1.857,2.277
|
||||
9782bfbf-9fbe-4e22-9a4b-18fa02924ccc,1024,ready,2.277,2.681
|
||||
d5594ebc-56bd-4007-b9cf-855db2f59e7f,2048,ready,4.218,5.553
|
||||
1061bf47-bde5-49a1-9971-ef41eefe499e,2048,ready,7.183,8.618
|
||||
70f8f306-73e6-4794-83fe-93c29bfd2c72,2048,ready,2.785,4.218
|
||||
c505d22f-01f2-4396-bb27-ebf2d1ff473b,2048,ready,8.618,9.958
|
||||
793328a6-cf5e-4078-814e-64d4c4e1e66e,2048,ready,5.553,6.981
|
||||
|
@@ -0,0 +1,16 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
a940e027-a230-4343-a771-96ef998b207c,512,ready,1.155,1.364
|
||||
3a603917-37dd-4ce7-91fc-04771c87e7aa,512,ready,1.568,1.774
|
||||
1d20bc66-466e-4bd3-a754-57500694cb51,512,ready,1.467,1.568
|
||||
efc77f3e-f5d0-40da-9f33-155a0eb4db43,512,ready,1.155,1.364
|
||||
32a3e8e9-bf48-4db0-9333-5cf0cc183acc,512,ready,1.364,1.467
|
||||
ecf65392-5165-4990-bcb4-42d80628cd86,1024,ready,1.774,2.19
|
||||
b8bcc085-d1d9-411a-ac1c-08688140d4c8,1024,ready,1.155,1.568
|
||||
7eeed358-a38b-4d39-a4ac-55476bfcdcfe,1024,ready,1.364,1.774
|
||||
1e0fb527-7023-4cc2-8292-56b7bbef10c8,1024,ready,1.155,1.568
|
||||
f29de97b-d1c1-4200-b80a-4b7babdfc2d1,1024,ready,1.774,2.19
|
||||
8e2bb5c9-e613-4343-8fc1-fe6d3eda869d,2048,ready,3.647,5.102
|
||||
2038744a-177e-4a3b-bc22-85d6b82fc879,2048,ready,1.568,3.544
|
||||
748dbf01-4049-4297-837f-cb3457875e10,2048,ready,2.19,3.859
|
||||
f578c650-71b8-472b-8a0f-c6c09bcd00e3,2048,ready,2.19,4.171
|
||||
3ed1e2a1-43d5-4f9f-a475-01fa3cac932e,2048,ready,1.568,3.544
|
||||
|
@@ -0,0 +1,64 @@
|
||||
elapsed_s,queue_ready,unacked,total_pending
|
||||
0.37,0,0,0
|
||||
1.74,0,0,0
|
||||
3.15,119,1,120
|
||||
4.5,118,1,119
|
||||
5.81,115,1,116
|
||||
7.16,114,1,115
|
||||
8.45,112,1,113
|
||||
9.75,111,1,112
|
||||
11.06,108,1,109
|
||||
12.37,107,1,108
|
||||
13.67,104,1,105
|
||||
14.98,101,1,102
|
||||
16.26,99,1,100
|
||||
17.55,96,1,97
|
||||
18.85,93,1,94
|
||||
20.14,92,1,93
|
||||
21.44,91,1,92
|
||||
22.74,86,1,87
|
||||
24.05,84,1,85
|
||||
25.35,82,1,83
|
||||
26.66,80,1,81
|
||||
27.98,75,1,76
|
||||
29.27,71,1,72
|
||||
30.57,68,1,69
|
||||
31.87,66,1,67
|
||||
33.18,65,1,66
|
||||
34.48,64,1,65
|
||||
35.77,63,1,64
|
||||
37.06,60,1,61
|
||||
38.43,58,1,59
|
||||
39.76,57,1,58
|
||||
41.05,57,1,58
|
||||
42.37,52,1,53
|
||||
43.67,51,1,52
|
||||
44.98,50,1,51
|
||||
46.27,49,1,50
|
||||
47.58,48,1,49
|
||||
48.87,46,1,47
|
||||
50.17,45,1,46
|
||||
51.47,44,1,45
|
||||
52.77,43,1,44
|
||||
54.08,42,1,43
|
||||
55.37,39,1,40
|
||||
56.68,38,1,39
|
||||
57.97,37,1,38
|
||||
59.26,35,1,36
|
||||
60.55,33,1,34
|
||||
61.84,31,1,32
|
||||
63.15,30,1,31
|
||||
64.44,29,1,30
|
||||
65.73,23,1,24
|
||||
67.03,22,1,23
|
||||
68.34,18,1,19
|
||||
69.63,17,1,18
|
||||
70.93,14,1,15
|
||||
72.23,12,1,13
|
||||
73.52,11,1,12
|
||||
74.82,9,1,10
|
||||
76.11,6,1,7
|
||||
77.4,5,1,6
|
||||
78.69,2,1,3
|
||||
79.98,1,1,2
|
||||
81.28,0,1,1
|
||||
|
@@ -0,0 +1,61 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
28bebbfc-799e-4217-8c30-4f47f1e46c7e,512,ready,5.683,5.893
|
||||
96b8fb1e-3ca4-497e-a4b0-0d1f73522c44,512,ready,10.223,10.328
|
||||
ea372b62-a051-4cc1-afc5-dde4fdfdb85f,512,ready,5.893,6.098
|
||||
6d4cae25-8487-401b-b89f-16fd3e012079,512,ready,6.098,6.201
|
||||
42bcd40d-5de7-42d9-9cac-511bc000160f,512,ready,4.137,4.241
|
||||
f6c50369-fcfa-48c4-b04d-8599c2c20711,512,ready,5.376,5.478
|
||||
f55f9964-9adc-4adc-b48e-9c93d4019889,512,ready,5.893,6.098
|
||||
b3ea2331-706e-4dd1-81ae-8972c922075f,512,ready,4.137,4.241
|
||||
2f0d8af7-ee6f-479a-96b9-c24c311b1790,512,ready,9.709,9.812
|
||||
9fca7598-037f-4f6a-81dd-038bc8f5c9d3,512,ready,3.93,4.137
|
||||
3c1673ca-f03b-41d4-8706-f8a65a1951af,512,ready,2.901,3.003
|
||||
0eaad040-3f62-446d-842e-fc4a21cb74cc,512,ready,5.893,6.098
|
||||
ee0404fd-8dd6-49dd-9a77-e849a0382ff2,512,ready,0.843,1.048
|
||||
20f6197b-e3fb-4ce8-9985-13aa260a3e9f,512,ready,5.478,5.683
|
||||
9d401a3c-d5d5-45b9-9153-8a392132c74a,512,ready,5.683,5.893
|
||||
b2de90a9-4354-4c66-b905-c2cd501e94b2,512,ready,3.829,4.034
|
||||
ed1ded18-31cc-495e-9e1e-7a15cb77fb67,512,ready,5.683,5.893
|
||||
a8e31c1d-0ef1-4fea-a538-3fd9e36a594c,512,ready,6.201,6.406
|
||||
5451263a-8d6b-4940-b44b-87c1bbb66b97,512,ready,8.671,8.773
|
||||
0af8a964-e248-4153-936a-7bd5f7ec9c82,512,ready,8.049,8.259
|
||||
c6ab01d9-cb1e-4154-8678-77453b549a82,1024,ready,9.812,10.223
|
||||
b3f6ab04-bfe9-45c3-a409-d8aed2efba92,1024,ready,5.893,6.303
|
||||
de9ed0d6-8d6d-41a4-8eb4-16eb6793eca3,1024,ready,9.297,9.709
|
||||
452d6dce-5947-4d08-8983-37b9b517ea0a,1024,ready,8.362,8.773
|
||||
91f662ce-6cec-4586-b2a0-1a5cc101340b,1024,ready,1.048,1.46
|
||||
b41f3d80-1b54-4402-9928-a83e67e2aa2f,1024,ready,2.491,2.901
|
||||
4b7cfdbf-8bdc-4569-96dc-8410b0fb965d,1024,ready,3.003,3.412
|
||||
19df5c43-84d5-4fc6-8469-daf469f8d096,1024,ready,1.46,1.975
|
||||
c5211831-85d2-4c44-a871-9acd71a17337,1024,ready,8.259,8.671
|
||||
369d13b1-ead7-4aed-bcd4-9f12e918d038,1024,ready,4.034,4.452
|
||||
7e9c5099-2f5e-4ea9-a6e2-d4acc540901f,1024,ready,7.946,8.362
|
||||
927b907a-fbc4-4383-a6e3-0fd02206f79c,1024,ready,6.098,6.51
|
||||
d96167a6-cda0-46a8-adad-df52ccc789bd,1024,ready,9.709,10.121
|
||||
e9577c3a-8d88-46c9-9ad5-1979469b38be,1024,ready,0.843,1.356
|
||||
52f8e432-ff40-4419-8968-17ef1089a361,1024,ready,4.966,5.376
|
||||
99a1e17e-94b0-4ca1-810f-fa5e3ff9691f,1024,ready,3.31,3.726
|
||||
33eb7872-0980-4336-a560-cf6f14a302f9,1024,ready,9.297,9.709
|
||||
4a3cacba-350f-4878-85f7-ead76d7cacbe,1024,ready,3.726,4.137
|
||||
fc3cb344-8f7c-4421-bae0-ce5e282d6143,1024,ready,1.975,2.39
|
||||
22228d29-952b-4ec0-aad0-b8762967dee0,1024,ready,1.356,1.768
|
||||
1d29e426-37fa-41e4-b5cc-55eedbf2af38,2048,ready,8.773,10.223
|
||||
d62cc431-925c-47c6-9e0a-bec4c2d561dc,2048,ready,0.843,2.491
|
||||
4d3d5437-fded-4c5a-9ab5-50460df9fade,2048,ready,4.452,5.893
|
||||
78668df5-05d3-4ca1-b309-fcb16181d973,2048,ready,4.241,5.683
|
||||
86af36aa-2193-4da2-a786-1787fb6229d5,2048,ready,2.39,3.829
|
||||
211fbe0b-a085-4549-812a-07a2c326a9b2,2048,ready,6.51,8.049
|
||||
f8e04547-8aaa-4b87-9e9d-383e387ade00,2048,ready,0.843,2.491
|
||||
9b065c78-8c00-4fe5-a890-bfa698c8bfea,2048,ready,4.241,5.683
|
||||
3906bf23-8afc-47d1-85c5-b2d0eb207236,2048,ready,10.223,11.673
|
||||
4a2f285d-327c-44be-ba2c-da0d627109cc,2048,ready,6.406,7.946
|
||||
f1f72a39-92e6-4284-a373-30214f1141bc,2048,ready,10.223,11.673
|
||||
9eee27d0-b3fb-4be4-ab0f-3f99b0d024f1,2048,ready,6.098,7.741
|
||||
cb946763-c80a-4284-85b3-3aea16d2eced,2048,ready,10.223,11.673
|
||||
f0aed629-1d01-4f7a-a699-eff85d4dd239,2048,ready,7.844,9.297
|
||||
b7957cd4-0942-40cd-a1f9-a4a614d4c5f3,2048,ready,8.773,10.223
|
||||
ea48ce0d-a3e8-44f3-bdbc-37d1ddfc9eb0,2048,ready,7.844,9.297
|
||||
e6a26bb8-4c70-4074-b130-357a742c6b5f,2048,ready,1.768,3.31
|
||||
1530b7a5-227c-4bb0-ad09-d975105a757c,2048,ready,3.412,4.864
|
||||
e9bcf42c-2a3b-43ee-8802-3d1432bfba5a,2048,ready,6.303,7.844
|
||||
390b5116-460d-45f6-976b-240c2b6066bf,2048,ready,2.491,3.93
|
||||
|
@@ -0,0 +1,61 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
00eb6bd3-3102-44e1-9be4-a5194b56cbd5,512,ready,2.814,3.021
|
||||
cde0e2c2-02c2-4c05-bf05-6a98d0acd71a,512,ready,17.363,17.464
|
||||
62fe77fe-bdf2-4dbb-85c7-9ff6981bc5f5,512,ready,16.524,16.626
|
||||
36b35f4e-5497-4165-b0f6-bb3724e9cd00,512,ready,11.442,11.544
|
||||
7f4286a2-d91e-40fb-8231-bd7674cfa8bc,512,ready,7.172,7.274
|
||||
f34c5f28-536f-45b6-abf3-88a8229dd945,512,ready,3.021,3.123
|
||||
1caa346a-80e8-44a7-9dfb-1e83bf71534e,512,ready,11.337,11.442
|
||||
cf10641f-fbb8-43ea-88e0-22daa1627188,512,ready,2.095,2.198
|
||||
98099d7b-724d-4e70-a4cf-9a6ab019333b,512,ready,12.906,13.111
|
||||
b26e4907-1919-4dce-a7da-6909890b2d5f,512,ready,3.854,3.958
|
||||
e3468d1b-5637-4e5a-aa70-0249559b4cfd,512,ready,2.712,2.814
|
||||
2f9f2e4a-d4e9-47d6-8c87-25043e5199ae,512,ready,1.89,2.095
|
||||
de8fbd2e-e392-4663-98a1-96d93542280f,512,ready,11.442,11.544
|
||||
9d3f5e09-c41f-4018-bdab-175b00a452e6,512,ready,7.274,7.484
|
||||
e56365eb-2d10-4d3c-ad32-931f8a31d030,512,ready,2.608,2.712
|
||||
4bc57fb6-32c7-47ca-8899-ea40df7da799,512,ready,11.134,11.337
|
||||
d2522db6-f17a-4e18-9d79-386a195fd633,512,ready,3.539,3.641
|
||||
6d1cc5f9-d992-44cf-9ed0-530467774337,512,ready,4.368,4.572
|
||||
985e8927-4f0b-44fa-bd09-fb19d6ea29fe,512,ready,11.235,11.442
|
||||
50c58946-727b-48d2-9809-3f7a390ea741,512,ready,3.641,3.854
|
||||
39ee9ce0-1494-4f3a-ab1f-7c03dcc546b2,1024,ready,1.058,1.578
|
||||
9778715e-f4c6-4fa0-8c3c-64e621611fda,1024,ready,3.641,4.059
|
||||
72ba59cb-3e56-483b-b65d-c53e0cf7c302,1024,ready,11.544,12.065
|
||||
cb97fe45-a6b2-43c8-bd1b-b51824014994,1024,ready,4.572,4.885
|
||||
df00bd9f-2470-4390-ace0-920e3f206b97,1024,ready,10.723,11.134
|
||||
9694c476-cb8a-46b8-9e8d-57cba2152a17,1024,ready,14.336,14.745
|
||||
42169eb9-3324-4c2a-99ca-36148faaa150,1024,ready,13.111,13.519
|
||||
6c26553f-30ba-4f18-9c3c-d08472afc7f0,1024,ready,18.499,18.913
|
||||
8315304d-7550-4eb9-82d8-0db4db41aa04,1024,ready,3.123,3.539
|
||||
d8ee601e-6b75-417c-affa-1b24b1c1a02a,1024,ready,6.76,7.172
|
||||
e4536000-5159-486f-97a3-8e0727854ab2,1024,ready,6.341,6.76
|
||||
77809b9b-ae97-41b3-9463-cbe70acf2a0c,1024,ready,13.519,13.926
|
||||
757b621c-fb42-499d-b4dd-721d9a9e876a,1024,ready,1.578,1.89
|
||||
18ba2c34-56eb-46ec-a54b-20b2d2752ac5,1024,ready,12.065,12.485
|
||||
75ebd991-3525-459a-a1dd-c71c6e78663b,1024,ready,13.926,14.336
|
||||
5b5a607f-76b6-48f7-ba9e-09d9aa732cf4,1024,ready,3.958,4.368
|
||||
813b7fe0-f445-4420-a8ab-757046d266b6,1024,ready,12.485,12.906
|
||||
19fb3f65-760e-4e53-bd98-dc51dcbf237f,1024,ready,16.733,17.056
|
||||
1ff822e0-69ae-461e-9df3-c27261f9009a,1024,ready,7.484,7.901
|
||||
b24dbee4-e1c5-4a18-8f62-3f5012559cd4,1024,ready,16.103,16.524
|
||||
baa5cea0-5554-4a32-90a6-e3ad6c204228,2048,ready,8.428,9.891
|
||||
c58ddcd7-ffa3-46f6-b3dc-ea8f0fe9c1f4,2048,ready,18.913,20.363
|
||||
c4636200-eae1-49d4-8e59-f5dfa5b86c36,2048,ready,9.891,11.235
|
||||
c5d2e767-ce88-4682-82f5-80b9ab209580,2048,ready,4.059,5.504
|
||||
bf65fcbd-3ec1-4851-ab11-7f9239cdf1e5,2048,ready,7.901,9.268
|
||||
04533086-f63d-4aa8-9926-fab91e5ebf52,2048,ready,15.894,17.363
|
||||
7d88df73-ae6b-4e4a-821e-3de54784e87c,2048,ready,2.198,3.641
|
||||
06bc64bf-f949-48a1-a0d4-61d8e2053523,2048,ready,14.745,16.103
|
||||
7751ba6d-1bd3-48a1-91dd-6f9d837e37f2,2048,ready,4.885,6.341
|
||||
d3a36f16-3a76-43a4-9bc3-b806fb264ae1,2048,ready,14.541,15.894
|
||||
e9daef68-894e-4433-9004-8e515f539203,2048,ready,9.268,10.723
|
||||
f1afb046-e7d2-46e3-a639-bb9abd841039,2048,ready,20.363,21.721
|
||||
29d51f63-9f4f-4f80-b493-7c55823c6015,2048,ready,13.111,14.541
|
||||
0770bd79-18dc-4c61-8daf-5eddebac520f,2048,ready,1.058,2.608
|
||||
eb6b54b5-c1ab-4f7a-a527-78dcb2c5cbac,2048,ready,5.504,6.964
|
||||
09f1c7ae-0050-437e-ad85-4a416b813010,2048,ready,6.964,8.428
|
||||
939d499e-3d5e-4e86-8e08-19dc70d73b18,2048,ready,11.544,13.111
|
||||
64942f86-b40b-4aa5-99f3-b64e2617ddfc,2048,ready,18.913,20.363
|
||||
37210ed4-847c-48f1-be14-924231466055,2048,ready,17.464,18.913
|
||||
e1506996-5a46-42af-8083-dd815d6c2050,2048,ready,17.056,18.499
|
||||
|
@@ -0,0 +1,61 @@
|
||||
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
|
||||
6d0948d1-018f-4649-bc93-3839a19d960b,512,ready,32.346,32.448
|
||||
821f8778-92d6-4570-a1f3-f2d0bc20a4bc,512,ready,7.948,8.055
|
||||
2c44d62f-bbaf-4443-95ae-9025bd84f48d,512,ready,11.024,11.132
|
||||
a5bbe073-3027-42ae-98a4-2e93451a7895,512,ready,10.819,11.024
|
||||
c6314a8d-0a91-4a9d-a466-ae1959418c44,512,ready,20.047,20.151
|
||||
93fe6aee-0a67-4186-b5de-a7c8911c9c92,512,ready,4.143,4.347
|
||||
0ca9a2cc-d750-4f9f-96ac-e1c26b14c751,512,ready,30.364,30.573
|
||||
58b114fd-2289-405a-b1b1-aeed97690be3,512,ready,14.81,15.014
|
||||
38d4dc82-c0af-4a2e-ba4d-5e83c7d39bb6,512,ready,15.014,15.118
|
||||
f4b6d8b6-0abf-4de9-92bc-79427df63c1b,512,ready,15.731,15.937
|
||||
53992b21-20c3-434d-b4c3-4375f50a2d5a,512,ready,20.668,20.873
|
||||
65e871d7-7b80-43e2-84a2-01c77dc4b0ef,512,ready,15.527,15.629
|
||||
0284edbb-fbde-481f-971f-ee483f41da53,512,ready,36.524,36.738
|
||||
8244a394-feef-493a-acad-784ee2f63985,512,ready,18.098,18.2
|
||||
289ecaf9-2ea5-42ed-9757-aaf20eace246,512,ready,38.816,39.023
|
||||
10863feb-0bdf-426c-8827-e779e397a2c9,512,ready,15.629,15.731
|
||||
9fa71750-b870-47e3-b95f-f814fcc2ec49,512,ready,33.289,33.391
|
||||
abac43bc-c866-4c99-bd32-7e7b91ee54ef,512,ready,20.563,20.668
|
||||
d6072440-3839-420d-b74d-98683f8ed80a,512,ready,3.628,3.732
|
||||
5c0b5e43-e8df-4e6c-8774-df2698340e00,512,ready,14.708,14.81
|
||||
e4cb92cf-7f15-4598-b8db-e2bb195cf897,1024,ready,30.573,30.99
|
||||
09390dfc-b29a-4746-8353-7355b93cd34e,1024,ready,39.023,39.336
|
||||
65464b2b-1e23-464b-86c5-f506248392a9,1024,ready,28.19,28.608
|
||||
6326723a-8739-4afc-91b1-c5f86dcc883d,1024,ready,24.988,25.399
|
||||
c0528510-a4e8-4b6b-a5fd-a8510730a91c,1024,ready,18.2,18.614
|
||||
353e643e-0d25-4ee5-a18d-9101a32049f2,1024,ready,20.151,20.563
|
||||
9d6d2329-d9bc-4299-a1ab-d6b27881457c,1024,ready,32.872,33.289
|
||||
b68909e6-f9f2-466f-9938-9e8fd125758f,1024,ready,7.539,7.948
|
||||
df4a4574-25ae-4eca-b967-0bd8f5a85aea,1024,ready,17.268,17.689
|
||||
2e704f53-ae9c-4948-bd09-b8110f935baf,1024,ready,33.391,33.806
|
||||
1a17e811-2645-49aa-8ad9-11e7d499b999,1024,ready,4.347,4.655
|
||||
b1f5e98c-6587-4fdb-aee9-ccb8c6cd6f99,1024,ready,28.608,29.021
|
||||
b7abcbbc-3ed9-4e85-b5e1-5d90b132034d,1024,ready,15.118,15.527
|
||||
3e3a19a7-5485-45f4-a8a4-387f9002e54c,1024,ready,3.732,4.143
|
||||
73aad940-7807-4f21-a83e-8ba1cd15863e,1024,ready,36.738,37.048
|
||||
015511fd-3538-47d9-b7a0-f14969501967,1024,ready,38.509,38.816
|
||||
cd33c6d5-6913-4e21-a36c-9f13ca3c7bce,1024,ready,12.46,12.869
|
||||
2a2915a9-a211-4ac3-bdbf-a06b6bccbb04,1024,ready,17.689,18.098
|
||||
9084e41a-d7cb-450b-8efb-0110b39d20a8,1024,ready,32.448,32.872
|
||||
47d491a2-8f43-4b09-86be-939ed73d3216,1024,ready,12.869,13.277
|
||||
ac499ba7-af23-4626-8386-18dd34b7d0cb,2048,ready,15.937,17.268
|
||||
361ac439-f549-41ce-ba31-f0fdcc3e5a9d,2048,ready,6.097,7.539
|
||||
f4831394-fdbe-421e-99b1-3ebb9d635f7a,2048,ready,8.055,9.487
|
||||
e7c14f44-5b76-4950-a77c-a2d8edcd14a1,2048,ready,18.614,20.047
|
||||
d40b2c16-d32c-4244-94f5-255d77d0d189,2048,ready,37.048,38.509
|
||||
c61f7efe-e5e4-49fd-ab9d-b4d813d42697,2048,ready,35.169,36.524
|
||||
3b1bfd22-01e8-4874-8ea2-7716785f88e1,2048,ready,4.655,6.097
|
||||
21fa4d34-4dcc-487c-9205-ab1e48d05d09,2048,ready,22.221,23.658
|
||||
b169f1bf-5ede-447e-aa4d-c7886b80e6a8,2048,ready,33.806,35.169
|
||||
2c9a01c7-e488-4ca6-bc70-19e7b20d91a6,2048,ready,13.277,14.708
|
||||
8f439608-d82f-4050-b939-5c35d9938eb2,2048,ready,0.735,2.179
|
||||
fe635dd1-5fc7-4c82-a63a-0819dcac246c,2048,ready,23.658,24.988
|
||||
1be24fbc-bf40-4883-9029-6075d2f4c557,2048,ready,30.99,32.346
|
||||
ba19044d-d21c-4706-a263-240fdafb5749,2048,ready,26.843,28.19
|
||||
52e7cc24-f8a3-443b-b62d-93c143ce75a7,2048,ready,2.179,3.628
|
||||
97f34f85-f51f-4c8c-8f27-ae573690f1c8,2048,ready,29.021,30.364
|
||||
4228d8dd-5da4-4532-a132-9b27308cc1e7,2048,ready,25.399,26.843
|
||||
d58514f2-17f8-481e-b5c5-ae054324f80e,2048,ready,20.873,22.221
|
||||
51f3190e-f63b-4b0e-880c-947089cdfef9,2048,ready,9.487,10.819
|
||||
e5de7654-15d1-4f47-9f22-d2cab4c76737,2048,ready,11.132,12.46
|
||||
|
@@ -0,0 +1,74 @@
|
||||
elapsed_s,rss_mb
|
||||
0.0018562499899417162,63.9609375
|
||||
0.5586906669195741,63.9609375
|
||||
1.1478328749071807,63.9609375
|
||||
1.7124463329091668,63.9609375
|
||||
2.3201119999866933,154.03125
|
||||
2.886150083038956,181.53125
|
||||
3.454367541940883,208.90625
|
||||
4.0352518749423325,172.203125
|
||||
4.596314958063886,199.328125
|
||||
5.158594999928027,94.390625
|
||||
5.731980707962066,90.484375
|
||||
6.294250082923099,160.44921875
|
||||
6.866898791864514,187.32421875
|
||||
7.449180916883051,214.32421875
|
||||
8.02075345790945,175.76953125
|
||||
8.586141041945666,203.89453125
|
||||
9.151447457959875,111.85546875
|
||||
9.7154242079705,164.8046875
|
||||
10.274698958033696,192.0546875
|
||||
10.821836207993329,214.4296875
|
||||
11.391208708053455,178.421875
|
||||
11.957092249998823,206.921875
|
||||
12.530426000012085,95.53515625
|
||||
13.091126542072743,180.48828125
|
||||
13.658847207902,208.73828125
|
||||
14.223303207894787,115.96875
|
||||
14.79374916688539,122.56640625
|
||||
15.352951207896695,183.80078125
|
||||
15.913661292055622,212.42578125
|
||||
16.488913499983028,94.2734375
|
||||
17.04698033304885,89.0625
|
||||
17.61788404197432,166.3125
|
||||
18.182629582937807,194.5625
|
||||
18.746452332939953,73.2265625
|
||||
19.316664916928858,110.76953125
|
||||
19.88158891699277,112.84765625
|
||||
20.447920500067994,173.1171875
|
||||
21.019828042015433,200.3671875
|
||||
21.583593416959047,95.69921875
|
||||
22.148048375034705,68.84375
|
||||
22.701521083014086,174.2734375
|
||||
23.266354499850422,202.7734375
|
||||
23.836700625019148,161.3046875
|
||||
24.401869332883507,188.8046875
|
||||
24.96454008296132,214.8046875
|
||||
25.525170125067234,175.265625
|
||||
26.084088041912764,203.515625
|
||||
26.649612499866635,111.0
|
||||
27.214416875038296,170.390625
|
||||
27.77859991695732,198.640625
|
||||
28.34497074992396,122.6796875
|
||||
28.900013708043844,184.03515625
|
||||
29.459034791914746,212.28515625
|
||||
30.019986792001873,106.3125
|
||||
30.584227999905124,160.7734375
|
||||
31.149346332997084,187.8984375
|
||||
31.712559124920517,214.6484375
|
||||
32.281295999884605,116.64453125
|
||||
32.84523691702634,176.890625
|
||||
33.407698791939765,204.765625
|
||||
33.96293962490745,104.8046875
|
||||
34.52196699986234,113.38671875
|
||||
35.08372745802626,116.0078125
|
||||
35.665793374879286,176.3828125
|
||||
36.23321974999271,204.6328125
|
||||
36.79392512491904,162.4609375
|
||||
37.351651957957074,190.7109375
|
||||
37.91607549996115,214.7109375
|
||||
38.483901750063524,118.4921875
|
||||
39.04901108285412,178.9453125
|
||||
39.61435758299194,207.6953125
|
||||
40.18525333306752,116.02734375
|
||||
40.75472962507047,117.7734375
|
||||
|
@@ -0,0 +1,43 @@
|
||||
elapsed_s,rss_mb
|
||||
0.006153792142868042,63.90234375
|
||||
0.5778734171763062,63.90234375
|
||||
1.147869042120874,63.90234375
|
||||
1.7034579580649734,63.90234375
|
||||
2.276205125031993,63.90234375
|
||||
2.8752599169965833,194.703125
|
||||
3.4416670829523355,230.83203125
|
||||
3.9975552500691265,302.11328125
|
||||
4.574312333017588,213.53515625
|
||||
5.14147137501277,263.47265625
|
||||
5.7091500831302255,110.90625
|
||||
6.2797843751031905,220.01171875
|
||||
6.835572292096913,304.51953125
|
||||
7.403642917051911,289.5234375
|
||||
7.963550708023831,196.44921875
|
||||
8.53717741696164,258.8515625
|
||||
9.096077542053536,186.86328125
|
||||
9.66255066706799,292.37890625
|
||||
10.221317625138909,277.51171875
|
||||
10.789583917008713,333.26171875
|
||||
11.363086167024449,317.81640625
|
||||
11.92844095802866,303.1171875
|
||||
12.479885042179376,243.38671875
|
||||
13.050684583140537,94.1015625
|
||||
13.61427737516351,182.80078125
|
||||
14.231398458126932,240.96484375
|
||||
14.796835124958307,197.81640625
|
||||
15.373798333108425,231.953125
|
||||
15.93250362505205,221.671875
|
||||
16.498262125067413,263.9453125
|
||||
17.066447000019252,319.6953125
|
||||
17.630022125085816,269.69140625
|
||||
18.18958625011146,206.35546875
|
||||
18.755617167102173,245.66015625
|
||||
19.313573125051335,277.03125
|
||||
19.88116762507707,332.53125
|
||||
20.441875083139166,269.94140625
|
||||
21.01258633309044,281.08984375
|
||||
21.573833167087287,337.21484375
|
||||
22.150734583148733,166.046875
|
||||
22.70690429210663,193.421875
|
||||
23.275886042043567,218.296875
|
||||
|
@@ -0,0 +1,25 @@
|
||||
elapsed_s,rss_mb
|
||||
0.0017751660197973251,63.90234375
|
||||
0.5725909580942243,63.90234375
|
||||
1.146317790960893,63.90234375
|
||||
1.712353958049789,63.90234375
|
||||
2.2800673749297857,63.90234375
|
||||
2.886254500132054,342.91015625
|
||||
3.4468102499376982,361.43359375
|
||||
4.018739791121334,518.78515625
|
||||
4.595743082929403,414.22265625
|
||||
5.1572650000452995,385.99609375
|
||||
5.722510583000258,270.40625
|
||||
6.291970041114837,495.10546875
|
||||
6.863853875081986,489.14453125
|
||||
7.440310999983922,251.58203125
|
||||
7.994805958122015,232.55078125
|
||||
8.561554083134979,507.96484375
|
||||
9.145290333079174,536.33984375
|
||||
9.71233612508513,399.1328125
|
||||
10.283997416030616,374.50390625
|
||||
10.863315333146602,531.59375
|
||||
11.457262207986787,337.00390625
|
||||
12.02924433303997,245.5234375
|
||||
12.59735562512651,383.62109375
|
||||
13.175934208091348,467.24609375
|
||||
|
@@ -0,0 +1,4 @@
|
||||
config,total,ready,failed,duration_s,throughput,peak_processing,peak_rss_mb,mean_rss_mb
|
||||
1/1,60,60,0,39.34,1.53,1.0,214.8,155.83
|
||||
2/2,60,60,0,21.72,2.76,2.0,337.21,224.02
|
||||
4/4,60,60,0,11.67,5.14,4.0,536.34,327.69
|
||||
|
@@ -0,0 +1,297 @@
|
||||
import csv
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
CONFIGS = [(1, 1), (2, 2), (4, 4)]
|
||||
COUNT_PER_SIZE = 20
|
||||
RESULTS_DIR = Path("benchmark_results")
|
||||
RESULTS_DIR.mkdir(exist_ok=True)
|
||||
|
||||
RSS_CODE = """
|
||||
import os
|
||||
|
||||
total_kb = 0
|
||||
current_pid = os.getpid()
|
||||
|
||||
for name in os.listdir("/proc"):
|
||||
if not name.isdigit():
|
||||
continue
|
||||
|
||||
if int(name) == current_pid:
|
||||
continue
|
||||
|
||||
try:
|
||||
with open(f"/proc/{name}/status") as file:
|
||||
for line in file:
|
||||
if line.startswith("VmRSS:"):
|
||||
total_kb += int(line.split()[1])
|
||||
break
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
print(total_kb / 1024)
|
||||
"""
|
||||
|
||||
|
||||
def command(args, check=True):
|
||||
return subprocess.run(
|
||||
args,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def remove_test_worker():
|
||||
command(
|
||||
["docker", "rm", "-f", "benchmark-worker"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def start_worker(concurrency, prefetch):
|
||||
remove_test_worker()
|
||||
|
||||
result = command([
|
||||
"docker", "compose", "run",
|
||||
"-d",
|
||||
"--name", "benchmark-worker",
|
||||
"--no-deps",
|
||||
"-e", f"WORKER_CONCURRENCY={concurrency}",
|
||||
"-e", f"WORKER_PREFETCH={prefetch}",
|
||||
"worker",
|
||||
])
|
||||
|
||||
expected = (
|
||||
f"[Worker] CONCURRENCY={concurrency}, "
|
||||
f"PREFETCH={prefetch}"
|
||||
)
|
||||
|
||||
for _ in range(30):
|
||||
logs = command(
|
||||
["docker", "logs", "benchmark-worker"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
if expected in logs.stdout + logs.stderr:
|
||||
print(expected)
|
||||
return
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
raise RuntimeError(
|
||||
f"Worker {concurrency}/{prefetch} не запустился.\n"
|
||||
+ command(
|
||||
["docker", "logs", "benchmark-worker"],
|
||||
check=False,
|
||||
).stderr
|
||||
)
|
||||
|
||||
|
||||
def measure_rss():
|
||||
result = command(
|
||||
[
|
||||
"docker", "exec",
|
||||
"benchmark-worker",
|
||||
"python", "-c", RSS_CODE,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
|
||||
try:
|
||||
return float(result.stdout.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def extract_number(text, label):
|
||||
match = re.search(
|
||||
rf"^{re.escape(label)}:\s*([0-9.]+)",
|
||||
text,
|
||||
re.MULTILINE,
|
||||
)
|
||||
return float(match.group(1)) if match else None
|
||||
|
||||
|
||||
def run_benchmark(concurrency, prefetch):
|
||||
config = f"{concurrency}_{prefetch}"
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(f"ЭКСПЕРИМЕНТ: {concurrency}/{prefetch}")
|
||||
print(f"{'=' * 55}")
|
||||
|
||||
start_worker(concurrency, prefetch)
|
||||
|
||||
log_path = RESULTS_DIR / f"load_{config}.log"
|
||||
rss_samples = []
|
||||
|
||||
load_command = [
|
||||
"docker", "compose", "exec",
|
||||
"-T",
|
||||
"-w", "/tmp",
|
||||
"-e", "PYTHONPATH=/app",
|
||||
"app1",
|
||||
"python", "-m", "app.load_test",
|
||||
"--count", str(COUNT_PER_SIZE),
|
||||
"--timeout", "300",
|
||||
]
|
||||
|
||||
print("Запускаем 60 фотографий...")
|
||||
start_time = time.monotonic()
|
||||
|
||||
with log_path.open("w") as log_file:
|
||||
process = subprocess.Popen(
|
||||
load_command,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
while process.poll() is None:
|
||||
elapsed = time.monotonic() - start_time
|
||||
rss = measure_rss()
|
||||
|
||||
if rss is not None:
|
||||
rss_samples.append((elapsed, rss))
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
return_code = process.wait()
|
||||
|
||||
output = log_path.read_text()
|
||||
|
||||
if return_code != 0:
|
||||
print(output)
|
||||
raise RuntimeError(
|
||||
f"Нагрузочный тест {concurrency}/{prefetch} завершился с ошибкой"
|
||||
)
|
||||
|
||||
print(output)
|
||||
|
||||
if not rss_samples:
|
||||
raise RuntimeError("Не удалось измерить RSS")
|
||||
|
||||
rss_values = [value for _, value in rss_samples]
|
||||
peak_rss = max(rss_values)
|
||||
mean_rss = sum(rss_values) / len(rss_values)
|
||||
|
||||
samples_path = RESULTS_DIR / f"rss_{config}.csv"
|
||||
|
||||
with samples_path.open("w", newline="") as file:
|
||||
writer = csv.writer(file)
|
||||
writer.writerow(["elapsed_s", "rss_mb"])
|
||||
writer.writerows(rss_samples)
|
||||
|
||||
csv_match = re.search(
|
||||
r"CSV:\s*(benchmark_[a-zA-Z0-9]+\.csv)",
|
||||
output,
|
||||
)
|
||||
|
||||
if csv_match:
|
||||
filename = csv_match.group(1)
|
||||
|
||||
copy_result = command(
|
||||
[
|
||||
"docker", "compose", "cp",
|
||||
f"app1:/tmp/{filename}",
|
||||
str(RESULTS_DIR / filename),
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
if copy_result.returncode != 0:
|
||||
print("Не удалось скопировать CSV:", copy_result.stderr)
|
||||
|
||||
result = {
|
||||
"config": f"{concurrency}/{prefetch}",
|
||||
"total": int(extract_number(output, "Total jobs") or 0),
|
||||
"ready": int(extract_number(output, "Ready") or 0),
|
||||
"failed": int(extract_number(output, "Failed") or 0),
|
||||
"duration_s": extract_number(output, "Duration"),
|
||||
"throughput": extract_number(output, "Throughput"),
|
||||
"peak_processing": extract_number(
|
||||
output, "Peak observed processing"
|
||||
),
|
||||
"peak_rss_mb": round(peak_rss, 2),
|
||||
"mean_rss_mb": round(mean_rss, 2),
|
||||
}
|
||||
|
||||
print(f"\n[RSS] Пиковое значение: {peak_rss:.2f} MB")
|
||||
print(f"[RSS] Среднее значение: {mean_rss:.2f} MB")
|
||||
print(f"[RSS] Количество замеров: {len(rss_samples)}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def save_summary(results):
|
||||
output = RESULTS_DIR / "summary.csv"
|
||||
|
||||
with output.open("w", newline="") as file:
|
||||
writer = csv.DictWriter(
|
||||
file,
|
||||
fieldnames=[
|
||||
"config",
|
||||
"total",
|
||||
"ready",
|
||||
"failed",
|
||||
"duration_s",
|
||||
"throughput",
|
||||
"peak_processing",
|
||||
"peak_rss_mb",
|
||||
"mean_rss_mb",
|
||||
],
|
||||
)
|
||||
|
||||
writer.writeheader()
|
||||
writer.writerows(results)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("ИТОГОВАЯ ТАБЛИЦА")
|
||||
print("=" * 70)
|
||||
|
||||
for item in results:
|
||||
print(
|
||||
f"{item['config']:5} | "
|
||||
f"Ready: {item['ready']:3} | "
|
||||
f"Time: {item['duration_s']} s | "
|
||||
f"Throughput: {item['throughput']} img/s | "
|
||||
f"Peak RSS: {item['peak_rss_mb']} MB"
|
||||
)
|
||||
|
||||
print(f"\nРезультаты сохранены в {RESULTS_DIR}/")
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
|
||||
try:
|
||||
print("Останавливаем обычный Worker...")
|
||||
command(["docker", "compose", "stop", "worker"])
|
||||
|
||||
for concurrency, prefetch in CONFIGS:
|
||||
result = run_benchmark(concurrency, prefetch)
|
||||
results.append(result)
|
||||
|
||||
finally:
|
||||
print("\nВосстанавливаем обычный Worker...")
|
||||
remove_test_worker()
|
||||
|
||||
restore = command(
|
||||
["docker", "compose", "up", "-d", "worker"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
if restore.returncode != 0:
|
||||
print("Ошибка восстановления Worker:")
|
||||
print(restore.stderr)
|
||||
|
||||
if results:
|
||||
save_summary(results)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,197 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
container_name: mtusi_messenger_db
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
command: postgres -c synchronous_commit=off
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: mtusi_messenger_redis
|
||||
command: redis-server --appendonly yes
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
app1:
|
||||
build: .
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./app:/app/app
|
||||
- ./alembic:/app/alembic
|
||||
- ./alembic.ini:/app/alembic.ini
|
||||
|
||||
app2:
|
||||
build: .
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_REGION: ${S3_REGION}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./app:/app/app
|
||||
- ./alembic:/app/alembic
|
||||
- ./alembic.ini:/app/alembic.ini
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.12-management-alpine
|
||||
container_name: mtusi_messenger_rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: guest
|
||||
RABBITMQ_DEFAULT_PASS: guest
|
||||
ports:
|
||||
- "127.0.0.1:5672:5672"
|
||||
- "127.0.0.1:15672:15672"
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: [ "CMD", "rabbitmq-diagnostics", "-q", "ping" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
worker:
|
||||
build: .
|
||||
mem_limit: 512m
|
||||
command: python -m app.worker
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./app:/app/app
|
||||
- ./scripts:/app/scripts
|
||||
|
||||
outbox:
|
||||
build: .
|
||||
command: python -m app.outbox_dispatcher
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./app:/app/app
|
||||
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
container_name: mtusi_messenger_nginx
|
||||
ports:
|
||||
- "127.0.0.1:8080:80"
|
||||
volumes:
|
||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
- app1
|
||||
- app2
|
||||
|
||||
minio:
|
||||
image: pgsty/silo:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
- "127.0.0.1:9000:9000"
|
||||
- "127.0.0.1:9001:9001"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
minio-init:
|
||||
image: pgsty/mc:latest
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
/usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin;
|
||||
/usr/bin/mc mb --ignore-existing myminio/${S3_BUCKET};
|
||||
/usr/bin/mc anonymous set none myminio/${S3_BUCKET};
|
||||
/usr/bin/mc admin user add myminio api-user api-user-secret-2026 || true;
|
||||
/usr/bin/mc admin policy attach myminio readwrite --user api-user || true;
|
||||
exit 0;
|
||||
"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
rabbitmq_data:
|
||||
@@ -0,0 +1,27 @@
|
||||
events { worker_connections 1024; }
|
||||
|
||||
http {
|
||||
upstream api_backend {
|
||||
least_conn;
|
||||
server app1:8080 max_fails=2 fail_timeout=5s;
|
||||
server app2:8080 max_fails=2 fail_timeout=5s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
|
||||
client_max_body_size 10M;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 5s;
|
||||
proxy_read_timeout 5s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,19 @@
|
||||
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
|
||||
openapi-spec-validator==0.7.2
|
||||
pytest==8.3.4
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.52.4
|
||||
pydantic==2.13.5
|
||||
asyncpg==0.31.0
|
||||
sqlalchemy[asyncio]==2.1.0
|
||||
redis==6.2.0
|
||||
passlib==1.7.4
|
||||
bcrypt==4.1.2
|
||||
alembic==1.14.0
|
||||
psycopg2-binary
|
||||
filetype==1.2.0
|
||||
Pillow==10.4.0
|
||||
minio==7.2.9
|
||||
python-multipart==0.0.9
|
||||
aio-pika==9.4.0
|
||||
Pillow==10.4.0
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from pydantic import BaseModel, Field, field_validator, ConfigDict
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def unique_members(cls, v):
|
||||
if len(set(v)) != len(v):
|
||||
raise ValueError("member_ids должны быть уникальными")
|
||||
return v
|
||||
@@ -0,0 +1,6 @@
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
@@ -0,0 +1,7 @@
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
REPLICA="${1:-app1}"
|
||||
|
||||
echo "Останавливаю $REPLICA"
|
||||
date +"%H:%M:%S.%3N"
|
||||
./scripts/stop-one.sh "$REPLICA"
|
||||
|
||||
echo ""
|
||||
echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id"
|
||||
for i in {1..20}; do
|
||||
TS=$(date +"%H:%M:%S.%3N")
|
||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live)
|
||||
INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r')
|
||||
echo "$TS code=$CODE $INST"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Поднимаю $REPLICA обратно"
|
||||
docker compose start "$REPLICA"
|
||||
@@ -0,0 +1,76 @@
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
COUNT = 50
|
||||
|
||||
|
||||
def make_png(width, height):
|
||||
img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG", optimize=False)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def register_and_login(username):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "Test", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def create_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "Cost Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def upload(token, chat_id, content, filename):
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
|
||||
headers=headers, data=content, timeout=60,
|
||||
)
|
||||
return resp.status_code
|
||||
|
||||
|
||||
def main():
|
||||
print(f"Загружаем {COUNT} файлов (по ~1 МБ)...")
|
||||
token = register_and_login("cost_tester")
|
||||
chat_id = create_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
|
||||
content = make_png(200, 200)
|
||||
print(f"Размер файла: {len(content) / 1024:.1f} КБ")
|
||||
|
||||
success = 0
|
||||
errors = 0
|
||||
start = time.monotonic()
|
||||
|
||||
for i in range(COUNT):
|
||||
status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png")
|
||||
if status == 201:
|
||||
success += 1
|
||||
else:
|
||||
errors += 1
|
||||
if (i + 1) % 10 == 0:
|
||||
print(f" Загружено: {i + 1}/{COUNT}")
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
print(f"\n=== Результаты ===")
|
||||
print(f"Успешно: {success}")
|
||||
print(f"Ошибок: {errors}")
|
||||
print(f"Время: {elapsed:.2f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import os
|
||||
|
||||
replicas = [1, 2, 4, 6]
|
||||
rps = [250, 255, 260, 257]
|
||||
|
||||
plt.figure(figsize=(8, 5))
|
||||
plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8)
|
||||
|
||||
plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14)
|
||||
plt.xlabel('Количество API-реплик', fontsize=12)
|
||||
plt.ylabel('RPS (запросов в секунду)', fontsize=12)
|
||||
|
||||
plt.grid(True, linestyle='--', alpha=0.7)
|
||||
plt.xticks(replicas)
|
||||
|
||||
plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252),
|
||||
arrowprops=dict(facecolor='red', shrink=0.05),
|
||||
fontsize=10, color='red')
|
||||
|
||||
os.makedirs('evidence', exist_ok=True)
|
||||
plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight')
|
||||
print("График сохранен в evidence/scaling.png")
|
||||
@@ -0,0 +1,90 @@
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import requests
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from PIL import Image
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
CONCURRENCY = 10
|
||||
|
||||
|
||||
def make_png(width, height):
|
||||
img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format='PNG', optimize=False)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def register_and_login(username):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "Test", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def create_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "RAM Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def upload(token, chat_id, content, filename):
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
|
||||
start = time.monotonic()
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
|
||||
headers=headers, data=content, timeout=60,
|
||||
)
|
||||
elapsed = time.monotonic() - start
|
||||
return resp.status_code, elapsed, len(content)
|
||||
|
||||
|
||||
def main():
|
||||
print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each")
|
||||
token = register_and_login("ram_tester")
|
||||
chat_id = create_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
|
||||
print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...")
|
||||
contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)]
|
||||
sizes = [len(c) / (1024 * 1024) for c in contents]
|
||||
print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ")
|
||||
print(f"Суммарно: {sum(sizes):.2f} МБ")
|
||||
|
||||
print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...")
|
||||
start = time.monotonic()
|
||||
results, errors = [], []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
||||
futures = [
|
||||
pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png")
|
||||
for i in range(CONCURRENCY)
|
||||
]
|
||||
for f in futures:
|
||||
try:
|
||||
results.append(f.result())
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
|
||||
total = time.monotonic() - start
|
||||
print(f"\n=== Результаты ===")
|
||||
print(f"Всего: {len(results) + len(errors)}")
|
||||
print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}")
|
||||
print(f"Ошибок: {len(errors)}")
|
||||
if results:
|
||||
lat = [r[1] for r in results]
|
||||
print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s")
|
||||
print(f"Общее время: {total:.2f}s")
|
||||
if errors:
|
||||
print(f"Ошибки: {errors[:5]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,146 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from minio import Minio
|
||||
from sqlalchemy import select, cast, Text
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.user import User
|
||||
from app.models.message import Message
|
||||
from app.database import async_session_maker
|
||||
from app.models.attachment import Attachment
|
||||
from app.config import (
|
||||
S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
|
||||
)
|
||||
|
||||
GRACE_PERIOD = timedelta(seconds=1)
|
||||
|
||||
|
||||
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
|
||||
minio_client = Minio(
|
||||
endpoint,
|
||||
access_key=S3_ACCESS_KEY_ID,
|
||||
secret_key=S3_SECRET_ACCESS_KEY,
|
||||
secure=False,
|
||||
)
|
||||
BUCKET = S3_BUCKET
|
||||
|
||||
|
||||
async def find_orphan_s3_objects(dry_run: bool) -> int:
|
||||
print("Поиск orphan объектов в S3, файлы без записи в БД")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(Attachment.object_key))
|
||||
db_keys = {row[0] for row in result.all()}
|
||||
|
||||
print(f"Файлов в БД: {len(db_keys)}")
|
||||
|
||||
objects = list(minio_client.list_objects(BUCKET, recursive=True))
|
||||
print(f"Объектов в MinIO: {len(objects)}")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
orphans = []
|
||||
|
||||
for obj in objects:
|
||||
if obj.object_name not in db_keys:
|
||||
age = now - obj.last_modified if obj.last_modified else timedelta(0)
|
||||
if age > GRACE_PERIOD:
|
||||
orphans.append(obj.object_name)
|
||||
print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)")
|
||||
else:
|
||||
print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)")
|
||||
|
||||
if not orphans:
|
||||
print("Orphan-объектов не найдено.")
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.")
|
||||
else:
|
||||
for key in orphans:
|
||||
minio_client.remove_object(BUCKET, key)
|
||||
print(f" [DELETED] {key}")
|
||||
print(f"Удалено {len(orphans)} orphan-объектов.")
|
||||
|
||||
return len(orphans)
|
||||
|
||||
|
||||
async def find_missing_s3_objects(dry_run: bool) -> int:
|
||||
print("\nПоиск missing объектов, записи в БД без файла в MinIO")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(Attachment.id, Attachment.object_key))
|
||||
attachments = result.all()
|
||||
|
||||
print(f"Записей в БД: {len(attachments)}")
|
||||
|
||||
missing = []
|
||||
used_missing = []
|
||||
|
||||
for att_id, key in attachments:
|
||||
try:
|
||||
minio_client.stat_object(BUCKET, key)
|
||||
except Exception:
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result_msg = await session.execute(
|
||||
select(Message).where(
|
||||
cast(Message.attachment_ids, Text).contains(f'"{att_id}"')
|
||||
)
|
||||
)
|
||||
is_used = result_msg.scalar_one_or_none() is not None
|
||||
|
||||
if is_used:
|
||||
used_missing.append((att_id, key))
|
||||
print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем")
|
||||
else:
|
||||
missing.append((att_id, key))
|
||||
print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить")
|
||||
|
||||
total = len(missing) + len(used_missing)
|
||||
if total == 0:
|
||||
print("Missing-записей не найдено.")
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.")
|
||||
else:
|
||||
if missing:
|
||||
async with async_session_maker() as session:
|
||||
for att_id, _ in missing:
|
||||
att = await session.get(Attachment, att_id)
|
||||
if att:
|
||||
await session.delete(att)
|
||||
await session.commit()
|
||||
print(f"Удалено {len(missing)} неиспользуемых missing из БД.")
|
||||
if used_missing:
|
||||
print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).")
|
||||
|
||||
return len(missing)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять")
|
||||
parser.add_argument("--execute", action="store_true", help="Удалить найденное")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dry_run and not args.execute:
|
||||
print("Укажите --dry-run или --execute")
|
||||
return
|
||||
|
||||
dry_run = args.dry_run and not args.execute
|
||||
|
||||
orphans = await find_orphan_s3_objects(dry_run)
|
||||
missing = await find_missing_s3_objects(dry_run)
|
||||
|
||||
print("\nИтог")
|
||||
print(f"Orphan S3 без БД: {orphans}")
|
||||
print(f"Missing БД без S3 (удалено): {missing}")
|
||||
if dry_run:
|
||||
print("Режим: DRY-RUN (ничего не удалено)")
|
||||
else:
|
||||
print("Режим: EXECUTE (изменения применены)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
|
||||
echo "Нагрузка: 30 запросов с интервалом 0.5 сек"
|
||||
for i in {1..30}; do
|
||||
TS=$(date +"%H:%M:%S")
|
||||
|
||||
RESP=$(curl -s -i http://localhost:8080/health/live)
|
||||
CODE=$(echo "$RESP" | head -1 | awk '{print $2}')
|
||||
INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}')
|
||||
|
||||
echo "$TS code=$CODE instance=${INST:-<нет>}"
|
||||
sleep 0.5
|
||||
done
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
|
||||
set -e
|
||||
|
||||
echo "Rolling update: app1"
|
||||
docker compose stop app1
|
||||
docker compose up -d --build app1
|
||||
echo "Жду, пока app1 станет healthy..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "Rolling update: app2"
|
||||
docker compose stop app2
|
||||
docker compose up -d --build app2
|
||||
echo "Жду, пока app2 станет healthy..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "Проверка: обе реплики живы"
|
||||
curl -i http://localhost:8080/health/live
|
||||
curl -i http://localhost:8080/health/live
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "=== GRACE PERIOD TEST — $(date) ==="
|
||||
echo ""
|
||||
|
||||
curl -s -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null
|
||||
|
||||
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "grace_tester:secret1234567" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
|
||||
echo "Токен получен: ${TOKEN:0:20}..."
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Attachment ID: $ATT_ID"
|
||||
echo "Object key: $OBJECT_KEY"
|
||||
|
||||
docker compose exec -T db psql -U messenger -d messenger_db -c \
|
||||
"DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null
|
||||
echo "Запись удалена из БД — ORPHAN"
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
|
||||
echo ""
|
||||
|
||||
echo "=== Пауза 11 секунд ==="
|
||||
sleep 11
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
|
||||
echo ""
|
||||
|
||||
echo "=== EXECUTE (удаляем созревший orphan) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов"
|
||||
echo ""
|
||||
|
||||
echo "=== Проверка: orphan удалён из MinIO ==="
|
||||
STAT=$(docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1)
|
||||
echo "Orphan в MinIO: $STAT"
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется"
|
||||
echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»"
|
||||
echo "3. Execute → [DELETED] — удалён из MinIO"
|
||||
echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются"
|
||||
|
||||
} 2>&1 | tee evidence/track2-grace-period.txt
|
||||
|
||||
echo ""
|
||||
echo "Лог сохранён в evidence/track2-grace-period.txt"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "================================================================"
|
||||
echo " RECONCILER TEST — $(date)"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
TEST_USER="reconcile_tester_$(date +%s)"
|
||||
TEST_PASS="secret1234567"
|
||||
|
||||
echo "=== 1. Создание пользователя ($TEST_USER) ==="
|
||||
REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}")
|
||||
REG_BODY=$(echo "$REG_RESP" | sed '$d')
|
||||
REG_CODE=$(echo "$REG_RESP" | tail -n 1)
|
||||
echo "HTTP $REG_CODE"
|
||||
echo "Body: $REG_BODY"
|
||||
|
||||
if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then
|
||||
echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "=== 2. Логин ==="
|
||||
LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "$TEST_USER:$TEST_PASS")
|
||||
LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d')
|
||||
LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1)
|
||||
echo "HTTP $LOGIN_CODE"
|
||||
|
||||
if [ "$LOGIN_CODE" != "201" ]; then
|
||||
echo "FAIL: логин не удался (HTTP $LOGIN_CODE)"
|
||||
echo "Body: $LOGIN_BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "")
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "FAIL: токен не получен из ответа"
|
||||
echo "Body: $LOGIN_BODY"
|
||||
exit 1
|
||||
fi
|
||||
echo "Токен получен: ${TOKEN:0:20}..."
|
||||
echo ""
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "User ID: $USER_ID"
|
||||
echo ""
|
||||
|
||||
echo "=== 3. Создание чата ==="
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
echo "=== 4. Создаём ORPHAN ==="
|
||||
RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n')
|
||||
echo "Orphan attachment_id: $ORPHAN_ATT_ID"
|
||||
echo "Orphan object_key: $ORPHAN_KEY"
|
||||
|
||||
docker compose exec -T db psql -U messenger -d messenger_db -c \
|
||||
"DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null
|
||||
echo "Запись удалена из БД — ORPHAN"
|
||||
echo ""
|
||||
|
||||
echo "=== 5. Создаём MISSING ==="
|
||||
RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
|
||||
echo "Missing attachment_id: $MISSING_ATT_ID"
|
||||
echo "Missing object_key: $MISSING_KEY"
|
||||
|
||||
docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY"
|
||||
echo "Файл удалён из MinIO — MISSING"
|
||||
echo ""
|
||||
|
||||
echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ==="
|
||||
sleep 11
|
||||
echo ""
|
||||
|
||||
echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --dry-run
|
||||
echo ""
|
||||
|
||||
echo "=== 7. EXECUTE ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --execute
|
||||
echo ""
|
||||
|
||||
echo "=== 8. Проверка: orphan удалён из MinIO ==="
|
||||
ORPHAN_STAT=$(docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1)
|
||||
echo "Orphan в MinIO: $ORPHAN_STAT"
|
||||
echo ""
|
||||
|
||||
echo "=== 9. Проверка: missing удалён из БД ==="
|
||||
MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
|
||||
echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT"
|
||||
echo ""
|
||||
|
||||
echo "=== 10. Идемпотентность: повторный DRY-RUN ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --dry-run
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
ORPHAN_GONE=false
|
||||
if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then
|
||||
ORPHAN_GONE=true
|
||||
fi
|
||||
|
||||
if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then
|
||||
echo "PASS:"
|
||||
echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен"
|
||||
echo " - ORPHAN удалён из MinIO"
|
||||
echo " - MISSING удалён из БД"
|
||||
echo " - Повторный dry-run: 0/0 (идемпотентность)"
|
||||
else
|
||||
echo "FAIL:"
|
||||
echo " - Orphan в MinIO: $ORPHAN_STAT"
|
||||
echo " - Missing count в БД: $MISSING_COUNT"
|
||||
fi
|
||||
|
||||
} 2>&1 | tee evidence/reconciler-full.txt
|
||||
|
||||
echo ""
|
||||
echo "Лог сохранён в evidence/reconciler-full.txt"
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "=== USED ATTACHMENT TEST — $(date) ==="
|
||||
echo ""
|
||||
|
||||
curl -s -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null
|
||||
|
||||
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "used_tester:secret1234567" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Attachment ID: $ATT_ID"
|
||||
echo "Object key: $KEY"
|
||||
|
||||
# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ
|
||||
IDEM_KEY="used-test-$(date +%s)"
|
||||
curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Idempotency-Key: $IDEM_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null
|
||||
echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ"
|
||||
echo ""
|
||||
|
||||
docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY"
|
||||
echo "Файл удалён из MinIO — missing (используется)"
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей"
|
||||
echo ""
|
||||
|
||||
echo "=== EXECUTE (используемый не удаляется) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено"
|
||||
echo ""
|
||||
|
||||
COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Записей с id=$ATT_ID в БД: $COUNT"
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
if [ "$COUNT" = "1" ]; then
|
||||
echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])"
|
||||
else
|
||||
echo "FAIL: используемый missing удалён (count=$COUNT)"
|
||||
fi
|
||||
|
||||
} 2>&1 | tee evidence/track2-used-attachment.txt
|
||||
@@ -0,0 +1,61 @@
|
||||
import time
|
||||
import requests
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
|
||||
|
||||
def get_token(username="stream_tester"):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "T", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def get_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "Stream Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print(" STREAM LIMIT TEST")
|
||||
print("=" * 60)
|
||||
token = get_token()
|
||||
chat_id = get_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===")
|
||||
|
||||
def gen():
|
||||
for _ in range(100):
|
||||
yield b"0" * (1024 * 1024)
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png",
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"},
|
||||
data=gen(), timeout=60,
|
||||
)
|
||||
status = resp.status_code
|
||||
except Exception as e:
|
||||
print(f"Ошибка: {e}")
|
||||
status = None
|
||||
elapsed = time.monotonic() - start
|
||||
print(f"Status: {status} (ожидаем 413)")
|
||||
print(f"Время: {elapsed:.2f}s")
|
||||
print("\n" + "=" * 60)
|
||||
if status == 413:
|
||||
print("PASS: потоковый лимит работает, 413")
|
||||
else:
|
||||
print(f"FAIL: ожидался 413, получен {status}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from app.worker import async_session_maker, engine
|
||||
from app.models.attachment import Attachment
|
||||
|
||||
|
||||
async def main():
|
||||
attachment_id = UUID(sys.argv[1])
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
att = await session.get(Attachment, attachment_id)
|
||||
|
||||
if att is None:
|
||||
print("Attachment not found")
|
||||
return
|
||||
|
||||
print("Attachment ID:", att.id)
|
||||
print("Status:", att.status)
|
||||
print("Error code:", att.error_code)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,29 @@
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from app.worker import async_session_maker, engine
|
||||
from app.models.attachment import Attachment
|
||||
|
||||
|
||||
async def main():
|
||||
attachment_id = UUID(sys.argv[1])
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
att = await session.get(Attachment, attachment_id)
|
||||
|
||||
if att is None:
|
||||
print("Attachment not found")
|
||||
return
|
||||
|
||||
print("Attachment ID:", att.id)
|
||||
print("Status:", att.status)
|
||||
print("Error code:", att.error_code)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,154 @@
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_worker_retry():
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "compose", "exec",
|
||||
"-T", "-w", "/app",
|
||||
"worker", "python", "-"
|
||||
],
|
||||
input=Path(__file__).read_text(encoding="utf-8"),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
print(result.stdout, end="")
|
||||
|
||||
if result.stderr:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
|
||||
assert result.returncode == 0, "Проверка Worker не прошла"
|
||||
|
||||
|
||||
async def main():
|
||||
import aio_pika
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.worker import (
|
||||
async_session_maker,
|
||||
engine,
|
||||
RABBITMQ_URL,
|
||||
QUEUE_NAME,
|
||||
)
|
||||
from app.models.attachment import Attachment
|
||||
|
||||
try:
|
||||
async with async_session_maker() as session:
|
||||
example = (
|
||||
await session.execute(
|
||||
select(Attachment).limit(1)
|
||||
)
|
||||
).scalars().first()
|
||||
|
||||
if example is None:
|
||||
print(
|
||||
"FAIL: нет вложений. Сначала запусти make test."
|
||||
)
|
||||
return False
|
||||
|
||||
test_id = uuid.uuid4()
|
||||
|
||||
test_att = Attachment(
|
||||
id=test_id,
|
||||
object_key=f"retry-test-{test_id}.jpg",
|
||||
filename="retry-test.jpg",
|
||||
mime_type="image/jpeg",
|
||||
size_bytes=1,
|
||||
sha256="0" * 64,
|
||||
chat_id=example.chat_id,
|
||||
user_id=example.user_id,
|
||||
status="queued",
|
||||
)
|
||||
|
||||
session.add(test_att)
|
||||
await session.commit()
|
||||
|
||||
print(f"Тестовое вложение: {test_id}", flush=True)
|
||||
|
||||
connection = await aio_pika.connect_robust(
|
||||
RABBITMQ_URL
|
||||
)
|
||||
|
||||
async with connection:
|
||||
channel = await connection.channel(
|
||||
publisher_confirms=True
|
||||
)
|
||||
|
||||
await channel.declare_queue(
|
||||
QUEUE_NAME,
|
||||
durable=True
|
||||
)
|
||||
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=json.dumps({
|
||||
"attachment_id": str(test_id)
|
||||
}).encode(),
|
||||
delivery_mode=(
|
||||
aio_pika.DeliveryMode.PERSISTENT
|
||||
),
|
||||
),
|
||||
routing_key=QUEUE_NAME,
|
||||
)
|
||||
|
||||
success = False
|
||||
|
||||
for _ in range(60):
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
async with async_session_maker() as session:
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
test_id
|
||||
)
|
||||
|
||||
if att and att.status == "failed":
|
||||
print("Финальный статус:", att.status)
|
||||
print(
|
||||
"Безопасный error_code:",
|
||||
att.error_code
|
||||
)
|
||||
|
||||
success = (
|
||||
att.error_code == "s3_download_error"
|
||||
)
|
||||
break
|
||||
|
||||
if success:
|
||||
async with async_session_maker() as session:
|
||||
att = await session.get(
|
||||
Attachment,
|
||||
test_id
|
||||
)
|
||||
|
||||
if att:
|
||||
await session.delete(att)
|
||||
await session.commit()
|
||||
|
||||
print("Тестовая запись удалена.")
|
||||
print("PASS: failed и безопасный error_code")
|
||||
else:
|
||||
print("FAIL: ожидаемый результат не получен.")
|
||||
print(
|
||||
"Тестовая запись оставлена для диагностики."
|
||||
)
|
||||
|
||||
print(f"ID для проверки логов: {test_id}")
|
||||
|
||||
return success
|
||||
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = asyncio.run(main())
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user