Compare commits

1 Commits
88 changed files with 5726 additions and 53 deletions
+10 -16
View File
@@ -1,25 +1,19 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
POSTGRES_USER=messenger
POSTGRES_PASSWORD=messenger_password
POSTGRES_DB=messenger_db
DATABASE_URL=postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db
REDIS_URL=redis://redis:6379/0
SESSION_STORE=redis
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_ACCESS_KEY_ID=api-user
S3_SECRET_ACCESS_KEY=api-user-secret-2026
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
MAX_IMAGE_PIXELS=40000000
+26 -14
View File
@@ -1,18 +1,30 @@
## Лабораторная 5
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
**Автор / группа:** Долда Кирилл, БПИ2301
**Целевая оценка:** 5
**Трек 5:** Трек 1 — «Очередь под перегрузкой».
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
**Что реализовано:** асинхронная обработка изображений через RabbitMQ, Worker, Transactional Outbox, retry, восстановление после сбоев, backpressure и ограничение очереди до 100 активных заданий. Из лабораторной 4 сохранены API, PostgreSQL, Redis, MinIO и система вложений.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
**Запуск из чистого clone:**
```bash
cp .env.example .env
docker compose up -d --build
docker compose exec app1 alembic upgrade head
make check
make test
```
**Результаты проверок:** обработано 120/120 изображений без ошибок. Throughput вырос с 1,53 до 5,14 изображений/сек. Проверены восстановление после сбоя, очистка staging и ограничение приёма заданий. Подробности — [REPORT.md](REPORT.md).
**Ограничения:** увеличение параллелизма повышает потребление RAM; queue lag измеряется приблизительно.
**Источники и ИИ:** FastAPI, SQLAlchemy, RabbitMQ, MinIO, Pillow, Docker;
**Чек-лист:**
- [x] Сохранена функциональность предыдущих лабораторных.
- [x] Проверены сценарии отказов и перегрузки.
- [x] Реализован трек 5.
- [x] Финально выполнить `make check` и `make test`, сохранить вывод.
- [x] Проверить отсутствие секретов в Git.
- [x] Подготовлен `REPORT.md`.
+24
View File
@@ -0,0 +1,24 @@
FROM python:3.13-slim
WORKDIR /app
COPY requirements-dev.txt .
RUN pip install --no-cache-dir -r requirements-dev.txt
RUN useradd --create-home --shell /bin/bash appuser
COPY app/ app/
COPY alembic.ini .
COPY alembic/ alembic/
RUN chown -R appuser:appuser /app/alembic
COPY scripts/ /app/scripts/
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/live')" || exit 1
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10", "--reload"]
+146 -4
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -25,6 +25,148 @@ test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
explain-10k:
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222' ORDER BY created_at, id LIMIT 50 OFFSET 0;"
explain-composite:
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_composite.txt
explain-simple:
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX IF EXISTS ix_messages_chat_created_id;"
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_only ON messages (chat_id); ANALYZE messages;"
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_simple.txt
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX ix_messages_chat_only;"
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_created_id ON messages (chat_id, created_at, id);"
backup-restore:
@echo "Делаем бекап pg_dump"
time docker exec -it mtusi_messenger_db pg_dump -U messenger -d messenger_db > evidence/backup.sql
@echo "Создаем новую базу messenger_restore"
docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS messenger_restore;"
docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "CREATE DATABASE messenger_restore;"
@echo "Копируем бекап в контейнер и восстанавливаем"
docker cp evidence/backup.sql mtusi_messenger_db:/tmp/
time docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -f /tmp/backup.sql > /dev/null
@echo "В основной базе messenger_db:"
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -t -c "SELECT count(*) FROM messages;"
@echo "В восстановленной базе messenger_restore:"
docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -t -c "SELECT count(*) FROM messages;"
test-race:
@echo "Запускаем два одновременных запроса на регистрацию"
@RACE_USER="race_$$(date +%s)"; \
curl -s -o /dev/null -w "Запрос 1: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \
curl -s -o /dev/null -w "Запрос 2: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \
wait
test-redis:
@echo "Ставим Redis на паузу"
docker pause mtusi_messenger_redis
@echo "Запрос"
time curl -s -w "\nHTTP Статус: %{http_code}\n" http://localhost:8080/api/v1/users/me -H "Authorization: Bearer test_token"
@echo "Снимаем паузу"
docker unpause mtusi_messenger_redis
test-health-db:
@echo "Останавливаем БД"
docker compose stop db
@echo "Проверяем health ready"
curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready
docker compose start db
test-health-redis:
@echo "Останавливаем"
docker compose stop redis
@echo "Проверяем health ожидая 503 "
curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready
docker compose start redis
test-dependencies:
$(PYTHON) -m pytest tests/test_resilience.py -v
failover:
./scripts/failover-test.sh app1
failover-app2:
./scripts/failover-test.sh app2
rolling-load:
./scripts/rolling-load.sh
rolling-update:
./scripts/rolling-update.sh
test-lab3:
$(PYTHON) -m pytest tests/test_lab3.py -v
test-attachments:
$(PYTHON) -m pytest tests/test_attachments.py -v
reconcile-dry:
docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run
reconcile-execute:
docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute
reconcile-test:
./scripts/run-reconciler-test.sh
ram-experiment:
@echo "В другом терминале для определения нагрузки: docker stats --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}'"
mkdir -p evidence
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" > evidence/ram-experiment.txt
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/ram-experiment.py 2>&1 | tee -a evidence/ram-experiment.txt
docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/ram-experiment.txt
stream-limit:
mkdir -p evidence
@echo "=== ДО ===" > evidence/stream-limit.txt
@docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt
@echo "" >> evidence/stream-limit.txt
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/test-stream-limit.py 2>&1 | tee -a evidence/stream-limit.txt
@echo "" >> evidence/stream-limit.txt
@echo "=== ПОСЛЕ ===" >> evidence/stream-limit.txt
@docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt
private-bucket:
mkdir -p evidence
@OBJECT_KEY=$$(docker compose exec -T db psql -U messenger -d messenger_db -t -c "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1;" | tr -d ' \n'); \
echo "Object key: $$OBJECT_KEY"; \
{ \
echo "=== Проверка private bucket ==="; \
echo "Object key: $$OBJECT_KEY"; \
echo ""; \
echo "=== Прямой GET без авторизации (ожидаем 403) ==="; \
curl -i "http://localhost:9000/messenger-attachments/$$OBJECT_KEY"; \
} > evidence/private-bucket.txt 2>&1; \
cat evidence/private-bucket.txt
s3-user-check:
@echo "=== 1. api-user пытается создать пользователя (admin) ==="
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user add myminio hacker hacker123 2>&1' | head -3
@echo ""
@echo "=== 2. api-user пытается изменить политику (admin) ==="
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin policy attach myminio readwrite --user hacker 2>&1' | head -3
@echo ""
@echo "=== 3. api-user пытается удалить пользователя (admin) ==="
@docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user remove myminio minioadmin 2>&1' | head -3
reconcile-cost:
@echo "=== Стоимость обхода reconciler ==="
time docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run 2>&1 | tee evidence/track2-cost-50.txt
grace-period-test:
./scripts/run-grace-period-test.sh
used-attachment-test:
./scripts/run-used-attachment-test.sh
test-retry:
$(PYTHON) -m pytest tests/test_worker_retry.py -v -s
test-queue-upload:
$(PYTHON) -c 'import sys; sys.path.insert(0, "tests"); from client import Client; c = Client(); alice, bob, eve, chat = c.bootstrap(); att, content = c.attachment(alice, chat); print("Attachment ID:", att["id"]); print("Status:", att["status"])'
test-queue-status:
@test -n "$(ATTACHMENT_ID)" || (echo "Укажи ATTACHMENT_ID"; exit 1)
docker compose exec -T worker python - "$(ATTACHMENT_ID)" < tests/check_attachment_status.py
+66 -19
View File
@@ -1,37 +1,84 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- **Автор, группа:** Долда Кирилл, БПИ2301
- **Целевая оценка:** 5, трек 1 «Очередь под перегрузкой»
- **Commit/PR:** TODO после публикации
- **Источники, библиотеки и помощь ИИ:** README лабораторной, FastAPI, SQLAlchemy, asyncpg, Alembic, RabbitMQ (aio-pika), PostgreSQL, Redis, MinIO, Pillow, Docker Compose.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
**Требования:** Docker Desktop, Docker Compose, Git, make, curl. Для локальной разработки — Python 3.13.
**Основные образы:** `python:3.13-slim`, `postgres:16-alpine`, `redis:7-alpine`, `rabbitmq:3.12-management-alpine`, `nginx:alpine`, `pgsty/silo:latest`, `pgsty/mc:latest`.
**Подготовка, сборка, миграции и проверка:**
```bash
cp .env.example .env
docker compose up -d --build
docker compose exec -T app1 alembic upgrade head
make check
make test
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
**Схема:** клиент → Nginx → FastAPI (app1/app2) → PostgreSQL -> MinIO (исходники и варианты) → Outbox dispatcher → RabbitMQ → Worker → subprocess Pillow → MinIO + PostgreSQL. Redis используется для сессий.
**Источник истины:** PostgreSQL хранит состояние задания (`queued`, `processing`, `ready`, `failed`), ошибки и lease. MinIO хранит файлы; RabbitMQ доставляет задания, но не определяет их итоговый статус.
**Транзакции и надёжность:**
- API сохраняет исходник в MinIO, затем создаёт `Attachment` и `OutboxEvent` в одной транзакции БД; возвращает HTTP `202`.
- Диспетчер публикует события в RabbitMQ с publisher confirm. Если публикация прервалась, неопубликованное событие остаётся в outbox.
- Worker подтверждает сообщение (`ack`) после фиксации результата. Повторная доставка уже готового задания безопасна (идемпотентность).
- Для зависших задач используется lease; для временных ошибок — ограниченные повторы (до 5) с backoff и jitter, затем `failed` с безопасным `error_code`.
**Обработка:** отдельный subprocess с таймаутом и ограничением памяти создаёт thumbnail JPEG 256×256, crop JPEG 128×128 и WebP.
**Лимиты:** загрузка до 10 МБ и 40 млн пикселей; `WORKER_CONCURRENCY` и `WORKER_PREFETCH` управляют параллелизмом. API принимает не более **100 активных** (`queued` + `processing`) задач; при переполнении — HTTP `429`.
**Из lab4 перенесено:** FastAPI, Nginx, PostgreSQL, Redis, MinIO, авторизация, ограничения загрузки, миграции, reconciler. **Добавлено в lab5:** RabbitMQ, worker, outbox, варианты изображений, retry, lease, prefetch и нагрузочные эксперименты.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда / сценарий | Наблюдаемый результат | Доказательство |
| --- | --- |--------------------------------------------------| --- |
| Публичный smoke | `make test` | PASS | `evidence/smoke-lab5.txt` |
| Асинхронная обработка | Upload → Outbox → RabbitMQ → Worker | Задания переходят в `ready` | `app/worker.py`, `app/outbox_dispatcher.py` |
| Retry и `failed` | Ошибка обработки, до 5 попыток | Реализовано; отдельный финальный лог не сохранён | `app/worker.py` |
| Варианты и изоляция | Обработка PNG/JPEG | Thumbnail, crop, WebP; subprocess с лимитами | `app/image_processor.py` |
| Outbox и идемпотентность | Crash после COMMIT до ack | `REDELIVERED: True`, `CRASH TEST: PASSED` | Вывод ручного теста |
| Трек 1: три режима | `benchmark_rss.py`, 60 задач на режим | 60/60 `ready` в каждом, 0 `failed` | `benchmark_results/summary.csv`, `rss_*.csv` |
| Трек 1: перегрузка | `backpressure_test.py`, 120 задач | `Recovery: PASSED`, потерь нет | `benchmark_results/backpressure_queue.csv` |
| Лимит приёма | Имитация 100/99 активных задач | HTTP `429` / разрешённая загрузка; `PASSED` | Вывод ручного теста |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Трек 1: очередь под перегрузкой
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
**Гипотеза:** большее число параллельных обработчиков повышает throughput, но увеличивает расход RAM. При перегрузке RabbitMQ должен накопить задания, а затем разобрать очередь без потерь.
**Методика:** `app/load_test.py` создаёт JPEG трёх размеров: 512×512 (~172 КБ), 1024×1024 (~688 КБ), 2048×2048 (~2751 КБ). На каждый режим — 60 задач (20 каждого размера). Время до `ready` и queue lag оценивались опросом БД; RSS — по процессам worker-контейнера.
| Concurrency / Prefetch | Ready / Failed | Время | Throughput | Пик RSS | p95 ready: 512 / 1024 / 2048, с |
| --- | --- | --- | --- | --- | --- |
| 1 / 1 | 60 / 0 | 39.34 с | 1.53 img/s | 214.80 МБ | 36.85 / 38.84 / 36.62 |
| 2 / 2 | 60 / 0 | 21.72 с | 2.76 img/s | 337.21 МБ | 16.67 / 17.15 / 20.43 |
| 4 / 4 | 60 / 0 | 11.67 с | 5.14 img/s | 536.34 МБ | 9.84 / 10.13 / 11.67 |
**Средний наблюдаемый queue lag (512 / 1024 / 2048), с:** 1/1 — 18.98 / 22.74 / 18.76; 2/2 — 7.42 / 9.58 / 11.14; 4/4 — 5.72 / 5.14 / 5.70.
**Вывод:** переход с 1/1 на 4/4 повысил throughput в 3.36 раза и сократил время обработки пакета примерно на 70%, но увеличил пиковый RSS в 2.5 раза. Режим 2/2 — компромисс.
**Перегрузка (120 задач, 1/1):** 120/120 `ready`, 0 ошибок; 79.06 с, 1.52 img/s. Максимум в очереди — **119**, максимум `unacked` — **1**. После обработки `queued=0`, `unacked=0`, `Recovery: PASSED`. Это подтверждает backpressure по prefetch и восстановление после всплеска нагрузки.
| Воздействие | Что наблюдали | Результат |
| --- | --- | --- |
| Перегрузка 120 изображениями | Очередь выросла до 119, затем опустела | 120/120 готовы, потерь нет |
| Истечение lease | Зависшая задача возвращена в обработку | Ручная проверка пройдена |
| 100 активных задач | Новый upload отклонён | HTTP 429, тест `PASSED` |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
**Подтверждено:** три режима worker обработали все задания; при перегрузке очередь восстановилась без потерь; повторная доставка после COMMIT безопасна; ветки лимита 100/99 работают.
+117
View File
@@ -0,0 +1,117 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
# Use forward slashes (/) also on windows to provide an os agnostic path
script_location = alembic
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python>=3.9 or backports.zoneinfo library.
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
# string value is passed to ZoneInfo()
# leave blank for localtime
# timezone =
# max length of characters to apply to the "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; This defaults
# to alembic/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "version_path_separator" below.
# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
# version path separator; As mentioned above, this is the character used to split
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
# Valid values for version_path_separator are:
#
# version_path_separator = :
# version_path_separator = ;
# version_path_separator = space
# version_path_separator = newline
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
# set to 'true' to search source files recursively
# in each "version_locations" directory
# new in Alembic version 1.10
# recursive_version_locations = false
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
# hooks = ruff
# ruff.type = exec
# ruff.executable = %(here)s/.venv/bin/ruff
# ruff.options = --fix REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+1
View File
@@ -0,0 +1 @@
Generic single-database configuration.
+92
View File
@@ -0,0 +1,92 @@
from app.models.base import Base
from app.models.user import User
from app.models.chat import Chat, ChatMember
from app.models.message import Message
from app.config import DATABASE_URL
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
from app.models.attachment import Attachment
from app.models.outbox import OutboxEvent
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
config.set_main_option("sqlalchemy.url", DATABASE_URL.replace("+asyncpg", "+psycopg2"))
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,30 @@
"""create tables
Revision ID: 08fe661df060
Revises: bc35c1b6b748
Create Date: 2026-10-31 11:52:26.342048
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '08fe661df060'
down_revision: Union[str, None] = 'bc35c1b6b748'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
@@ -0,0 +1,35 @@
from alembic import op
import sqlalchemy as sa
revision = "12043b148948"
down_revision = "ffdfa8869ddc"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"outbox_events",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False
),
sa.Column(
"published_at",
sa.DateTime(timezone=True),
nullable=True
),
sa.ForeignKeyConstraint(
["attachment_id"],
["attachments.id"]
),
sa.PrimaryKeyConstraint("id")
)
def downgrade() -> None:
op.drop_table("outbox_events")
@@ -0,0 +1,33 @@
"""add attachment lease
Revision ID: 157ed8ce0e22
Revises: 12043b148948
Create Date: 2026-11-20 13:07:19.055182
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "157ed8ce0e22"
down_revision: Union[str, None] = "12043b148948"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"attachments",
sa.Column(
"lease_expires_at",
sa.DateTime(timezone=True),
nullable=True,
),
)
def downgrade() -> None:
op.drop_column("attachments", "lease_expires_at")
@@ -0,0 +1,28 @@
"""add status to attachments
Revision ID: 1c0bb32fbe88
Revises: 8203fbed8a46
Create Date: 2026-11-17 15:38:59.622410
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '1c0bb32fbe88'
down_revision: Union[str, None] = '8203fbed8a46'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column('attachments', sa.Column('status', sa.String(), server_default='queued', nullable=False))
op.add_column('attachments', sa.Column('error_code', sa.String(), nullable=True))
def downgrade() -> None:
op.drop_column('attachments', 'error_code')
op.drop_column('attachments', 'status')
@@ -0,0 +1,23 @@
from alembic import op
import sqlalchemy as sa
revision = "2a1b3c4d5e6f"
down_revision = "1c0bb32fbe88"
branch_labels = None
depends_on = None
def upgrade():
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_sha256", sa.String(), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_size_bytes", sa.Integer(), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_width", sa.Integer(), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_height", sa.Integer(), nullable=True))
def downgrade():
op.drop_column("attachments", "thumbnail_height")
op.drop_column("attachments", "thumbnail_width")
op.drop_column("attachments", "thumbnail_size_bytes")
op.drop_column("attachments", "thumbnail_sha256")
op.drop_column("attachments", "thumbnail_key")
@@ -0,0 +1,31 @@
"""add timestamps to attachments
Revision ID: 8203fbed8a46
Revises: b0f31212df2c
Create Date: 2026-11-11 15:59:23.045039
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '8203fbed8a46'
down_revision: Union[str, None] = 'b0f31212df2c'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Добавляем колонки created_at и updated_at
op.add_column('attachments', sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False))
op.add_column('attachments', sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False))
# УДАЛЕНО: op.alter_column('attachments', 'user_id', ...) - чтобы не упасть на старых записях
def downgrade() -> None:
op.drop_column('attachments', 'updated_at')
op.drop_column('attachments', 'created_at')
# УДАЛЕНО: op.alter_column('attachments', 'user_id', nullable=True)
@@ -0,0 +1,35 @@
"""add owner and sha256 to attachments
Revision ID: b0f31212df2c
Revises: d78a7aaedac6
Create Date: 2026-11-11 15:39:07.391800
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'b0f31212df2c'
down_revision: Union[str, None] = 'd78a7aaedac6'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# 1. Добавляем user_id с nullable=True, чтобы не упасть на старых данных
op.add_column('attachments', sa.Column('user_id', sa.UUID(), nullable=True))
# 2. Добавляем sha256
op.add_column('attachments', sa.Column('sha256', sa.String(), nullable=True))
# 3. Создаем внешний ключ
op.create_foreign_key(None, 'attachments', 'users', ['user_id'], ['id'], ondelete='CASCADE')
def downgrade() -> None:
op.drop_constraint(None, 'attachments', type_='foreignkey')
op.drop_column('attachments', 'sha256')
op.drop_column('attachments', 'user_id')
@@ -0,0 +1,67 @@
"""create tables
Revision ID: b24373e3b620
Revises:
Create Date: 2026-10-24 06:54:22.471827
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'b24373e3b620'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('chats',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('users',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('username', sa.String(length=32), nullable=False),
sa.Column('display_name', sa.String(length=100), nullable=False),
sa.Column('password_hash', sa.String(length=255), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True)
op.create_table('chat_members',
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('user_id', sa.UUID(), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('chat_id', 'user_id')
)
op.create_table('messages',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('sender_id', sa.UUID(), nullable=False),
sa.Column('text', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('ix_messages_chat_created_id', table_name='messages')
op.drop_table('messages')
op.drop_table('chat_members')
op.drop_index(op.f('ix_users_username'), table_name='users')
op.drop_table('users')
op.drop_table('chats')
# ### end Alembic commands ###
@@ -0,0 +1,43 @@
"""seed explain test 10k
Revision ID: bc35c1b6b748
Revises: f9b85be2b710
Create Date: 2026-10-24 07:04:09.294404
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'bc35c1b6b748'
down_revision: Union[str, None] = 'f9b85be2b710'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ('11111111-1111-1111-1111-111111111111', 'explain_test', 'Test', 'hash', NOW())
ON CONFLICT DO NOTHING;
""")
op.execute("""
INSERT INTO chats (id, title, created_at)
VALUES ('22222222-2222-2222-2222-222222222222', 'Explain Chat', NOW())
ON CONFLICT DO NOTHING;
""")
op.execute("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '22222222-2222-2222-2222-222222222222', '11111111-1111-1111-1111-111111111111', 'Message ' || i, NOW() - (i || ' minutes')::interval
FROM generate_series(1, 10000) AS i;
""")
def downgrade() -> None:
op.execute("DELETE FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222';")
op.execute("DELETE FROM chats WHERE id = '22222222-2222-2222-2222-222222222222';")
op.execute("DELETE FROM users WHERE id = '11111111-1111-1111-1111-111111111111';")
@@ -0,0 +1,30 @@
"""add attachment_ids to messages
Revision ID: d78a7aaedac6
Revises: ea4f73899364
Create Date: 2026-11-11 15:16:50.773392
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'd78a7aaedac6'
down_revision: Union[str, None] = 'ea4f73899364'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column('messages', sa.Column('attachment_ids', sa.JSON(), nullable=True))
op.execute("UPDATE messages SET attachment_ids = '[]'::json WHERE attachment_ids IS NULL")
op.alter_column('messages', 'attachment_ids', nullable=False, server_default='[]')
def downgrade() -> None:
op.drop_column('messages', 'attachment_ids')
@@ -0,0 +1,20 @@
from alembic import op
import sqlalchemy as sa
revision = "e090e7a3e071"
down_revision = "2a1b3c4d5e6f"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"attachments",
sa.Column("crop_key", sa.String(), nullable=True)
)
def downgrade() -> None:
op.drop_column("attachments", "crop_key")
@@ -0,0 +1,40 @@
"""add attachments table
Revision ID: ea4f73899364
Revises: 08fe661df060
Create Date: 2026-11-11 14:37:24.096041
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'ea4f73899364'
down_revision: Union[str, None] = '08fe661df060'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('attachments',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('object_key', sa.String(), nullable=False),
sa.Column('filename', sa.String(), nullable=False),
sa.Column('mime_type', sa.String(), nullable=False),
sa.Column('size_bytes', sa.Integer(), nullable=False),
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('object_key')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('attachments')
# ### end Alembic commands ###
@@ -0,0 +1,69 @@
"""seed load test data
Revision ID: f9b85be2b710
Revises: b24373e3b620
Create Date: ...
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'f9b85be2b710'
down_revision: Union[str, None] = 'b24373e3b620'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ('55555555-5555-4555-8555-555555555555', 'load_test', 'Load', 'hash', NOW())
ON CONFLICT DO NOTHING;
""")
op.execute("""
INSERT INTO chats (id, title, created_at) VALUES
('66666666-6666-4666-8666-666666666666', 'Big Chat', NOW()),
('77777777-7777-4777-8777-777777777777', 'Small Chat 1', NOW()),
('88888888-8888-4888-8888-888888888888', 'Small Chat 2', NOW())
ON CONFLICT DO NOTHING;
""")
op.execute("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '66666666-6666-4666-8666-666666666666', '55555555-5555-4555-8555-555555555555', 'Big ' || i, NOW() - (i || ' minutes')::interval
FROM generate_series(1, 80000) AS i;
""")
op.execute("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '77777777-7777-4777-8777-777777777777', '55555555-5555-4555-8555-555555555555', 'Small1 ' || i, NOW() - (i || ' minutes')::interval
FROM generate_series(1, 10000) AS i;
""")
op.execute("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '88888888-8888-4888-8888-888888888888', '55555555-5555-4555-8555-555555555555', 'Small2 ' || i, NOW() - (i || ' minutes')::interval
FROM generate_series(1, 10000) AS i;
""")
def downgrade() -> None:
op.execute("""
DELETE FROM messages WHERE chat_id IN (
'66666666-6666-4666-8666-666666666666',
'77777777-7777-4777-8777-777777777777',
'88888888-8888-4888-8888-888888888888'
);
""")
op.execute("""
DELETE FROM chats WHERE id IN (
'66666666-6666-4666-8666-666666666666',
'77777777-7777-4777-8777-777777777777',
'88888888-8888-4888-8888-888888888888'
);
""")
op.execute("DELETE FROM users WHERE id = '55555555-5555-4555-8555-555555555555';")
@@ -0,0 +1,20 @@
from alembic import op
import sqlalchemy as sa
revision = "ffdfa8869ddc"
down_revision = "e090e7a3e071"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"attachments",
sa.Column("webp_key", sa.String(), nullable=True)
)
def downgrade() -> None:
op.drop_column("attachments", "webp_key")
View File
+28
View File
@@ -0,0 +1,28 @@
import os
DATABASE_URL = os.getenv(
"DATABASE_URL",
"postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db",
)
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
CODE_MAP = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
503: "unavailable",
500: "internal_error",
}
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "minioadmin")
S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadmin")
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
+13
View File
@@ -0,0 +1,13 @@
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
from app.config import DATABASE_URL
engine = create_async_engine(
DATABASE_URL,
pool_size=5,
max_overflow=10,
pool_timeout=30,
pool_recycle=1800,
echo=False,
)
async_session_maker = async_sessionmaker(engine, expire_on_commit=False)
+6
View File
@@ -0,0 +1,6 @@
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import async_session_maker
async def get_session():
async with async_session_maker() as session:
yield session
+37
View File
@@ -0,0 +1,37 @@
from fastapi import HTTPException, Request
from fastapi.responses import JSONResponse
from fastapi.exceptions import RequestValidationError
from app.config import CODE_MAP
def error_response(request: Request, status_code: int, code: str, message: str, extra_headers: dict = None):
request_id = getattr(request.state, "request_id", "unknown")
headers = {"X-Request-Id": request_id}
if extra_headers:
headers.update(extra_headers)
return JSONResponse(
status_code=status_code,
content={
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
},
headers=headers,
)
async def http_exception_handler(request: Request, exc: HTTPException):
code = CODE_MAP.get(exc.status_code, "internal_error")
return error_response(request, exc.status_code, code, str(exc.detail), extra_headers=exc.headers)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Ошибка валидации данных")
async def internal_error_handler(request: Request, exc: Exception):
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
+62
View File
@@ -0,0 +1,62 @@
import sys
from pathlib import Path
from PIL import Image, ImageOps, UnidentifiedImageError
MAX_INPUT_PIXELS = 20_000_000
def main():
input_path = Path(sys.argv[1])
output_dir = Path(sys.argv[2])
try:
with Image.open(input_path) as source:
if source.width * source.height > MAX_INPUT_PIXELS:
return 20
source.load()
img = source.convert("RGB")
crop = ImageOps.fit(
img,
(128, 128),
method=Image.Resampling.LANCZOS,
centering=(0.5, 0.5)
)
crop.save(
output_dir / "crop.jpg",
format="JPEG",
quality=85
)
img.save(
output_dir / "optimized.webp",
format="WEBP",
quality=80,
method=6
)
img.thumbnail((256, 256))
img.save(
output_dir / "thumbnail.jpg",
format="JPEG",
quality=85
)
except (
UnidentifiedImageError,
OSError,
ValueError,
Image.DecompressionBombError,
):
return 21
return 0
if __name__ == "__main__":
sys.exit(main())
+336
View File
@@ -0,0 +1,336 @@
import argparse
import asyncio
import csv
import hashlib
import io
import statistics
import time
import uuid
from pathlib import Path
from PIL import Image
from sqlalchemy import select
from app.config import S3_BUCKET
from app.models.attachment import Attachment
from app.models.outbox import OutboxEvent
from app.outbox_dispatcher import session_maker
from app.worker import minio_client
IMAGE_SIZES = [512, 1024, 2048]
def make_image(size):
image = Image.effect_noise((size, size), 80).convert("RGB")
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=85)
return buffer.getvalue()
def percentile(values, percent):
values = sorted(values)
if not values:
return None
position = (len(values) - 1) * percent / 100
lower = int(position)
upper = min(lower + 1, len(values) - 1)
return values[lower] + (
values[upper] - values[lower]
) * (position - lower)
async def get_source():
async with session_maker() as session:
result = await session.execute(
select(Attachment)
.where(Attachment.status == "ready")
.order_by(Attachment.created_at.desc())
.limit(1)
)
source = result.scalar_one_or_none()
if source is None:
raise RuntimeError(
"Нет готовой фотографии для создания тестовых записей"
)
return source.chat_id, source.user_id
async def create_jobs(count_per_size, run_id):
chat_id, user_id = await get_source()
jobs = []
for size in IMAGE_SIZES:
data = make_image(size)
print(
f"[LOAD] Изображение {size}x{size}: "
f"{len(data) / 1024:.1f} KB",
flush=True,
)
for index in range(count_per_size):
key = f"benchmark/{run_id}/{size}_{index}.jpg"
minio_client.put_object(
S3_BUCKET,
key,
io.BytesIO(data),
len(data),
content_type="image/jpeg",
)
jobs.append({
"size": size,
"key": key,
"data": data,
})
async with session_maker() as session:
async with session.begin():
for job in jobs:
data = job["data"]
attachment = Attachment(
object_key=job["key"],
filename=f"benchmark_{job['size']}.jpg",
mime_type="image/jpeg",
size_bytes=len(data),
chat_id=chat_id,
user_id=user_id,
sha256=hashlib.sha256(data).hexdigest(),
status="queued",
)
session.add(attachment)
await session.flush()
session.add(
OutboxEvent(attachment_id=attachment.id)
)
job["id"] = attachment.id
del job["data"]
return jobs
async def monitor_jobs(jobs, timeout):
ids = [job["id"] for job in jobs]
started = time.perf_counter()
last_report = started
first_processing = {}
ready_times = {}
failed = set()
peak_processing = 0
while True:
now = time.perf_counter()
elapsed = now - started
async with session_maker() as session:
result = await session.execute(
select(Attachment.id, Attachment.status)
.where(Attachment.id.in_(ids))
)
statuses = result.all()
processing_count = 0
for attachment_id, status in statuses:
if status == "processing":
processing_count += 1
if attachment_id not in first_processing:
first_processing[attachment_id] = elapsed
elif status == "ready":
if attachment_id not in ready_times:
ready_times[attachment_id] = elapsed
elif status == "failed":
failed.add(attachment_id)
peak_processing = max(
peak_processing,
processing_count,
)
completed = len(ready_times) + len(failed)
if now - last_report >= 5:
print(
f"[LOAD] Готово: {len(ready_times)}/{len(jobs)}, "
f"failed={len(failed)}, "
f"processing={processing_count}, "
f"elapsed={elapsed:.1f}s",
flush=True,
)
last_report = now
if completed == len(jobs):
break
if elapsed >= timeout:
print("[LOAD] Достигнут таймаут", flush=True)
break
await asyncio.sleep(0.1)
return {
"ready_times": ready_times,
"first_processing": first_processing,
"failed": failed,
"peak_processing": peak_processing,
"elapsed": time.perf_counter() - started,
}
def save_results(jobs, results, run_id):
output = Path(f"benchmark_{run_id}.csv")
with output.open("w", newline="") as file:
writer = csv.writer(file)
writer.writerow([
"attachment_id",
"image_size",
"status",
"queue_lag_estimate_s",
"time_to_ready_s",
])
for job in jobs:
attachment_id = job["id"]
ready = results["ready_times"].get(attachment_id)
lag = results["first_processing"].get(attachment_id)
status = (
"ready" if ready is not None
else "failed" if attachment_id in results["failed"]
else "unfinished"
)
writer.writerow([
str(attachment_id),
job["size"],
status,
round(lag, 3) if lag is not None else "",
round(ready, 3) if ready is not None else "",
])
return output
def print_results(jobs, results):
ready_times = results["ready_times"]
first_processing = results["first_processing"]
total = len(jobs)
completed = len(ready_times)
throughput = (
completed / results["elapsed"]
if results["elapsed"] > 0
else 0
)
print("\n========== BENCHMARK RESULTS ==========")
print(f"Total jobs: {total}")
print(f"Ready: {completed}")
print(f"Failed: {len(results['failed'])}")
print(f"Duration: {results['elapsed']:.2f} s")
print(f"Throughput: {throughput:.2f} images/s")
print(f"Peak observed processing: {results['peak_processing']}")
for size in IMAGE_SIZES:
size_jobs = [
job for job in jobs
if job["size"] == size
]
times = [
ready_times[job["id"]]
for job in size_jobs
if job["id"] in ready_times
]
lags = [
first_processing[job["id"]]
for job in size_jobs
if job["id"] in first_processing
]
print(f"\nImage size: {size}x{size}")
print(f"Completed: {len(times)}/{len(size_jobs)}")
if times:
print(f"Mean ready: {statistics.mean(times):.2f} s")
print(f"P95 ready: {percentile(times, 95):.2f} s")
if lags:
print(f"Mean observed queue lag: {statistics.mean(lags):.2f} s")
print(f"Queue lag samples: {len(lags)}/{len(size_jobs)}")
print("\n=======================================")
async def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--count",
type=int,
default=10,
help="Количество изображений каждого размера",
)
parser.add_argument(
"--timeout",
type=int,
default=300,
)
args = parser.parse_args()
if args.count < 1 or args.timeout < 1:
parser.error("--count и --timeout должны быть положительными")
run_id = uuid.uuid4().hex[:8]
print(f"[LOAD] Run ID: {run_id}")
print("[LOAD] Подготовка изображений...")
jobs = await create_jobs(args.count, run_id)
print(f"[LOAD] Создано заданий: {len(jobs)}")
print("[LOAD] Ожидание обработки...")
results = await monitor_jobs(jobs, args.timeout)
print_results(jobs, results)
output = save_results(jobs, results, run_id)
print(f"\nCSV: {output}")
if len(results["ready_times"]) != len(jobs):
raise SystemExit(1)
if __name__ == "__main__":
asyncio.run(main())
+31
View File
@@ -0,0 +1,31 @@
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from fastapi import HTTPException
import uuid
from app.middleware import RequestIdMiddleware
from app.exceptions import http_exception_handler, validation_exception_handler
from app.routers import health, users, chats, messages, auth, attachments
app = FastAPI(
title="MTUSI Messenger",
version="1.1.0",
)
INSTANCE_ID = str(uuid.uuid4())
@app.middleware("http")
async def add_instance_id_header(request, call_next):
response = await call_next(request)
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
app.add_middleware(RequestIdMiddleware)
app.add_exception_handler(HTTPException, http_exception_handler)
app.add_exception_handler(RequestValidationError, validation_exception_handler)
app.include_router(health.router)
app.include_router(users.router)
app.include_router(chats.router)
app.include_router(messages.router)
app.include_router(auth.router)
app.include_router(attachments.router_upload)
app.include_router(attachments.router_download)
+11
View File
@@ -0,0 +1,11 @@
import uuid as uuid_lib
from starlette.middleware.base import BaseHTTPMiddleware
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
request_id = str(uuid_lib.uuid4())
request.state.request_id = request_id
response = await call_next(request)
response.headers["X-Request-Id"] = request_id
return response
View File
+48
View File
@@ -0,0 +1,48 @@
import uuid
from datetime import datetime
from sqlalchemy import String, Integer, ForeignKey, DateTime, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Attachment(Base):
__tablename__ = "attachments"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
object_key: Mapped[str] = mapped_column(String, unique=True)
filename: Mapped[str] = mapped_column(String)
mime_type: Mapped[str] = mapped_column(String)
size_bytes: Mapped[int] = mapped_column(Integer)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
sha256: Mapped[str] = mapped_column(String, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now()
)
status: Mapped[str] = mapped_column(String, default="queued")
error_code: Mapped[str] = mapped_column(String, nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True),
nullable=True
)
thumbnail_key: Mapped[str] = mapped_column(String, nullable=True)
thumbnail_sha256: Mapped[str] = mapped_column(String, nullable=True)
thumbnail_size_bytes: Mapped[int] = mapped_column(Integer, nullable=True)
thumbnail_width: Mapped[int] = mapped_column(Integer, nullable=True)
thumbnail_height: Mapped[int] = mapped_column(Integer, nullable=True)
crop_key: Mapped[str] = mapped_column(String, nullable=True)
webp_key: Mapped[str] = mapped_column(String, nullable=True)
+4
View File
@@ -0,0 +1,4 @@
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
pass
+35
View File
@@ -0,0 +1,35 @@
import uuid
from datetime import datetime
from sqlalchemy import String, DateTime, ForeignKey, func
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.dialects.postgresql import UUID
from app.models.base import Base
class Chat(Base):
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
title: Mapped[str] = mapped_column(String(100), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
class ChatMember(Base):
__tablename__ = "chat_members"
chat_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("chats.id", ondelete="CASCADE"),
primary_key=True,
)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
primary_key=True,
)
+36
View File
@@ -0,0 +1,36 @@
import uuid
from datetime import datetime
from sqlalchemy import Text, DateTime, ForeignKey, Index, func, JSON
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.dialects.postgresql import UUID
from app.models.base import Base
class Message(Base):
__tablename__ = "messages"
attachment_ids: Mapped[list] = mapped_column(JSON, default=list)
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
chat_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("chats.id", ondelete="CASCADE"),
nullable=False,
)
sender_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
)
text: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
__table_args__ = (
Index("ix_messages_chat_created_id", "chat_id", "created_at", "id"),
)
+33
View File
@@ -0,0 +1,33 @@
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.attachment import Base
class OutboxEvent(Base):
__tablename__ = "outbox_events"
id: Mapped[uuid.UUID] = mapped_column(
primary_key=True,
default=uuid.uuid4
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("attachments.id"),
nullable=False
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
nullable=False
)
published_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True),
nullable=True
)
+24
View File
@@ -0,0 +1,24 @@
import uuid
from datetime import datetime
from sqlalchemy import String, DateTime, func
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.dialects.postgresql import UUID
from app.models.base import Base
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
username: Mapped[str] = mapped_column(
String(32), unique=True, nullable=False, index=True
)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
+137
View File
@@ -0,0 +1,137 @@
import asyncio
import json
import os
from datetime import datetime, timezone
import aio_pika
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.config import DATABASE_URL
from app.models.attachment import Attachment
from app.models.chat import Chat, ChatMember
from app.models.outbox import OutboxEvent
from app.models.user import User
RABBITMQ_URL = os.getenv("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/")
QUEUE_NAME = "image_tasks"
POLL_INTERVAL_SECONDS = 1
engine = create_async_engine(DATABASE_URL)
session_maker = async_sessionmaker(engine, expire_on_commit=False)
async def recover_expired_lease():
now = datetime.now(timezone.utc)
async with session_maker() as session:
async with session.begin():
result = await session.execute(
select(Attachment)
.where(
Attachment.status == "processing",
Attachment.lease_expires_at.is_not(None),
Attachment.lease_expires_at <= now,
)
.order_by(Attachment.lease_expires_at)
.limit(1)
.with_for_update(skip_locked=True)
)
att = result.scalar_one_or_none()
if att is None:
return False
att.status = "queued"
att.lease_expires_at = None
session.add(
OutboxEvent(attachment_id=att.id)
)
attachment_id = att.id
print(f"[Outbox] Recovered expired lease: {attachment_id}", flush=True)
return True
async def dispatch_one(channel):
async with session_maker() as session:
async with session.begin():
result = await session.execute(
select(OutboxEvent)
.where(OutboxEvent.published_at.is_(None))
.order_by(OutboxEvent.created_at, OutboxEvent.id)
.limit(1)
.with_for_update(skip_locked=True)
)
event = result.scalar_one_or_none()
if event is None:
return False
confirmation = await channel.default_exchange.publish(
aio_pika.Message(
body=json.dumps(
{"attachment_id": str(event.attachment_id)}
).encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=str(event.id),
content_type="application/json",
),
routing_key=QUEUE_NAME,
mandatory=True,
)
if confirmation is False:
raise RuntimeError("RabbitMQ did not confirm publication")
event.published_at = datetime.now(timezone.utc)
print(f"[Outbox] Published {event.attachment_id}", flush=True)
return True
async def main():
try:
while True:
try:
connection = await aio_pika.connect_robust(RABBITMQ_URL)
async with connection:
channel = await connection.channel(
publisher_confirms=True,
on_return_raises=True,
)
await channel.declare_queue(
QUEUE_NAME,
durable=True,
)
while True:
recovered = await recover_expired_lease()
published = await dispatch_one(channel)
if not recovered and not published:
await asyncio.sleep(POLL_INTERVAL_SECONDS)
except asyncio.CancelledError:
raise
except Exception as exc:
print(
f"[Outbox] Retry after error: {type(exc).__name__}",
flush=True,
)
await asyncio.sleep(2)
finally:
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
+20
View File
@@ -0,0 +1,20 @@
def paginate(items: list, limit: int, cursor: str | None):
start = 0
if cursor is not None:
try:
start = int(cursor)
except ValueError:
start = 0
if start < 0:
start = 0
page = items[start : start + limit]
next_index = start + limit
if next_index < len(items):
next_cursor = str(next_index)
else:
next_cursor = None
return page, next_cursor
+9
View File
@@ -0,0 +1,9 @@
import redis.asyncio as redis
from app.config import REDIS_URL
redis_client = redis.from_url(
REDIS_URL,
decode_responses=True,
socket_timeout=3,
socket_connect_timeout=3,
)
+91
View File
@@ -0,0 +1,91 @@
import asyncio
import sys
import uuid
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.config import DATABASE_URL
from app.models.attachment import Attachment
from app.models.outbox import OutboxEvent
from app.models.user import User
from app.models.chat import Chat, ChatMember
RETRYABLE_ERROR_CODES = {
"s3_download_error",
"s3_upload_error",
"image_processing_timeout",
"image_processor_start_error",
"image_processor_error",
"image_processor_output_error",
"worker_error",
}
async def replay_failed(attachment_id: uuid.UUID):
engine = create_async_engine(DATABASE_URL)
session_maker = async_sessionmaker(engine, expire_on_commit=False)
try:
async with session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
attachment_id,
with_for_update=True,
)
if att is None:
print("Файл не найден")
return
if att.status != "failed":
print(f"Replay запрещён: status={att.status}")
return
if att.error_code not in RETRYABLE_ERROR_CODES:
print(
f"Replay запрещён: error_code={att.error_code}"
)
return
pending = await session.scalar(
select(OutboxEvent.id)
.where(
OutboxEvent.attachment_id == attachment_id,
OutboxEvent.published_at.is_(None),
)
.limit(1)
)
if pending is not None:
print("Задание уже ожидает публикации")
return
att.status = "queued"
att.error_code = None
session.add(
OutboxEvent(attachment_id=attachment_id)
)
print(f"Задание {attachment_id} отправлено на повторную обработку")
finally:
await engine.dispose()
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Использование: python -m app.replay_failed UUID")
sys.exit(1)
try:
attachment_id = uuid.UUID(sys.argv[1])
except ValueError:
print("Некорректный UUID")
sys.exit(1)
asyncio.run(replay_failed(attachment_id))
View File
+286
View File
@@ -0,0 +1,286 @@
import uuid
import filetype
import io
import hashlib
from datetime import datetime, timezone
from PIL import Image
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, text
from app.dependencies import get_session
from app.security import get_current_user
from app.models.user import User
from app.models.attachment import Attachment
from app.models.outbox import OutboxEvent
from app.models.chat import ChatMember
from app.storage import minio_client, BUCKET
router_upload = APIRouter(
prefix="/api/v1/chats/{chat_id}/attachments",
tags=["attachments"]
)
router_download = APIRouter(
prefix="/api/v1/attachments",
tags=["attachments"]
)
MAX_SIZE = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
MAX_ACTIVE_ATTACHMENTS = 100
ADMISSION_LOCK_KEY = 823164001
def format_utc_timestamp(value: datetime | None) -> str:
if value is None:
value = datetime.now(timezone.utc)
elif value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return (
value.astimezone(timezone.utc)
.isoformat()
.replace("+00:00", "Z")
)
def build_attachment_response(att: Attachment) -> dict:
variants = []
if (
att.status == "ready"
and att.thumbnail_key
and att.thumbnail_sha256
and att.thumbnail_size_bytes is not None
and att.thumbnail_width is not None
and att.thumbnail_height is not None
):
variants.append({
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": att.thumbnail_size_bytes,
"width": att.thumbnail_width,
"height": att.thumbnail_height,
"sha256": att.thumbnail_sha256,
})
return {
"id": str(att.id),
"chat_id": str(att.chat_id),
"owner_id": str(att.user_id),
"filename": att.filename,
"content_type": att.mime_type,
"size_bytes": att.size_bytes,
"sha256": att.sha256,
"status": att.status,
"variants": variants,
"error_code": att.error_code,
"created_at": format_utc_timestamp(att.created_at),
"updated_at": format_utc_timestamp(att.updated_at),
}
@router_upload.post("", status_code=202)
async def upload_attachment(
chat_id: uuid.UUID,
filename: str = Query(
...,
min_length=1,
max_length=255,
pattern=r"^[^/\\\x00]+$"
),
request: Request = None,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session),
):
result = await session.execute(
select(ChatMember).where(
ChatMember.chat_id == chat_id,
ChatMember.user_id == user.id,
)
)
if not result.scalar_one_or_none():
raise HTTPException(404, "Чат не найден")
sha_hasher = hashlib.sha256()
buffer = io.BytesIO()
total = 0
first_bytes = b""
async for chunk in request.stream():
if not chunk:
continue
total += len(chunk)
if total > MAX_SIZE:
raise HTTPException(413, "File too large")
sha_hasher.update(chunk)
buffer.write(chunk)
if len(first_bytes) < 262:
need = 262 - len(first_bytes)
first_bytes += chunk[:need]
kind = filetype.guess(first_bytes)
if not kind or kind.mime not in ("image/png", "image/jpeg"):
raise HTTPException(400, "Only PNG/JPEG allowed")
contents = buffer.getvalue()
try:
img = Image.open(io.BytesIO(contents))
if img.width * img.height > MAX_PIXELS:
raise HTTPException(400, "Image too large (pixels)")
except HTTPException:
raise
except Exception:
raise HTTPException(400, "Corrupted image")
# Serialize admission checks across all API instances.
# This transaction-level PostgreSQL lock is released on commit/rollback.
await session.execute(
text("SELECT pg_advisory_xact_lock(:lock_key)"),
{"lock_key": ADMISSION_LOCK_KEY},
)
active_count = await session.scalar(
select(func.count())
.select_from(Attachment)
.where(Attachment.status.in_(("queued", "processing")))
)
if active_count >= MAX_ACTIVE_ATTACHMENTS:
await session.rollback()
raise HTTPException(
status_code=429,
detail="Image processing queue is full; try again later",
headers={"Retry-After": "30"},
)
object_key = f"{uuid.uuid4()}.{kind.extension}"
minio_client.put_object(
BUCKET,
object_key,
io.BytesIO(contents),
len(contents),
content_type=kind.mime,
)
sha_hash = sha_hasher.hexdigest()
att = Attachment(
object_key=object_key,
filename=filename,
mime_type=kind.mime,
size_bytes=len(contents),
chat_id=chat_id,
user_id=user.id,
sha256=sha_hash,
status="queued"
)
session.add(att)
await session.flush()
event = OutboxEvent(attachment_id=att.id)
session.add(event)
await session.commit()
await session.refresh(att)
return build_attachment_response(att)
@router_download.get("/{attachment_id}/content")
async def download_attachment_content(
attachment_id: uuid.UUID,
variant: str = "original",
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session),
):
att = await session.get(Attachment, attachment_id)
if not att:
raise HTTPException(404, "Not found")
result = await session.execute(
select(ChatMember).where(
ChatMember.chat_id == att.chat_id,
ChatMember.user_id == user.id,
)
)
if not result.scalar_one_or_none():
raise HTTPException(404, "Not found")
if att.status != "ready":
raise HTTPException(409, "File is not ready yet")
if variant == "thumbnail":
if not att.thumbnail_key:
raise HTTPException(404, "Thumbnail not found")
key_to_download = att.thumbnail_key
mime = "image/jpeg"
elif variant == "original":
key_to_download = att.object_key
mime = att.mime_type
else:
raise HTTPException(404, "Variant not found")
try:
response = minio_client.get_object(
BUCKET,
key_to_download
)
return StreamingResponse(
response,
media_type=mime,
headers={
"Content-Disposition":
f'attachment; filename="{att.filename}"'
},
)
except Exception:
raise HTTPException(
404,
"File not found in storage"
)
@router_download.get("/{attachment_id}")
async def get_attachment_metadata(
attachment_id: uuid.UUID,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session),
):
att = await session.get(Attachment, attachment_id)
if not att:
raise HTTPException(404, "Not found")
result = await session.execute(
select(ChatMember).where(
ChatMember.chat_id == att.chat_id,
ChatMember.user_id == user.id,
)
)
if not result.scalar_one_or_none():
raise HTTPException(404, "Not found")
return build_attachment_response(att)
+83
View File
@@ -0,0 +1,83 @@
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPAuthorizationCredentials
from pydantic import BaseModel, ConfigDict, field_serializer
from app.redis_client import redis_client
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
import uuid
from datetime import datetime
from app.database import async_session_maker
from app.models.user import User
from app.security import verify_password, create_session, delete_session, bearer_scheme
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
security = HTTPBasic(realm="messenger")
class UserResponse(BaseModel):
id: uuid.UUID
username: str
display_name: str
created_at: datetime
model_config = ConfigDict(from_attributes=True)
@field_serializer('created_at')
def serialize_dt(self, dt: datetime):
return dt.isoformat().replace("+00:00", "Z")
class SessionResponse(BaseModel):
token: str
token_type: str
expires_at: str
user: UserResponse
@field_serializer('expires_at')
def serialize_dt(self, dt_str: str):
if dt_str and dt_str.endswith("+00:00"):
return dt_str.replace("+00:00", "Z")
return dt_str
# Basic Auth в Bearer Token
@router.post("/sessions", status_code=201, response_model=SessionResponse)
async def login(credentials: HTTPBasicCredentials = Depends(security)):
async with async_session_maker() as session:
result = await session.execute(select(User).where(User.username == credentials.username))
user = result.scalar_one_or_none()
if not user or not verify_password(credentials.password, user.password_hash):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
session_data = await create_session(user.id)
session_data["user"] = user
return session_data
@router.delete("/sessions/current", status_code=204)
async def logout(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)):
if not credentials:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
token = credentials.credentials
user_id = await redis_client.get(f"session:{token}")
if not user_id:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or expired token",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
await delete_session(token)
return None
+139
View File
@@ -0,0 +1,139 @@
import uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException, Depends
from pydantic import BaseModel, Field, field_validator, ConfigDict, field_serializer
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.models.chat import Chat, ChatMember
from app.models.user import User
from app.security import get_current_user
from app.dependencies import get_session
from app.pagination import paginate
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("member_ids")
@classmethod
def unique_members(cls, v):
if len(set(v)) != len(v):
raise ValueError("member_ids должны быть уникальными")
return v
class ChatResponse(BaseModel):
id: uuid.UUID
title: str
member_ids: list[uuid.UUID]
created_at: datetime
model_config = ConfigDict(from_attributes=True)
@field_serializer('created_at')
def serialize_dt(self, dt: datetime):
return dt.isoformat().replace("+00:00", "Z")
@router.post("", status_code=201)
async def create_chat(
payload: CreateChat,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session)
):
if str(user.id) not in payload.member_ids:
raise HTTPException(status_code=400, detail="Пользователь должен быть участником чата")
for member_id in payload.member_ids:
result = await session.execute(select(User).where(User.id == member_id))
if not result.scalar_one_or_none():
raise HTTPException(status_code=400, detail=f"несуществующий пользователь: {member_id}")
chat = Chat(title=payload.title)
session.add(chat)
await session.flush()
for member_id in payload.member_ids:
session.add(ChatMember(chat_id=chat.id, user_id=member_id))
await session.commit()
await session.refresh(chat)
return {
"id": chat.id,
"title": chat.title,
"member_ids": payload.member_ids,
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
}
@router.get("/{chat_id}")
async def get_chat(
chat_id: str,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session)
):
result = await session.execute(select(Chat).where(Chat.id == chat_id))
chat = result.scalar_one_or_none()
if not chat:
raise HTTPException(status_code=404, detail="чат не найден")
result = await session.execute(
select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user.id)
)
if not result.scalar_one_or_none():
raise HTTPException(status_code=404, detail="чат не найден")
result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id))
member_ids = []
for row in result.all():
user_id = row[0]
member_ids.append(str(user_id))
return {
"id": chat.id,
"title": chat.title,
"member_ids": member_ids,
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
}
@router.get("")
async def list_chats(
limit: int = 20,
cursor: str | None = None,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session)
):
if limit < 1 or limit > 100:
raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100")
result = await session.execute(
select(Chat)
.join(ChatMember, Chat.id == ChatMember.chat_id)
.where(ChatMember.user_id == user.id)
.order_by(Chat.created_at, Chat.id)
)
chats = result.scalars().all()
my_chats = []
for chat in chats:
result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id))
member_ids = []
for row in result.all():
user_id = row[0]
member_ids.append(str(user_id))
my_chats.append({
"id": chat.id,
"title": chat.title,
"member_ids": member_ids,
"created_at": chat.created_at.isoformat().replace("+00:00", "Z"),
})
page, next_cursor = paginate(my_chats, limit, cursor)
return {"items": page, "next_cursor": next_cursor}
+34
View File
@@ -0,0 +1,34 @@
from fastapi import APIRouter
from fastapi import HTTPException
from sqlalchemy import text
from app.database import async_session_maker
from app.redis_client import redis_client
router = APIRouter(tags=["health"])
@router.get("/health/live")
def live():
return {"status": "ok"}
@router.get("/health/ready")
async def ready():
try:
async with async_session_maker() as session:
await session.execute(text("SELECT 1"))
await redis_client.ping()
return {"status": "ok"}
except Exception:
raise HTTPException(status_code=503, detail="unavailable")
@router.get("/")
def home():
return {
"service": "MTUSI Messenger",
"version": "1.1.0",
"docs": "/docs",
}
+181
View File
@@ -0,0 +1,181 @@
import uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException, Depends, Header
from pydantic import BaseModel, Field, ConfigDict, field_serializer, model_validator
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from app.redis_client import redis_client
import json
from app.models.message import Message
from app.models.chat import Chat, ChatMember
from app.models.user import User
from app.models.attachment import Attachment
from app.security import get_current_user
from app.dependencies import get_session
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[uuid.UUID] = []
@model_validator(mode="after")
def check_text_or_attachment(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Сообщение должно содержать текст или вложение")
return self
class MessageResponse(BaseModel):
id: uuid.UUID
chat_id: uuid.UUID
sender_id: uuid.UUID
text: str
created_at: datetime
model_config = ConfigDict(from_attributes=True)
attachment_ids: list[uuid.UUID] = []
@field_serializer('created_at')
def serialize_dt(self, dt: datetime):
return dt.isoformat().replace("+00:00", "Z")
async def check_chat_access(chat_id: str, user_id: uuid.UUID, session: AsyncSession):
chat = await session.get(Chat, chat_id)
if not chat:
raise HTTPException(status_code=404, detail="чат не найден")
result = await session.execute(
select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user_id)
)
if not result.scalar_one_or_none():
raise HTTPException(status_code=404, detail="чат не найден")
return chat
@router.post("", status_code=201, response_model=MessageResponse)
async def create_message(
chat_id: str,
payload: CreateMessage,
idempotency_key: str = Header(
...,
alias="Idempotency-Key",
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9._:-]+$",
),
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session)
):
await check_chat_access(chat_id, user.id, session)
if payload.attachment_ids:
result = await session.execute(
select(Attachment.id).where(
Attachment.id.in_(payload.attachment_ids),
Attachment.chat_id == chat_id,
)
)
found_ids = {str(row[0]) for row in result.all()}
requested_ids = {str(a) for a in payload.attachment_ids}
if found_ids != requested_ids:
raise HTTPException(
status_code=400,
detail="attachment_ids содержат файлы из другого чата или несуществующие"
)
redis_key = f"idem:{user.id}:{chat_id}:{idempotency_key}"
body = payload.model_dump(mode="json")
raw = await redis_client.get(redis_key)
if raw:
stored = json.loads(raw)
if stored["state"] == "done":
if stored["request_body"] == body:
return stored["response"]
raise HTTPException(status_code=409, detail="Idempotency-Key переиспользован с другим телом запроса")
raise HTTPException(status_code=409, detail="Запрос с этим Idempotency-Key ещё обрабатывается")
acquired = await redis_client.set(
redis_key,
json.dumps({"state": "processing", "request_body": body}),
nx=True,
ex=86400,
)
if not acquired:
raise HTTPException(409, "Одновременный запрос с тем же самым Idempotency-Key")
message = Message(
chat_id=chat_id,
sender_id=user.id,
text=payload.text,
attachment_ids=[str(a) for a in payload.attachment_ids]
)
session.add(message)
await session.commit()
await session.refresh(message)
response_data = {
"id": str(message.id),
"chat_id": str(message.chat_id),
"sender_id": str(message.sender_id),
"text": message.text,
"created_at": message.created_at.isoformat().replace("+00:00", "Z"),
"attachment_ids": message.attachment_ids,
}
await redis_client.set(
redis_key,
json.dumps({"state": "done",
"request_body": body,
"response": response_data}),
ex=86400,
)
return response_data
@router.get("")
async def list_messages(
chat_id: str,
limit: int = 50,
cursor: str | None = None,
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_session)
):
if limit < 1 or limit > 100:
raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100")
await check_chat_access(chat_id, user.id, session)
offset = int(cursor) if cursor else 0
result = await session.execute(
select(Message)
.where(Message.chat_id == chat_id)
.order_by(Message.created_at, Message.id)
.limit(limit + 1)
.offset(offset)
)
msgs = result.scalars().all()
has_next = len(msgs) > limit
if has_next:
msgs = msgs[:limit]
my_messages = [
{
"id": m.id,
"chat_id": m.chat_id,
"sender_id": m.sender_id,
"text": m.text,
"created_at": m.created_at.isoformat().replace("+00:00", "Z"),
"attachment_ids": m.attachment_ids,
} for m in msgs
]
next_cursor = str(offset + limit) if has_next else None
return {"items": my_messages, "next_cursor": next_cursor}
+55
View File
@@ -0,0 +1,55 @@
import uuid
from datetime import datetime
from fastapi import APIRouter, HTTPException, Depends
from pydantic import BaseModel, Field, ConfigDict, field_serializer
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from app.models.user import User
from app.security import get_password_hash, get_current_user
from app.dependencies import get_session
router = APIRouter(prefix="/api/v1/users", tags=["users"])
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=5, max_length=128)
class UserResponse(BaseModel):
id: uuid.UUID
username: str
display_name: str
created_at: datetime
model_config = ConfigDict(from_attributes=True)
@field_serializer('created_at')
def serialize_dt(self, dt: datetime):
return dt.isoformat().replace("+00:00", "Z")
@router.post("", status_code=201, response_model=UserResponse)
async def register_user(payload: CreateUser, session: AsyncSession = Depends(get_session)):
result = await session.execute(select(User).where(User.username == payload.username))
if result.scalar_one_or_none():
raise HTTPException(status_code=409, detail="Имя пользователя уже существует")
user = User(
username=payload.username,
display_name=payload.display_name,
password_hash=get_password_hash(payload.password)
)
session.add(user)
try:
await session.commit()
except IntegrityError:
await session.rollback()
raise HTTPException(status_code=409, detail="Имя пользователя уже существует")
await session.refresh(user)
return user
@router.get("/me", response_model=UserResponse)
async def current_user(user: User = Depends(get_current_user)):
return user
+81
View File
@@ -0,0 +1,81 @@
import secrets
from datetime import datetime, timezone, timedelta
from passlib.context import CryptContext
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from sqlalchemy import select
from app.database import async_session_maker
from app.models.user import User
from app.redis_client import redis_client
from app.config import SESSION_TTL_SECONDS
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
bearer_scheme = HTTPBearer(auto_error=False)
def get_password_hash(password: str) -> str:
return pwd_context.hash(password)
def verify_password(plain: str, hashed: str) -> bool:
return pwd_context.verify(plain, hashed)
async def create_session(user_id: str) -> dict:
token = secrets.token_urlsafe(32)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)
await redis_client.setex(
name=f"session:{token}",
time=SESSION_TTL_SECONDS,
value=str(user_id),
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires_at.isoformat().replace("+00:00", "Z"),
}
async def delete_session(token: str) -> None:
await redis_client.delete(f"session:{token}")
async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)):
if not credentials:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
token = credentials.credentials
try:
user_id = await redis_client.get(f"session:{token}")
except Exception:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Внутренняя ошибка сервера"
)
if not user_id:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or expired token",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
async with async_session_maker() as session:
result = await session.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User not found",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
+22
View File
@@ -0,0 +1,22 @@
import threading
from minio import Minio
from app.config import S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
users_lock = threading.Lock()
messages_lock = threading.Lock()
users_by_id: dict[str, dict] = {}
users_by_username: dict[str, dict] = {}
chats_by_id: dict[str, dict] = {}
messages_by_chat: dict[str, list[dict]] = {}
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
minio_client = Minio(
endpoint,
access_key=S3_ACCESS_KEY_ID,
secret_key=S3_SECRET_ACCESS_KEY,
secure=False,
)
BUCKET = S3_BUCKET
+639
View File
@@ -0,0 +1,639 @@
import asyncio
import hashlib
import io
import json
import os
import random
import sys
import tempfile
import time
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
import aio_pika
from minio import Minio
from PIL import Image
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
from app.models.attachment import Attachment
from app.models.user import User
from app.models.chat import Chat, ChatMember
from app.config import (
DATABASE_URL,
S3_ENDPOINT,
S3_ACCESS_KEY_ID,
S3_SECRET_ACCESS_KEY,
S3_BUCKET,
)
RABBITMQ_URL = os.getenv(
"RABBITMQ_URL",
"amqp://guest:guest@rabbitmq:5672/",
)
QUEUE_NAME = "image_tasks"
MAX_ATTEMPTS = max(
1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5"))
)
CONCURRENCY = max(
1, int(os.getenv("WORKER_CONCURRENCY", "1"))
)
PREFETCH = max(
1, int(os.getenv("WORKER_PREFETCH", "1"))
)
MAX_IMAGE_BYTES = 10 * 1024 * 1024
PROCESS_TIMEOUT_SECONDS = 30
LEASE_SECONDS = 300
endpoint = S3_ENDPOINT.replace(
"http://", ""
).replace(
"https://", ""
)
minio_client = Minio(
endpoint,
access_key=S3_ACCESS_KEY_ID,
secret_key=S3_SECRET_ACCESS_KEY,
secure=False,
)
engine = create_async_engine(DATABASE_URL)
async_session_maker = async_sessionmaker(
engine,
expire_on_commit=False,
)
class RetryableProcessingError(Exception):
def __init__(self, error_code: str):
self.error_code = error_code
super().__init__(error_code)
class PermanentProcessingError(Exception):
def __init__(self, error_code: str):
self.error_code = error_code
super().__init__(error_code)
class LeaseBusyError(Exception):
pass
class LeaseLostError(Exception):
pass
def get_rss_mb():
try:
with open("/proc/self/status") as file:
for line in file:
if line.startswith("VmRSS:"):
kb = int(line.split()[1])
return round(kb / 1024, 2)
except (OSError, ValueError, IndexError):
pass
return None
async def claim_attachment(attachment_id: str):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if att is None:
print(
f"[Worker] Файл {attachment_id} не найден",
flush=True,
)
return None
if att.status == "ready" and att.thumbnail_key:
print(
f"[Worker] Файл {attachment_id} уже обработан",
flush=True,
)
return None
if att.status == "failed":
print(
f"[Worker] Файл {attachment_id} уже failed",
flush=True,
)
return None
now = datetime.now(timezone.utc)
if (
att.status == "processing"
and att.lease_expires_at is not None
and att.lease_expires_at > now
):
raise LeaseBusyError()
lease_expires_at = now + timedelta(
seconds=LEASE_SECONDS
)
att.status = "processing"
att.error_code = None
att.lease_expires_at = lease_expires_at
object_key = att.object_key
return object_key, lease_expires_at
async def release_lease(
attachment_id: str,
lease_expires_at: datetime,
):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if (
att is not None
and att.status == "processing"
and att.lease_expires_at == lease_expires_at
):
att.status = "queued"
att.lease_expires_at = None
async def mark_failed(
attachment_id: str,
error_code: str,
):
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if att is None or att.status == "ready":
return
if att.status == "processing":
raise LeaseBusyError()
att.status = "failed"
att.error_code = error_code
att.lease_expires_at = None
print(
f"[Worker] Файл {attachment_id}: "
f"failed, error_code={error_code}",
flush=True,
)
async def process_image(image_data: bytes):
with tempfile.TemporaryDirectory() as temp_dir:
directory = Path(temp_dir)
input_path = directory / "original"
input_path.write_bytes(image_data)
try:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.image_processor",
str(input_path),
str(directory),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
except OSError as exc:
raise RetryableProcessingError(
"image_processor_start_error"
) from exc
try:
await asyncio.wait_for(
process.communicate(),
timeout=PROCESS_TIMEOUT_SECONDS,
)
except asyncio.TimeoutError as exc:
if process.returncode is None:
try:
process.kill()
except ProcessLookupError:
pass
await process.communicate()
raise RetryableProcessingError(
"image_processing_timeout"
) from exc
if process.returncode == 20:
raise PermanentProcessingError(
"image_too_many_pixels"
)
if process.returncode == 21:
raise PermanentProcessingError(
"image_processing_error"
)
if process.returncode != 0:
raise RetryableProcessingError(
"image_processor_error"
)
try:
thumb_bytes = (
directory / "thumbnail.jpg"
).read_bytes()
crop_bytes = (
directory / "crop.jpg"
).read_bytes()
webp_bytes = (
directory / "optimized.webp"
).read_bytes()
with Image.open(
io.BytesIO(thumb_bytes)
) as thumbnail:
thumb_width, thumb_height = thumbnail.size
except OSError as exc:
raise RetryableProcessingError(
"image_processor_output_error"
) from exc
return (
thumb_bytes,
crop_bytes,
webp_bytes,
thumb_width,
thumb_height,
)
async def process_attachment(attachment_id: str):
print(
f"[Worker] Обработка файла: {attachment_id}",
flush=True,
)
claim = await claim_attachment(attachment_id)
if claim is None:
return
object_key, lease_expires_at = claim
try:
try:
response = minio_client.get_object(
S3_BUCKET,
object_key,
)
try:
image_data = response.read(
MAX_IMAGE_BYTES + 1
)
finally:
response.close()
response.release_conn()
except Exception as exc:
raise RetryableProcessingError(
"s3_download_error"
) from exc
if len(image_data) > MAX_IMAGE_BYTES:
raise PermanentProcessingError(
"image_too_large"
)
(
thumb_bytes,
crop_bytes,
webp_bytes,
thumb_width,
thumb_height,
) = await process_image(image_data)
thumb_key = f"thumbnails/{object_key}"
crop_key = f"crops/{attachment_id}.jpg"
webp_key = f"webp/{attachment_id}.webp"
try:
minio_client.put_object(
S3_BUCKET,
thumb_key,
io.BytesIO(thumb_bytes),
len(thumb_bytes),
content_type="image/jpeg",
)
minio_client.put_object(
S3_BUCKET,
crop_key,
io.BytesIO(crop_bytes),
len(crop_bytes),
content_type="image/jpeg",
)
minio_client.put_object(
S3_BUCKET,
webp_key,
io.BytesIO(webp_bytes),
len(webp_bytes),
content_type="image/webp",
)
except Exception as exc:
raise RetryableProcessingError(
"s3_upload_error"
) from exc
async with async_session_maker() as session:
async with session.begin():
att = await session.get(
Attachment,
uuid.UUID(attachment_id),
with_for_update=True,
)
if (
att is None
or att.status != "processing"
or att.lease_expires_at != lease_expires_at
):
raise LeaseLostError()
att.thumbnail_key = thumb_key
att.thumbnail_sha256 = hashlib.sha256(
thumb_bytes
).hexdigest()
att.thumbnail_size_bytes = len(thumb_bytes)
att.thumbnail_width = thumb_width
att.thumbnail_height = thumb_height
att.crop_key = crop_key
att.webp_key = webp_key
att.status = "ready"
att.error_code = None
att.lease_expires_at = None
print(
f"[Worker] Успешно обработан {attachment_id}",
flush=True,
)
except Exception:
await release_lease(
attachment_id,
lease_expires_at,
)
raise
async def main():
connection = await aio_pika.connect_robust(
RABBITMQ_URL
)
async with connection:
channel = await connection.channel(
publisher_confirms=True
)
await channel.set_qos(
prefetch_count=PREFETCH
)
queue = await channel.declare_queue(
QUEUE_NAME,
durable=True,
)
semaphore = asyncio.Semaphore(CONCURRENCY)
print(
f"[Worker] CONCURRENCY={CONCURRENCY}, "
f"PREFETCH={PREFETCH}",
flush=True,
)
async def on_message(
message: aio_pika.IncomingMessage
):
async with semaphore:
started = time.perf_counter()
try:
data = json.loads(
message.body.decode()
)
attachment_id = str(
uuid.UUID(data["attachment_id"])
)
except (
ValueError,
KeyError,
TypeError,
):
print(
"[Worker] Некорректное сообщение",
flush=True,
)
await message.reject(
requeue=False
)
return
headers = dict(message.headers or {})
try:
attempt = max(
1,
int(headers.get("x-attempt", 1)),
)
except (ValueError, TypeError):
attempt = 1
print(
f"[Worker] Попытка "
f"{attempt}/{MAX_ATTEMPTS} "
f"для {attachment_id}",
flush=True,
)
try:
await process_attachment(
attachment_id
)
except LeaseBusyError:
print(
"[Worker] Задание уже "
"обрабатывается, ждём Lease",
flush=True,
)
await asyncio.sleep(5)
await message.nack(
requeue=True
)
return
except LeaseLostError:
print(
"[Worker] Lease перешёл "
"другому Worker",
flush=True,
)
await message.ack()
return
except PermanentProcessingError as exc:
try:
await mark_failed(
attachment_id,
exc.error_code,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
except Exception as exc:
error_code = (
exc.error_code
if isinstance(
exc,
RetryableProcessingError,
)
else "worker_error"
)
if attempt < MAX_ATTEMPTS:
next_attempt = attempt + 1
print(
f"[Worker] Ошибка {error_code}. "
f"Повтор "
f"{next_attempt}/{MAX_ATTEMPTS}",
flush=True,
)
headers["x-attempt"] = next_attempt
base_delay = min(
30,
2 ** (attempt - 1),
)
delay = base_delay + random.uniform(
0,
base_delay / 2,
)
print(
f"[Worker] Повтор через "
f"{delay:.1f} сек.",
flush=True,
)
await asyncio.sleep(delay)
try:
await channel.default_exchange.publish(
aio_pika.Message(
body=message.body,
headers=headers,
delivery_mode=(
aio_pika.DeliveryMode.PERSISTENT
),
content_type="application/json",
),
routing_key=QUEUE_NAME,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
print(
f"[Worker] Попытки исчерпаны: "
f"{attachment_id}",
flush=True,
)
try:
await mark_failed(
attachment_id,
error_code,
)
except Exception:
await message.nack(
requeue=True
)
return
await message.ack()
return
await message.ack()
duration = time.perf_counter() - started
rss_mb = get_rss_mb()
print(
f"[Metrics] attachment_id={attachment_id} "
f"duration_s={duration:.3f} "
f"rss_mb={rss_mb}",
flush=True,
)
print(
"[Worker] Ждем сообщения из RabbitMQ...",
flush=True,
)
await queue.consume(on_message)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())
+259
View File
@@ -0,0 +1,259 @@
import csv
import re
import subprocess
import time
from pathlib import Path
from benchmark_rss import command, start_worker, remove_test_worker
RESULTS_DIR = Path("benchmark_results")
RESULTS_DIR.mkdir(exist_ok=True)
QUEUE_NAME = "image_tasks"
COUNT_PER_SIZE = 40
TOTAL = COUNT_PER_SIZE * 3
def queue_stats():
try:
result = subprocess.run(
[
"docker", "exec",
"mtusi_messenger_rabbitmq",
"rabbitmqctl", "-q", "list_queues",
"name",
"messages_ready",
"messages_unacknowledged",
],
capture_output=True,
text=True,
timeout=20,
)
except subprocess.TimeoutExpired as exc:
raise RuntimeError(
"RabbitMQ не ответил на запрос статистики"
) from exc
if result.returncode != 0:
raise RuntimeError(
"Не удалось получить статистику RabbitMQ:\n"
+ result.stderr
)
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) >= 3 and parts[0] == QUEUE_NAME:
return int(parts[1]), int(parts[2])
raise RuntimeError(
f"Очередь {QUEUE_NAME} не найдена:\n"
+ result.stdout
)
def run_test():
print("Останавливаем обычный Worker...")
command(["docker", "compose", "stop", "worker"])
print("Запускаем Worker 1/1...")
start_worker(1, 1)
initial_ready, initial_unacked = queue_stats()
print(
f"Начальная очередь: ready={initial_ready}, "
f"unacked={initial_unacked}"
)
if initial_ready or initial_unacked:
raise RuntimeError(
"Очередь не пустая. Для чистого эксперимента "
"сначала нужно завершить предыдущие задания."
)
print(f"\nЗапускаем нагрузку: {TOTAL} фотографий")
log_path = RESULTS_DIR / "backpressure_load.log"
samples = []
with log_path.open("w") as log_file:
process = subprocess.Popen(
[
"docker", "compose", "exec",
"-T",
"-w", "/tmp",
"-e", "PYTHONPATH=/app",
"app1",
"python", "-m", "app.load_test",
"--count", str(COUNT_PER_SIZE),
"--timeout", "600",
],
stdout=log_file,
stderr=subprocess.STDOUT,
)
started = time.monotonic()
try:
while process.poll() is None:
ready, unacked = queue_stats()
elapsed = time.monotonic() - started
samples.append({
"elapsed_s": round(elapsed, 2),
"queue_ready": ready,
"unacked": unacked,
"total_pending": ready + unacked,
})
print(
f"[QUEUE] {elapsed:6.1f}s | "
f"ready={ready:3} | "
f"unacked={unacked} | "
f"pending={ready + unacked:3}",
flush=True,
)
time.sleep(1)
return_code = process.wait()
except BaseException:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
raise
output = log_path.read_text()
if return_code != 0:
print(output)
raise RuntimeError("Нагрузочный тест завершился с ошибкой")
print("\n" + output)
ready_match = re.search(
r"^Ready:\s*(\d+)",
output,
re.MULTILINE,
)
failed_match = re.search(
r"^Failed:\s*(\d+)",
output,
re.MULTILINE,
)
completed = (
int(ready_match.group(1))
if ready_match else -1
)
failed = (
int(failed_match.group(1))
if failed_match else -1
)
print("Проверяем окончательное состояние очереди...")
final_ready = None
final_unacked = None
for _ in range(15):
final_ready, final_unacked = queue_stats()
if final_ready == 0 and final_unacked == 0:
break
time.sleep(1)
peak_ready = max(
(row["queue_ready"] for row in samples),
default=0,
)
peak_unacked = max(
(row["unacked"] for row in samples),
default=0,
)
csv_path = RESULTS_DIR / "backpressure_queue.csv"
with csv_path.open("w", newline="") as file:
writer = csv.DictWriter(
file,
fieldnames=[
"elapsed_s",
"queue_ready",
"unacked",
"total_pending",
],
)
writer.writeheader()
writer.writerows(samples)
print("\n" + "=" * 60)
print("BACKPRESSURE RESULTS")
print("=" * 60)
print(f"Total jobs: {TOTAL}")
print(f"Ready: {completed}")
print(f"Failed: {failed}")
print(f"Peak queued: {peak_ready}")
print(f"Peak unacked: {peak_unacked}")
print(f"Final queued: {final_ready}")
print(f"Final unacked: {final_unacked}")
backlog_observed = peak_ready > 0
prefetch_respected = peak_unacked <= 1
recovered = (
completed == TOTAL
and failed == 0
and final_ready == 0
and final_unacked == 0
)
print(
"Backlog observed:",
"YES" if backlog_observed else "NO",
)
print(
"Prefetch limit respected:",
"YES" if prefetch_respected else "NO",
)
print(
"Recovery:",
"PASSED" if recovered else "FAILED",
)
print(f"CSV: {csv_path}")
print("=" * 60)
def main():
try:
run_test()
finally:
print("\nВосстанавливаем обычный Worker...")
remove_test_worker()
result = command(
["docker", "compose", "up", "-d", "worker"],
check=False,
)
if result.returncode != 0:
print(result.stderr)
if __name__ == "__main__":
main()
+16
View File
@@ -0,0 +1,16 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
fa479f2c-54ac-44cf-baf0-e14d84b3fff2,512,ready,2.288,2.393
8da11c17-a222-4532-888a-92cdf1bbab94,512,ready,4.895,5.104
1441ad9e-fcc3-47c3-84e6-490559ad289a,512,ready,2.704,2.81
a01f4d3e-8eb5-46d5-b778-cd956d9db214,512,ready,4.376,4.584
53019d9f-949e-4aef-ab42-636d3fd330d2,512,ready,4.272,4.376
a91a25c2-c464-4166-9c69-ec382a729bb8,1024,ready,4.584,4.895
648e14f9-3596-44d3-870b-2c7e8357181a,1024,ready,5.419,5.835
e127fe4e-b18d-4914-8945-3cb0aa9ef2ab,1024,ready,5.104,5.419
4db57861-f63d-4971-8ccc-7fc1b5490f84,1024,ready,3.855,4.272
eb1b8add-1ad5-41a9-a745-e8ee5fdaf779,1024,ready,0.831,1.249
32dc271d-676c-4834-8575-8a273c13722b,2048,ready,0.831,2.288
ffd1408c-cf40-4805-bf30-5a515bebd116,2048,ready,2.81,4.272
a981c934-34b2-44fb-a09e-987c8ee66ac6,2048,ready,2.393,3.855
5e69aead-9343-4ca7-bee9-27ae9dcbd785,2048,ready,1.249,2.704
c0d08a45-7aa1-4afd-bd40-1d6864a4801e,2048,ready,4.272,5.627
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 fa479f2c-54ac-44cf-baf0-e14d84b3fff2 512 ready 2.288 2.393
3 8da11c17-a222-4532-888a-92cdf1bbab94 512 ready 4.895 5.104
4 1441ad9e-fcc3-47c3-84e6-490559ad289a 512 ready 2.704 2.81
5 a01f4d3e-8eb5-46d5-b778-cd956d9db214 512 ready 4.376 4.584
6 53019d9f-949e-4aef-ab42-636d3fd330d2 512 ready 4.272 4.376
7 a91a25c2-c464-4166-9c69-ec382a729bb8 1024 ready 4.584 4.895
8 648e14f9-3596-44d3-870b-2c7e8357181a 1024 ready 5.419 5.835
9 e127fe4e-b18d-4914-8945-3cb0aa9ef2ab 1024 ready 5.104 5.419
10 4db57861-f63d-4971-8ccc-7fc1b5490f84 1024 ready 3.855 4.272
11 eb1b8add-1ad5-41a9-a745-e8ee5fdaf779 1024 ready 0.831 1.249
12 32dc271d-676c-4834-8575-8a273c13722b 2048 ready 0.831 2.288
13 ffd1408c-cf40-4805-bf30-5a515bebd116 2048 ready 2.81 4.272
14 a981c934-34b2-44fb-a09e-987c8ee66ac6 2048 ready 2.393 3.855
15 5e69aead-9343-4ca7-bee9-27ae9dcbd785 2048 ready 1.249 2.704
16 c0d08a45-7aa1-4afd-bd40-1d6864a4801e 2048 ready 4.272 5.627
+16
View File
@@ -0,0 +1,16 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
e2b24b97-ad9b-46d1-8790-2041f58914ca,512,ready,1.235,1.337
971a55e8-3fd0-4215-b752-fff9b31ab126,512,ready,6.981,7.082
f2e85596-fbcc-4e43-a534-d1d27cd75fc9,512,ready,7.082,7.183
0ad8283c-968d-4525-a80f-5860f1da4b8b,512,ready,2.681,2.785
bcdaec76-41ae-454e-b45f-174a26326b19,512,ready,1.337,1.544
1eaccbad-b7a5-49d1-afdf-a6a137137034,1024,ready,0.821,1.235
b78d8044-cb8f-4501-9462-7ac880cc1bc4,1024,ready,1.544,1.857
86200ea3-a17e-4074-b1ae-90641804c48c,1024,ready,0.208,0.821
383bbc63-331c-4510-b74b-f316c91ad902,1024,ready,1.857,2.277
9782bfbf-9fbe-4e22-9a4b-18fa02924ccc,1024,ready,2.277,2.681
d5594ebc-56bd-4007-b9cf-855db2f59e7f,2048,ready,4.218,5.553
1061bf47-bde5-49a1-9971-ef41eefe499e,2048,ready,7.183,8.618
70f8f306-73e6-4794-83fe-93c29bfd2c72,2048,ready,2.785,4.218
c505d22f-01f2-4396-bb27-ebf2d1ff473b,2048,ready,8.618,9.958
793328a6-cf5e-4078-814e-64d4c4e1e66e,2048,ready,5.553,6.981
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 e2b24b97-ad9b-46d1-8790-2041f58914ca 512 ready 1.235 1.337
3 971a55e8-3fd0-4215-b752-fff9b31ab126 512 ready 6.981 7.082
4 f2e85596-fbcc-4e43-a534-d1d27cd75fc9 512 ready 7.082 7.183
5 0ad8283c-968d-4525-a80f-5860f1da4b8b 512 ready 2.681 2.785
6 bcdaec76-41ae-454e-b45f-174a26326b19 512 ready 1.337 1.544
7 1eaccbad-b7a5-49d1-afdf-a6a137137034 1024 ready 0.821 1.235
8 b78d8044-cb8f-4501-9462-7ac880cc1bc4 1024 ready 1.544 1.857
9 86200ea3-a17e-4074-b1ae-90641804c48c 1024 ready 0.208 0.821
10 383bbc63-331c-4510-b74b-f316c91ad902 1024 ready 1.857 2.277
11 9782bfbf-9fbe-4e22-9a4b-18fa02924ccc 1024 ready 2.277 2.681
12 d5594ebc-56bd-4007-b9cf-855db2f59e7f 2048 ready 4.218 5.553
13 1061bf47-bde5-49a1-9971-ef41eefe499e 2048 ready 7.183 8.618
14 70f8f306-73e6-4794-83fe-93c29bfd2c72 2048 ready 2.785 4.218
15 c505d22f-01f2-4396-bb27-ebf2d1ff473b 2048 ready 8.618 9.958
16 793328a6-cf5e-4078-814e-64d4c4e1e66e 2048 ready 5.553 6.981
+16
View File
@@ -0,0 +1,16 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
a940e027-a230-4343-a771-96ef998b207c,512,ready,1.155,1.364
3a603917-37dd-4ce7-91fc-04771c87e7aa,512,ready,1.568,1.774
1d20bc66-466e-4bd3-a754-57500694cb51,512,ready,1.467,1.568
efc77f3e-f5d0-40da-9f33-155a0eb4db43,512,ready,1.155,1.364
32a3e8e9-bf48-4db0-9333-5cf0cc183acc,512,ready,1.364,1.467
ecf65392-5165-4990-bcb4-42d80628cd86,1024,ready,1.774,2.19
b8bcc085-d1d9-411a-ac1c-08688140d4c8,1024,ready,1.155,1.568
7eeed358-a38b-4d39-a4ac-55476bfcdcfe,1024,ready,1.364,1.774
1e0fb527-7023-4cc2-8292-56b7bbef10c8,1024,ready,1.155,1.568
f29de97b-d1c1-4200-b80a-4b7babdfc2d1,1024,ready,1.774,2.19
8e2bb5c9-e613-4343-8fc1-fe6d3eda869d,2048,ready,3.647,5.102
2038744a-177e-4a3b-bc22-85d6b82fc879,2048,ready,1.568,3.544
748dbf01-4049-4297-837f-cb3457875e10,2048,ready,2.19,3.859
f578c650-71b8-472b-8a0f-c6c09bcd00e3,2048,ready,2.19,4.171
3ed1e2a1-43d5-4f9f-a475-01fa3cac932e,2048,ready,1.568,3.544
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 a940e027-a230-4343-a771-96ef998b207c 512 ready 1.155 1.364
3 3a603917-37dd-4ce7-91fc-04771c87e7aa 512 ready 1.568 1.774
4 1d20bc66-466e-4bd3-a754-57500694cb51 512 ready 1.467 1.568
5 efc77f3e-f5d0-40da-9f33-155a0eb4db43 512 ready 1.155 1.364
6 32a3e8e9-bf48-4db0-9333-5cf0cc183acc 512 ready 1.364 1.467
7 ecf65392-5165-4990-bcb4-42d80628cd86 1024 ready 1.774 2.19
8 b8bcc085-d1d9-411a-ac1c-08688140d4c8 1024 ready 1.155 1.568
9 7eeed358-a38b-4d39-a4ac-55476bfcdcfe 1024 ready 1.364 1.774
10 1e0fb527-7023-4cc2-8292-56b7bbef10c8 1024 ready 1.155 1.568
11 f29de97b-d1c1-4200-b80a-4b7babdfc2d1 1024 ready 1.774 2.19
12 8e2bb5c9-e613-4343-8fc1-fe6d3eda869d 2048 ready 3.647 5.102
13 2038744a-177e-4a3b-bc22-85d6b82fc879 2048 ready 1.568 3.544
14 748dbf01-4049-4297-837f-cb3457875e10 2048 ready 2.19 3.859
15 f578c650-71b8-472b-8a0f-c6c09bcd00e3 2048 ready 2.19 4.171
16 3ed1e2a1-43d5-4f9f-a475-01fa3cac932e 2048 ready 1.568 3.544
+64
View File
@@ -0,0 +1,64 @@
elapsed_s,queue_ready,unacked,total_pending
0.37,0,0,0
1.74,0,0,0
3.15,119,1,120
4.5,118,1,119
5.81,115,1,116
7.16,114,1,115
8.45,112,1,113
9.75,111,1,112
11.06,108,1,109
12.37,107,1,108
13.67,104,1,105
14.98,101,1,102
16.26,99,1,100
17.55,96,1,97
18.85,93,1,94
20.14,92,1,93
21.44,91,1,92
22.74,86,1,87
24.05,84,1,85
25.35,82,1,83
26.66,80,1,81
27.98,75,1,76
29.27,71,1,72
30.57,68,1,69
31.87,66,1,67
33.18,65,1,66
34.48,64,1,65
35.77,63,1,64
37.06,60,1,61
38.43,58,1,59
39.76,57,1,58
41.05,57,1,58
42.37,52,1,53
43.67,51,1,52
44.98,50,1,51
46.27,49,1,50
47.58,48,1,49
48.87,46,1,47
50.17,45,1,46
51.47,44,1,45
52.77,43,1,44
54.08,42,1,43
55.37,39,1,40
56.68,38,1,39
57.97,37,1,38
59.26,35,1,36
60.55,33,1,34
61.84,31,1,32
63.15,30,1,31
64.44,29,1,30
65.73,23,1,24
67.03,22,1,23
68.34,18,1,19
69.63,17,1,18
70.93,14,1,15
72.23,12,1,13
73.52,11,1,12
74.82,9,1,10
76.11,6,1,7
77.4,5,1,6
78.69,2,1,3
79.98,1,1,2
81.28,0,1,1
1 elapsed_s queue_ready unacked total_pending
2 0.37 0 0 0
3 1.74 0 0 0
4 3.15 119 1 120
5 4.5 118 1 119
6 5.81 115 1 116
7 7.16 114 1 115
8 8.45 112 1 113
9 9.75 111 1 112
10 11.06 108 1 109
11 12.37 107 1 108
12 13.67 104 1 105
13 14.98 101 1 102
14 16.26 99 1 100
15 17.55 96 1 97
16 18.85 93 1 94
17 20.14 92 1 93
18 21.44 91 1 92
19 22.74 86 1 87
20 24.05 84 1 85
21 25.35 82 1 83
22 26.66 80 1 81
23 27.98 75 1 76
24 29.27 71 1 72
25 30.57 68 1 69
26 31.87 66 1 67
27 33.18 65 1 66
28 34.48 64 1 65
29 35.77 63 1 64
30 37.06 60 1 61
31 38.43 58 1 59
32 39.76 57 1 58
33 41.05 57 1 58
34 42.37 52 1 53
35 43.67 51 1 52
36 44.98 50 1 51
37 46.27 49 1 50
38 47.58 48 1 49
39 48.87 46 1 47
40 50.17 45 1 46
41 51.47 44 1 45
42 52.77 43 1 44
43 54.08 42 1 43
44 55.37 39 1 40
45 56.68 38 1 39
46 57.97 37 1 38
47 59.26 35 1 36
48 60.55 33 1 34
49 61.84 31 1 32
50 63.15 30 1 31
51 64.44 29 1 30
52 65.73 23 1 24
53 67.03 22 1 23
54 68.34 18 1 19
55 69.63 17 1 18
56 70.93 14 1 15
57 72.23 12 1 13
58 73.52 11 1 12
59 74.82 9 1 10
60 76.11 6 1 7
61 77.4 5 1 6
62 78.69 2 1 3
63 79.98 1 1 2
64 81.28 0 1 1
+61
View File
@@ -0,0 +1,61 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
28bebbfc-799e-4217-8c30-4f47f1e46c7e,512,ready,5.683,5.893
96b8fb1e-3ca4-497e-a4b0-0d1f73522c44,512,ready,10.223,10.328
ea372b62-a051-4cc1-afc5-dde4fdfdb85f,512,ready,5.893,6.098
6d4cae25-8487-401b-b89f-16fd3e012079,512,ready,6.098,6.201
42bcd40d-5de7-42d9-9cac-511bc000160f,512,ready,4.137,4.241
f6c50369-fcfa-48c4-b04d-8599c2c20711,512,ready,5.376,5.478
f55f9964-9adc-4adc-b48e-9c93d4019889,512,ready,5.893,6.098
b3ea2331-706e-4dd1-81ae-8972c922075f,512,ready,4.137,4.241
2f0d8af7-ee6f-479a-96b9-c24c311b1790,512,ready,9.709,9.812
9fca7598-037f-4f6a-81dd-038bc8f5c9d3,512,ready,3.93,4.137
3c1673ca-f03b-41d4-8706-f8a65a1951af,512,ready,2.901,3.003
0eaad040-3f62-446d-842e-fc4a21cb74cc,512,ready,5.893,6.098
ee0404fd-8dd6-49dd-9a77-e849a0382ff2,512,ready,0.843,1.048
20f6197b-e3fb-4ce8-9985-13aa260a3e9f,512,ready,5.478,5.683
9d401a3c-d5d5-45b9-9153-8a392132c74a,512,ready,5.683,5.893
b2de90a9-4354-4c66-b905-c2cd501e94b2,512,ready,3.829,4.034
ed1ded18-31cc-495e-9e1e-7a15cb77fb67,512,ready,5.683,5.893
a8e31c1d-0ef1-4fea-a538-3fd9e36a594c,512,ready,6.201,6.406
5451263a-8d6b-4940-b44b-87c1bbb66b97,512,ready,8.671,8.773
0af8a964-e248-4153-936a-7bd5f7ec9c82,512,ready,8.049,8.259
c6ab01d9-cb1e-4154-8678-77453b549a82,1024,ready,9.812,10.223
b3f6ab04-bfe9-45c3-a409-d8aed2efba92,1024,ready,5.893,6.303
de9ed0d6-8d6d-41a4-8eb4-16eb6793eca3,1024,ready,9.297,9.709
452d6dce-5947-4d08-8983-37b9b517ea0a,1024,ready,8.362,8.773
91f662ce-6cec-4586-b2a0-1a5cc101340b,1024,ready,1.048,1.46
b41f3d80-1b54-4402-9928-a83e67e2aa2f,1024,ready,2.491,2.901
4b7cfdbf-8bdc-4569-96dc-8410b0fb965d,1024,ready,3.003,3.412
19df5c43-84d5-4fc6-8469-daf469f8d096,1024,ready,1.46,1.975
c5211831-85d2-4c44-a871-9acd71a17337,1024,ready,8.259,8.671
369d13b1-ead7-4aed-bcd4-9f12e918d038,1024,ready,4.034,4.452
7e9c5099-2f5e-4ea9-a6e2-d4acc540901f,1024,ready,7.946,8.362
927b907a-fbc4-4383-a6e3-0fd02206f79c,1024,ready,6.098,6.51
d96167a6-cda0-46a8-adad-df52ccc789bd,1024,ready,9.709,10.121
e9577c3a-8d88-46c9-9ad5-1979469b38be,1024,ready,0.843,1.356
52f8e432-ff40-4419-8968-17ef1089a361,1024,ready,4.966,5.376
99a1e17e-94b0-4ca1-810f-fa5e3ff9691f,1024,ready,3.31,3.726
33eb7872-0980-4336-a560-cf6f14a302f9,1024,ready,9.297,9.709
4a3cacba-350f-4878-85f7-ead76d7cacbe,1024,ready,3.726,4.137
fc3cb344-8f7c-4421-bae0-ce5e282d6143,1024,ready,1.975,2.39
22228d29-952b-4ec0-aad0-b8762967dee0,1024,ready,1.356,1.768
1d29e426-37fa-41e4-b5cc-55eedbf2af38,2048,ready,8.773,10.223
d62cc431-925c-47c6-9e0a-bec4c2d561dc,2048,ready,0.843,2.491
4d3d5437-fded-4c5a-9ab5-50460df9fade,2048,ready,4.452,5.893
78668df5-05d3-4ca1-b309-fcb16181d973,2048,ready,4.241,5.683
86af36aa-2193-4da2-a786-1787fb6229d5,2048,ready,2.39,3.829
211fbe0b-a085-4549-812a-07a2c326a9b2,2048,ready,6.51,8.049
f8e04547-8aaa-4b87-9e9d-383e387ade00,2048,ready,0.843,2.491
9b065c78-8c00-4fe5-a890-bfa698c8bfea,2048,ready,4.241,5.683
3906bf23-8afc-47d1-85c5-b2d0eb207236,2048,ready,10.223,11.673
4a2f285d-327c-44be-ba2c-da0d627109cc,2048,ready,6.406,7.946
f1f72a39-92e6-4284-a373-30214f1141bc,2048,ready,10.223,11.673
9eee27d0-b3fb-4be4-ab0f-3f99b0d024f1,2048,ready,6.098,7.741
cb946763-c80a-4284-85b3-3aea16d2eced,2048,ready,10.223,11.673
f0aed629-1d01-4f7a-a699-eff85d4dd239,2048,ready,7.844,9.297
b7957cd4-0942-40cd-a1f9-a4a614d4c5f3,2048,ready,8.773,10.223
ea48ce0d-a3e8-44f3-bdbc-37d1ddfc9eb0,2048,ready,7.844,9.297
e6a26bb8-4c70-4074-b130-357a742c6b5f,2048,ready,1.768,3.31
1530b7a5-227c-4bb0-ad09-d975105a757c,2048,ready,3.412,4.864
e9bcf42c-2a3b-43ee-8802-3d1432bfba5a,2048,ready,6.303,7.844
390b5116-460d-45f6-976b-240c2b6066bf,2048,ready,2.491,3.93
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 28bebbfc-799e-4217-8c30-4f47f1e46c7e 512 ready 5.683 5.893
3 96b8fb1e-3ca4-497e-a4b0-0d1f73522c44 512 ready 10.223 10.328
4 ea372b62-a051-4cc1-afc5-dde4fdfdb85f 512 ready 5.893 6.098
5 6d4cae25-8487-401b-b89f-16fd3e012079 512 ready 6.098 6.201
6 42bcd40d-5de7-42d9-9cac-511bc000160f 512 ready 4.137 4.241
7 f6c50369-fcfa-48c4-b04d-8599c2c20711 512 ready 5.376 5.478
8 f55f9964-9adc-4adc-b48e-9c93d4019889 512 ready 5.893 6.098
9 b3ea2331-706e-4dd1-81ae-8972c922075f 512 ready 4.137 4.241
10 2f0d8af7-ee6f-479a-96b9-c24c311b1790 512 ready 9.709 9.812
11 9fca7598-037f-4f6a-81dd-038bc8f5c9d3 512 ready 3.93 4.137
12 3c1673ca-f03b-41d4-8706-f8a65a1951af 512 ready 2.901 3.003
13 0eaad040-3f62-446d-842e-fc4a21cb74cc 512 ready 5.893 6.098
14 ee0404fd-8dd6-49dd-9a77-e849a0382ff2 512 ready 0.843 1.048
15 20f6197b-e3fb-4ce8-9985-13aa260a3e9f 512 ready 5.478 5.683
16 9d401a3c-d5d5-45b9-9153-8a392132c74a 512 ready 5.683 5.893
17 b2de90a9-4354-4c66-b905-c2cd501e94b2 512 ready 3.829 4.034
18 ed1ded18-31cc-495e-9e1e-7a15cb77fb67 512 ready 5.683 5.893
19 a8e31c1d-0ef1-4fea-a538-3fd9e36a594c 512 ready 6.201 6.406
20 5451263a-8d6b-4940-b44b-87c1bbb66b97 512 ready 8.671 8.773
21 0af8a964-e248-4153-936a-7bd5f7ec9c82 512 ready 8.049 8.259
22 c6ab01d9-cb1e-4154-8678-77453b549a82 1024 ready 9.812 10.223
23 b3f6ab04-bfe9-45c3-a409-d8aed2efba92 1024 ready 5.893 6.303
24 de9ed0d6-8d6d-41a4-8eb4-16eb6793eca3 1024 ready 9.297 9.709
25 452d6dce-5947-4d08-8983-37b9b517ea0a 1024 ready 8.362 8.773
26 91f662ce-6cec-4586-b2a0-1a5cc101340b 1024 ready 1.048 1.46
27 b41f3d80-1b54-4402-9928-a83e67e2aa2f 1024 ready 2.491 2.901
28 4b7cfdbf-8bdc-4569-96dc-8410b0fb965d 1024 ready 3.003 3.412
29 19df5c43-84d5-4fc6-8469-daf469f8d096 1024 ready 1.46 1.975
30 c5211831-85d2-4c44-a871-9acd71a17337 1024 ready 8.259 8.671
31 369d13b1-ead7-4aed-bcd4-9f12e918d038 1024 ready 4.034 4.452
32 7e9c5099-2f5e-4ea9-a6e2-d4acc540901f 1024 ready 7.946 8.362
33 927b907a-fbc4-4383-a6e3-0fd02206f79c 1024 ready 6.098 6.51
34 d96167a6-cda0-46a8-adad-df52ccc789bd 1024 ready 9.709 10.121
35 e9577c3a-8d88-46c9-9ad5-1979469b38be 1024 ready 0.843 1.356
36 52f8e432-ff40-4419-8968-17ef1089a361 1024 ready 4.966 5.376
37 99a1e17e-94b0-4ca1-810f-fa5e3ff9691f 1024 ready 3.31 3.726
38 33eb7872-0980-4336-a560-cf6f14a302f9 1024 ready 9.297 9.709
39 4a3cacba-350f-4878-85f7-ead76d7cacbe 1024 ready 3.726 4.137
40 fc3cb344-8f7c-4421-bae0-ce5e282d6143 1024 ready 1.975 2.39
41 22228d29-952b-4ec0-aad0-b8762967dee0 1024 ready 1.356 1.768
42 1d29e426-37fa-41e4-b5cc-55eedbf2af38 2048 ready 8.773 10.223
43 d62cc431-925c-47c6-9e0a-bec4c2d561dc 2048 ready 0.843 2.491
44 4d3d5437-fded-4c5a-9ab5-50460df9fade 2048 ready 4.452 5.893
45 78668df5-05d3-4ca1-b309-fcb16181d973 2048 ready 4.241 5.683
46 86af36aa-2193-4da2-a786-1787fb6229d5 2048 ready 2.39 3.829
47 211fbe0b-a085-4549-812a-07a2c326a9b2 2048 ready 6.51 8.049
48 f8e04547-8aaa-4b87-9e9d-383e387ade00 2048 ready 0.843 2.491
49 9b065c78-8c00-4fe5-a890-bfa698c8bfea 2048 ready 4.241 5.683
50 3906bf23-8afc-47d1-85c5-b2d0eb207236 2048 ready 10.223 11.673
51 4a2f285d-327c-44be-ba2c-da0d627109cc 2048 ready 6.406 7.946
52 f1f72a39-92e6-4284-a373-30214f1141bc 2048 ready 10.223 11.673
53 9eee27d0-b3fb-4be4-ab0f-3f99b0d024f1 2048 ready 6.098 7.741
54 cb946763-c80a-4284-85b3-3aea16d2eced 2048 ready 10.223 11.673
55 f0aed629-1d01-4f7a-a699-eff85d4dd239 2048 ready 7.844 9.297
56 b7957cd4-0942-40cd-a1f9-a4a614d4c5f3 2048 ready 8.773 10.223
57 ea48ce0d-a3e8-44f3-bdbc-37d1ddfc9eb0 2048 ready 7.844 9.297
58 e6a26bb8-4c70-4074-b130-357a742c6b5f 2048 ready 1.768 3.31
59 1530b7a5-227c-4bb0-ad09-d975105a757c 2048 ready 3.412 4.864
60 e9bcf42c-2a3b-43ee-8802-3d1432bfba5a 2048 ready 6.303 7.844
61 390b5116-460d-45f6-976b-240c2b6066bf 2048 ready 2.491 3.93
+61
View File
@@ -0,0 +1,61 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
00eb6bd3-3102-44e1-9be4-a5194b56cbd5,512,ready,2.814,3.021
cde0e2c2-02c2-4c05-bf05-6a98d0acd71a,512,ready,17.363,17.464
62fe77fe-bdf2-4dbb-85c7-9ff6981bc5f5,512,ready,16.524,16.626
36b35f4e-5497-4165-b0f6-bb3724e9cd00,512,ready,11.442,11.544
7f4286a2-d91e-40fb-8231-bd7674cfa8bc,512,ready,7.172,7.274
f34c5f28-536f-45b6-abf3-88a8229dd945,512,ready,3.021,3.123
1caa346a-80e8-44a7-9dfb-1e83bf71534e,512,ready,11.337,11.442
cf10641f-fbb8-43ea-88e0-22daa1627188,512,ready,2.095,2.198
98099d7b-724d-4e70-a4cf-9a6ab019333b,512,ready,12.906,13.111
b26e4907-1919-4dce-a7da-6909890b2d5f,512,ready,3.854,3.958
e3468d1b-5637-4e5a-aa70-0249559b4cfd,512,ready,2.712,2.814
2f9f2e4a-d4e9-47d6-8c87-25043e5199ae,512,ready,1.89,2.095
de8fbd2e-e392-4663-98a1-96d93542280f,512,ready,11.442,11.544
9d3f5e09-c41f-4018-bdab-175b00a452e6,512,ready,7.274,7.484
e56365eb-2d10-4d3c-ad32-931f8a31d030,512,ready,2.608,2.712
4bc57fb6-32c7-47ca-8899-ea40df7da799,512,ready,11.134,11.337
d2522db6-f17a-4e18-9d79-386a195fd633,512,ready,3.539,3.641
6d1cc5f9-d992-44cf-9ed0-530467774337,512,ready,4.368,4.572
985e8927-4f0b-44fa-bd09-fb19d6ea29fe,512,ready,11.235,11.442
50c58946-727b-48d2-9809-3f7a390ea741,512,ready,3.641,3.854
39ee9ce0-1494-4f3a-ab1f-7c03dcc546b2,1024,ready,1.058,1.578
9778715e-f4c6-4fa0-8c3c-64e621611fda,1024,ready,3.641,4.059
72ba59cb-3e56-483b-b65d-c53e0cf7c302,1024,ready,11.544,12.065
cb97fe45-a6b2-43c8-bd1b-b51824014994,1024,ready,4.572,4.885
df00bd9f-2470-4390-ace0-920e3f206b97,1024,ready,10.723,11.134
9694c476-cb8a-46b8-9e8d-57cba2152a17,1024,ready,14.336,14.745
42169eb9-3324-4c2a-99ca-36148faaa150,1024,ready,13.111,13.519
6c26553f-30ba-4f18-9c3c-d08472afc7f0,1024,ready,18.499,18.913
8315304d-7550-4eb9-82d8-0db4db41aa04,1024,ready,3.123,3.539
d8ee601e-6b75-417c-affa-1b24b1c1a02a,1024,ready,6.76,7.172
e4536000-5159-486f-97a3-8e0727854ab2,1024,ready,6.341,6.76
77809b9b-ae97-41b3-9463-cbe70acf2a0c,1024,ready,13.519,13.926
757b621c-fb42-499d-b4dd-721d9a9e876a,1024,ready,1.578,1.89
18ba2c34-56eb-46ec-a54b-20b2d2752ac5,1024,ready,12.065,12.485
75ebd991-3525-459a-a1dd-c71c6e78663b,1024,ready,13.926,14.336
5b5a607f-76b6-48f7-ba9e-09d9aa732cf4,1024,ready,3.958,4.368
813b7fe0-f445-4420-a8ab-757046d266b6,1024,ready,12.485,12.906
19fb3f65-760e-4e53-bd98-dc51dcbf237f,1024,ready,16.733,17.056
1ff822e0-69ae-461e-9df3-c27261f9009a,1024,ready,7.484,7.901
b24dbee4-e1c5-4a18-8f62-3f5012559cd4,1024,ready,16.103,16.524
baa5cea0-5554-4a32-90a6-e3ad6c204228,2048,ready,8.428,9.891
c58ddcd7-ffa3-46f6-b3dc-ea8f0fe9c1f4,2048,ready,18.913,20.363
c4636200-eae1-49d4-8e59-f5dfa5b86c36,2048,ready,9.891,11.235
c5d2e767-ce88-4682-82f5-80b9ab209580,2048,ready,4.059,5.504
bf65fcbd-3ec1-4851-ab11-7f9239cdf1e5,2048,ready,7.901,9.268
04533086-f63d-4aa8-9926-fab91e5ebf52,2048,ready,15.894,17.363
7d88df73-ae6b-4e4a-821e-3de54784e87c,2048,ready,2.198,3.641
06bc64bf-f949-48a1-a0d4-61d8e2053523,2048,ready,14.745,16.103
7751ba6d-1bd3-48a1-91dd-6f9d837e37f2,2048,ready,4.885,6.341
d3a36f16-3a76-43a4-9bc3-b806fb264ae1,2048,ready,14.541,15.894
e9daef68-894e-4433-9004-8e515f539203,2048,ready,9.268,10.723
f1afb046-e7d2-46e3-a639-bb9abd841039,2048,ready,20.363,21.721
29d51f63-9f4f-4f80-b493-7c55823c6015,2048,ready,13.111,14.541
0770bd79-18dc-4c61-8daf-5eddebac520f,2048,ready,1.058,2.608
eb6b54b5-c1ab-4f7a-a527-78dcb2c5cbac,2048,ready,5.504,6.964
09f1c7ae-0050-437e-ad85-4a416b813010,2048,ready,6.964,8.428
939d499e-3d5e-4e86-8e08-19dc70d73b18,2048,ready,11.544,13.111
64942f86-b40b-4aa5-99f3-b64e2617ddfc,2048,ready,18.913,20.363
37210ed4-847c-48f1-be14-924231466055,2048,ready,17.464,18.913
e1506996-5a46-42af-8083-dd815d6c2050,2048,ready,17.056,18.499
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 00eb6bd3-3102-44e1-9be4-a5194b56cbd5 512 ready 2.814 3.021
3 cde0e2c2-02c2-4c05-bf05-6a98d0acd71a 512 ready 17.363 17.464
4 62fe77fe-bdf2-4dbb-85c7-9ff6981bc5f5 512 ready 16.524 16.626
5 36b35f4e-5497-4165-b0f6-bb3724e9cd00 512 ready 11.442 11.544
6 7f4286a2-d91e-40fb-8231-bd7674cfa8bc 512 ready 7.172 7.274
7 f34c5f28-536f-45b6-abf3-88a8229dd945 512 ready 3.021 3.123
8 1caa346a-80e8-44a7-9dfb-1e83bf71534e 512 ready 11.337 11.442
9 cf10641f-fbb8-43ea-88e0-22daa1627188 512 ready 2.095 2.198
10 98099d7b-724d-4e70-a4cf-9a6ab019333b 512 ready 12.906 13.111
11 b26e4907-1919-4dce-a7da-6909890b2d5f 512 ready 3.854 3.958
12 e3468d1b-5637-4e5a-aa70-0249559b4cfd 512 ready 2.712 2.814
13 2f9f2e4a-d4e9-47d6-8c87-25043e5199ae 512 ready 1.89 2.095
14 de8fbd2e-e392-4663-98a1-96d93542280f 512 ready 11.442 11.544
15 9d3f5e09-c41f-4018-bdab-175b00a452e6 512 ready 7.274 7.484
16 e56365eb-2d10-4d3c-ad32-931f8a31d030 512 ready 2.608 2.712
17 4bc57fb6-32c7-47ca-8899-ea40df7da799 512 ready 11.134 11.337
18 d2522db6-f17a-4e18-9d79-386a195fd633 512 ready 3.539 3.641
19 6d1cc5f9-d992-44cf-9ed0-530467774337 512 ready 4.368 4.572
20 985e8927-4f0b-44fa-bd09-fb19d6ea29fe 512 ready 11.235 11.442
21 50c58946-727b-48d2-9809-3f7a390ea741 512 ready 3.641 3.854
22 39ee9ce0-1494-4f3a-ab1f-7c03dcc546b2 1024 ready 1.058 1.578
23 9778715e-f4c6-4fa0-8c3c-64e621611fda 1024 ready 3.641 4.059
24 72ba59cb-3e56-483b-b65d-c53e0cf7c302 1024 ready 11.544 12.065
25 cb97fe45-a6b2-43c8-bd1b-b51824014994 1024 ready 4.572 4.885
26 df00bd9f-2470-4390-ace0-920e3f206b97 1024 ready 10.723 11.134
27 9694c476-cb8a-46b8-9e8d-57cba2152a17 1024 ready 14.336 14.745
28 42169eb9-3324-4c2a-99ca-36148faaa150 1024 ready 13.111 13.519
29 6c26553f-30ba-4f18-9c3c-d08472afc7f0 1024 ready 18.499 18.913
30 8315304d-7550-4eb9-82d8-0db4db41aa04 1024 ready 3.123 3.539
31 d8ee601e-6b75-417c-affa-1b24b1c1a02a 1024 ready 6.76 7.172
32 e4536000-5159-486f-97a3-8e0727854ab2 1024 ready 6.341 6.76
33 77809b9b-ae97-41b3-9463-cbe70acf2a0c 1024 ready 13.519 13.926
34 757b621c-fb42-499d-b4dd-721d9a9e876a 1024 ready 1.578 1.89
35 18ba2c34-56eb-46ec-a54b-20b2d2752ac5 1024 ready 12.065 12.485
36 75ebd991-3525-459a-a1dd-c71c6e78663b 1024 ready 13.926 14.336
37 5b5a607f-76b6-48f7-ba9e-09d9aa732cf4 1024 ready 3.958 4.368
38 813b7fe0-f445-4420-a8ab-757046d266b6 1024 ready 12.485 12.906
39 19fb3f65-760e-4e53-bd98-dc51dcbf237f 1024 ready 16.733 17.056
40 1ff822e0-69ae-461e-9df3-c27261f9009a 1024 ready 7.484 7.901
41 b24dbee4-e1c5-4a18-8f62-3f5012559cd4 1024 ready 16.103 16.524
42 baa5cea0-5554-4a32-90a6-e3ad6c204228 2048 ready 8.428 9.891
43 c58ddcd7-ffa3-46f6-b3dc-ea8f0fe9c1f4 2048 ready 18.913 20.363
44 c4636200-eae1-49d4-8e59-f5dfa5b86c36 2048 ready 9.891 11.235
45 c5d2e767-ce88-4682-82f5-80b9ab209580 2048 ready 4.059 5.504
46 bf65fcbd-3ec1-4851-ab11-7f9239cdf1e5 2048 ready 7.901 9.268
47 04533086-f63d-4aa8-9926-fab91e5ebf52 2048 ready 15.894 17.363
48 7d88df73-ae6b-4e4a-821e-3de54784e87c 2048 ready 2.198 3.641
49 06bc64bf-f949-48a1-a0d4-61d8e2053523 2048 ready 14.745 16.103
50 7751ba6d-1bd3-48a1-91dd-6f9d837e37f2 2048 ready 4.885 6.341
51 d3a36f16-3a76-43a4-9bc3-b806fb264ae1 2048 ready 14.541 15.894
52 e9daef68-894e-4433-9004-8e515f539203 2048 ready 9.268 10.723
53 f1afb046-e7d2-46e3-a639-bb9abd841039 2048 ready 20.363 21.721
54 29d51f63-9f4f-4f80-b493-7c55823c6015 2048 ready 13.111 14.541
55 0770bd79-18dc-4c61-8daf-5eddebac520f 2048 ready 1.058 2.608
56 eb6b54b5-c1ab-4f7a-a527-78dcb2c5cbac 2048 ready 5.504 6.964
57 09f1c7ae-0050-437e-ad85-4a416b813010 2048 ready 6.964 8.428
58 939d499e-3d5e-4e86-8e08-19dc70d73b18 2048 ready 11.544 13.111
59 64942f86-b40b-4aa5-99f3-b64e2617ddfc 2048 ready 18.913 20.363
60 37210ed4-847c-48f1-be14-924231466055 2048 ready 17.464 18.913
61 e1506996-5a46-42af-8083-dd815d6c2050 2048 ready 17.056 18.499
+61
View File
@@ -0,0 +1,61 @@
attachment_id,image_size,status,queue_lag_estimate_s,time_to_ready_s
6d0948d1-018f-4649-bc93-3839a19d960b,512,ready,32.346,32.448
821f8778-92d6-4570-a1f3-f2d0bc20a4bc,512,ready,7.948,8.055
2c44d62f-bbaf-4443-95ae-9025bd84f48d,512,ready,11.024,11.132
a5bbe073-3027-42ae-98a4-2e93451a7895,512,ready,10.819,11.024
c6314a8d-0a91-4a9d-a466-ae1959418c44,512,ready,20.047,20.151
93fe6aee-0a67-4186-b5de-a7c8911c9c92,512,ready,4.143,4.347
0ca9a2cc-d750-4f9f-96ac-e1c26b14c751,512,ready,30.364,30.573
58b114fd-2289-405a-b1b1-aeed97690be3,512,ready,14.81,15.014
38d4dc82-c0af-4a2e-ba4d-5e83c7d39bb6,512,ready,15.014,15.118
f4b6d8b6-0abf-4de9-92bc-79427df63c1b,512,ready,15.731,15.937
53992b21-20c3-434d-b4c3-4375f50a2d5a,512,ready,20.668,20.873
65e871d7-7b80-43e2-84a2-01c77dc4b0ef,512,ready,15.527,15.629
0284edbb-fbde-481f-971f-ee483f41da53,512,ready,36.524,36.738
8244a394-feef-493a-acad-784ee2f63985,512,ready,18.098,18.2
289ecaf9-2ea5-42ed-9757-aaf20eace246,512,ready,38.816,39.023
10863feb-0bdf-426c-8827-e779e397a2c9,512,ready,15.629,15.731
9fa71750-b870-47e3-b95f-f814fcc2ec49,512,ready,33.289,33.391
abac43bc-c866-4c99-bd32-7e7b91ee54ef,512,ready,20.563,20.668
d6072440-3839-420d-b74d-98683f8ed80a,512,ready,3.628,3.732
5c0b5e43-e8df-4e6c-8774-df2698340e00,512,ready,14.708,14.81
e4cb92cf-7f15-4598-b8db-e2bb195cf897,1024,ready,30.573,30.99
09390dfc-b29a-4746-8353-7355b93cd34e,1024,ready,39.023,39.336
65464b2b-1e23-464b-86c5-f506248392a9,1024,ready,28.19,28.608
6326723a-8739-4afc-91b1-c5f86dcc883d,1024,ready,24.988,25.399
c0528510-a4e8-4b6b-a5fd-a8510730a91c,1024,ready,18.2,18.614
353e643e-0d25-4ee5-a18d-9101a32049f2,1024,ready,20.151,20.563
9d6d2329-d9bc-4299-a1ab-d6b27881457c,1024,ready,32.872,33.289
b68909e6-f9f2-466f-9938-9e8fd125758f,1024,ready,7.539,7.948
df4a4574-25ae-4eca-b967-0bd8f5a85aea,1024,ready,17.268,17.689
2e704f53-ae9c-4948-bd09-b8110f935baf,1024,ready,33.391,33.806
1a17e811-2645-49aa-8ad9-11e7d499b999,1024,ready,4.347,4.655
b1f5e98c-6587-4fdb-aee9-ccb8c6cd6f99,1024,ready,28.608,29.021
b7abcbbc-3ed9-4e85-b5e1-5d90b132034d,1024,ready,15.118,15.527
3e3a19a7-5485-45f4-a8a4-387f9002e54c,1024,ready,3.732,4.143
73aad940-7807-4f21-a83e-8ba1cd15863e,1024,ready,36.738,37.048
015511fd-3538-47d9-b7a0-f14969501967,1024,ready,38.509,38.816
cd33c6d5-6913-4e21-a36c-9f13ca3c7bce,1024,ready,12.46,12.869
2a2915a9-a211-4ac3-bdbf-a06b6bccbb04,1024,ready,17.689,18.098
9084e41a-d7cb-450b-8efb-0110b39d20a8,1024,ready,32.448,32.872
47d491a2-8f43-4b09-86be-939ed73d3216,1024,ready,12.869,13.277
ac499ba7-af23-4626-8386-18dd34b7d0cb,2048,ready,15.937,17.268
361ac439-f549-41ce-ba31-f0fdcc3e5a9d,2048,ready,6.097,7.539
f4831394-fdbe-421e-99b1-3ebb9d635f7a,2048,ready,8.055,9.487
e7c14f44-5b76-4950-a77c-a2d8edcd14a1,2048,ready,18.614,20.047
d40b2c16-d32c-4244-94f5-255d77d0d189,2048,ready,37.048,38.509
c61f7efe-e5e4-49fd-ab9d-b4d813d42697,2048,ready,35.169,36.524
3b1bfd22-01e8-4874-8ea2-7716785f88e1,2048,ready,4.655,6.097
21fa4d34-4dcc-487c-9205-ab1e48d05d09,2048,ready,22.221,23.658
b169f1bf-5ede-447e-aa4d-c7886b80e6a8,2048,ready,33.806,35.169
2c9a01c7-e488-4ca6-bc70-19e7b20d91a6,2048,ready,13.277,14.708
8f439608-d82f-4050-b939-5c35d9938eb2,2048,ready,0.735,2.179
fe635dd1-5fc7-4c82-a63a-0819dcac246c,2048,ready,23.658,24.988
1be24fbc-bf40-4883-9029-6075d2f4c557,2048,ready,30.99,32.346
ba19044d-d21c-4706-a263-240fdafb5749,2048,ready,26.843,28.19
52e7cc24-f8a3-443b-b62d-93c143ce75a7,2048,ready,2.179,3.628
97f34f85-f51f-4c8c-8f27-ae573690f1c8,2048,ready,29.021,30.364
4228d8dd-5da4-4532-a132-9b27308cc1e7,2048,ready,25.399,26.843
d58514f2-17f8-481e-b5c5-ae054324f80e,2048,ready,20.873,22.221
51f3190e-f63b-4b0e-880c-947089cdfef9,2048,ready,9.487,10.819
e5de7654-15d1-4f47-9f22-d2cab4c76737,2048,ready,11.132,12.46
1 attachment_id image_size status queue_lag_estimate_s time_to_ready_s
2 6d0948d1-018f-4649-bc93-3839a19d960b 512 ready 32.346 32.448
3 821f8778-92d6-4570-a1f3-f2d0bc20a4bc 512 ready 7.948 8.055
4 2c44d62f-bbaf-4443-95ae-9025bd84f48d 512 ready 11.024 11.132
5 a5bbe073-3027-42ae-98a4-2e93451a7895 512 ready 10.819 11.024
6 c6314a8d-0a91-4a9d-a466-ae1959418c44 512 ready 20.047 20.151
7 93fe6aee-0a67-4186-b5de-a7c8911c9c92 512 ready 4.143 4.347
8 0ca9a2cc-d750-4f9f-96ac-e1c26b14c751 512 ready 30.364 30.573
9 58b114fd-2289-405a-b1b1-aeed97690be3 512 ready 14.81 15.014
10 38d4dc82-c0af-4a2e-ba4d-5e83c7d39bb6 512 ready 15.014 15.118
11 f4b6d8b6-0abf-4de9-92bc-79427df63c1b 512 ready 15.731 15.937
12 53992b21-20c3-434d-b4c3-4375f50a2d5a 512 ready 20.668 20.873
13 65e871d7-7b80-43e2-84a2-01c77dc4b0ef 512 ready 15.527 15.629
14 0284edbb-fbde-481f-971f-ee483f41da53 512 ready 36.524 36.738
15 8244a394-feef-493a-acad-784ee2f63985 512 ready 18.098 18.2
16 289ecaf9-2ea5-42ed-9757-aaf20eace246 512 ready 38.816 39.023
17 10863feb-0bdf-426c-8827-e779e397a2c9 512 ready 15.629 15.731
18 9fa71750-b870-47e3-b95f-f814fcc2ec49 512 ready 33.289 33.391
19 abac43bc-c866-4c99-bd32-7e7b91ee54ef 512 ready 20.563 20.668
20 d6072440-3839-420d-b74d-98683f8ed80a 512 ready 3.628 3.732
21 5c0b5e43-e8df-4e6c-8774-df2698340e00 512 ready 14.708 14.81
22 e4cb92cf-7f15-4598-b8db-e2bb195cf897 1024 ready 30.573 30.99
23 09390dfc-b29a-4746-8353-7355b93cd34e 1024 ready 39.023 39.336
24 65464b2b-1e23-464b-86c5-f506248392a9 1024 ready 28.19 28.608
25 6326723a-8739-4afc-91b1-c5f86dcc883d 1024 ready 24.988 25.399
26 c0528510-a4e8-4b6b-a5fd-a8510730a91c 1024 ready 18.2 18.614
27 353e643e-0d25-4ee5-a18d-9101a32049f2 1024 ready 20.151 20.563
28 9d6d2329-d9bc-4299-a1ab-d6b27881457c 1024 ready 32.872 33.289
29 b68909e6-f9f2-466f-9938-9e8fd125758f 1024 ready 7.539 7.948
30 df4a4574-25ae-4eca-b967-0bd8f5a85aea 1024 ready 17.268 17.689
31 2e704f53-ae9c-4948-bd09-b8110f935baf 1024 ready 33.391 33.806
32 1a17e811-2645-49aa-8ad9-11e7d499b999 1024 ready 4.347 4.655
33 b1f5e98c-6587-4fdb-aee9-ccb8c6cd6f99 1024 ready 28.608 29.021
34 b7abcbbc-3ed9-4e85-b5e1-5d90b132034d 1024 ready 15.118 15.527
35 3e3a19a7-5485-45f4-a8a4-387f9002e54c 1024 ready 3.732 4.143
36 73aad940-7807-4f21-a83e-8ba1cd15863e 1024 ready 36.738 37.048
37 015511fd-3538-47d9-b7a0-f14969501967 1024 ready 38.509 38.816
38 cd33c6d5-6913-4e21-a36c-9f13ca3c7bce 1024 ready 12.46 12.869
39 2a2915a9-a211-4ac3-bdbf-a06b6bccbb04 1024 ready 17.689 18.098
40 9084e41a-d7cb-450b-8efb-0110b39d20a8 1024 ready 32.448 32.872
41 47d491a2-8f43-4b09-86be-939ed73d3216 1024 ready 12.869 13.277
42 ac499ba7-af23-4626-8386-18dd34b7d0cb 2048 ready 15.937 17.268
43 361ac439-f549-41ce-ba31-f0fdcc3e5a9d 2048 ready 6.097 7.539
44 f4831394-fdbe-421e-99b1-3ebb9d635f7a 2048 ready 8.055 9.487
45 e7c14f44-5b76-4950-a77c-a2d8edcd14a1 2048 ready 18.614 20.047
46 d40b2c16-d32c-4244-94f5-255d77d0d189 2048 ready 37.048 38.509
47 c61f7efe-e5e4-49fd-ab9d-b4d813d42697 2048 ready 35.169 36.524
48 3b1bfd22-01e8-4874-8ea2-7716785f88e1 2048 ready 4.655 6.097
49 21fa4d34-4dcc-487c-9205-ab1e48d05d09 2048 ready 22.221 23.658
50 b169f1bf-5ede-447e-aa4d-c7886b80e6a8 2048 ready 33.806 35.169
51 2c9a01c7-e488-4ca6-bc70-19e7b20d91a6 2048 ready 13.277 14.708
52 8f439608-d82f-4050-b939-5c35d9938eb2 2048 ready 0.735 2.179
53 fe635dd1-5fc7-4c82-a63a-0819dcac246c 2048 ready 23.658 24.988
54 1be24fbc-bf40-4883-9029-6075d2f4c557 2048 ready 30.99 32.346
55 ba19044d-d21c-4706-a263-240fdafb5749 2048 ready 26.843 28.19
56 52e7cc24-f8a3-443b-b62d-93c143ce75a7 2048 ready 2.179 3.628
57 97f34f85-f51f-4c8c-8f27-ae573690f1c8 2048 ready 29.021 30.364
58 4228d8dd-5da4-4532-a132-9b27308cc1e7 2048 ready 25.399 26.843
59 d58514f2-17f8-481e-b5c5-ae054324f80e 2048 ready 20.873 22.221
60 51f3190e-f63b-4b0e-880c-947089cdfef9 2048 ready 9.487 10.819
61 e5de7654-15d1-4f47-9f22-d2cab4c76737 2048 ready 11.132 12.46
+74
View File
@@ -0,0 +1,74 @@
elapsed_s,rss_mb
0.0018562499899417162,63.9609375
0.5586906669195741,63.9609375
1.1478328749071807,63.9609375
1.7124463329091668,63.9609375
2.3201119999866933,154.03125
2.886150083038956,181.53125
3.454367541940883,208.90625
4.0352518749423325,172.203125
4.596314958063886,199.328125
5.158594999928027,94.390625
5.731980707962066,90.484375
6.294250082923099,160.44921875
6.866898791864514,187.32421875
7.449180916883051,214.32421875
8.02075345790945,175.76953125
8.586141041945666,203.89453125
9.151447457959875,111.85546875
9.7154242079705,164.8046875
10.274698958033696,192.0546875
10.821836207993329,214.4296875
11.391208708053455,178.421875
11.957092249998823,206.921875
12.530426000012085,95.53515625
13.091126542072743,180.48828125
13.658847207902,208.73828125
14.223303207894787,115.96875
14.79374916688539,122.56640625
15.352951207896695,183.80078125
15.913661292055622,212.42578125
16.488913499983028,94.2734375
17.04698033304885,89.0625
17.61788404197432,166.3125
18.182629582937807,194.5625
18.746452332939953,73.2265625
19.316664916928858,110.76953125
19.88158891699277,112.84765625
20.447920500067994,173.1171875
21.019828042015433,200.3671875
21.583593416959047,95.69921875
22.148048375034705,68.84375
22.701521083014086,174.2734375
23.266354499850422,202.7734375
23.836700625019148,161.3046875
24.401869332883507,188.8046875
24.96454008296132,214.8046875
25.525170125067234,175.265625
26.084088041912764,203.515625
26.649612499866635,111.0
27.214416875038296,170.390625
27.77859991695732,198.640625
28.34497074992396,122.6796875
28.900013708043844,184.03515625
29.459034791914746,212.28515625
30.019986792001873,106.3125
30.584227999905124,160.7734375
31.149346332997084,187.8984375
31.712559124920517,214.6484375
32.281295999884605,116.64453125
32.84523691702634,176.890625
33.407698791939765,204.765625
33.96293962490745,104.8046875
34.52196699986234,113.38671875
35.08372745802626,116.0078125
35.665793374879286,176.3828125
36.23321974999271,204.6328125
36.79392512491904,162.4609375
37.351651957957074,190.7109375
37.91607549996115,214.7109375
38.483901750063524,118.4921875
39.04901108285412,178.9453125
39.61435758299194,207.6953125
40.18525333306752,116.02734375
40.75472962507047,117.7734375
1 elapsed_s rss_mb
2 0.0018562499899417162 63.9609375
3 0.5586906669195741 63.9609375
4 1.1478328749071807 63.9609375
5 1.7124463329091668 63.9609375
6 2.3201119999866933 154.03125
7 2.886150083038956 181.53125
8 3.454367541940883 208.90625
9 4.0352518749423325 172.203125
10 4.596314958063886 199.328125
11 5.158594999928027 94.390625
12 5.731980707962066 90.484375
13 6.294250082923099 160.44921875
14 6.866898791864514 187.32421875
15 7.449180916883051 214.32421875
16 8.02075345790945 175.76953125
17 8.586141041945666 203.89453125
18 9.151447457959875 111.85546875
19 9.7154242079705 164.8046875
20 10.274698958033696 192.0546875
21 10.821836207993329 214.4296875
22 11.391208708053455 178.421875
23 11.957092249998823 206.921875
24 12.530426000012085 95.53515625
25 13.091126542072743 180.48828125
26 13.658847207902 208.73828125
27 14.223303207894787 115.96875
28 14.79374916688539 122.56640625
29 15.352951207896695 183.80078125
30 15.913661292055622 212.42578125
31 16.488913499983028 94.2734375
32 17.04698033304885 89.0625
33 17.61788404197432 166.3125
34 18.182629582937807 194.5625
35 18.746452332939953 73.2265625
36 19.316664916928858 110.76953125
37 19.88158891699277 112.84765625
38 20.447920500067994 173.1171875
39 21.019828042015433 200.3671875
40 21.583593416959047 95.69921875
41 22.148048375034705 68.84375
42 22.701521083014086 174.2734375
43 23.266354499850422 202.7734375
44 23.836700625019148 161.3046875
45 24.401869332883507 188.8046875
46 24.96454008296132 214.8046875
47 25.525170125067234 175.265625
48 26.084088041912764 203.515625
49 26.649612499866635 111.0
50 27.214416875038296 170.390625
51 27.77859991695732 198.640625
52 28.34497074992396 122.6796875
53 28.900013708043844 184.03515625
54 29.459034791914746 212.28515625
55 30.019986792001873 106.3125
56 30.584227999905124 160.7734375
57 31.149346332997084 187.8984375
58 31.712559124920517 214.6484375
59 32.281295999884605 116.64453125
60 32.84523691702634 176.890625
61 33.407698791939765 204.765625
62 33.96293962490745 104.8046875
63 34.52196699986234 113.38671875
64 35.08372745802626 116.0078125
65 35.665793374879286 176.3828125
66 36.23321974999271 204.6328125
67 36.79392512491904 162.4609375
68 37.351651957957074 190.7109375
69 37.91607549996115 214.7109375
70 38.483901750063524 118.4921875
71 39.04901108285412 178.9453125
72 39.61435758299194 207.6953125
73 40.18525333306752 116.02734375
74 40.75472962507047 117.7734375
+43
View File
@@ -0,0 +1,43 @@
elapsed_s,rss_mb
0.006153792142868042,63.90234375
0.5778734171763062,63.90234375
1.147869042120874,63.90234375
1.7034579580649734,63.90234375
2.276205125031993,63.90234375
2.8752599169965833,194.703125
3.4416670829523355,230.83203125
3.9975552500691265,302.11328125
4.574312333017588,213.53515625
5.14147137501277,263.47265625
5.7091500831302255,110.90625
6.2797843751031905,220.01171875
6.835572292096913,304.51953125
7.403642917051911,289.5234375
7.963550708023831,196.44921875
8.53717741696164,258.8515625
9.096077542053536,186.86328125
9.66255066706799,292.37890625
10.221317625138909,277.51171875
10.789583917008713,333.26171875
11.363086167024449,317.81640625
11.92844095802866,303.1171875
12.479885042179376,243.38671875
13.050684583140537,94.1015625
13.61427737516351,182.80078125
14.231398458126932,240.96484375
14.796835124958307,197.81640625
15.373798333108425,231.953125
15.93250362505205,221.671875
16.498262125067413,263.9453125
17.066447000019252,319.6953125
17.630022125085816,269.69140625
18.18958625011146,206.35546875
18.755617167102173,245.66015625
19.313573125051335,277.03125
19.88116762507707,332.53125
20.441875083139166,269.94140625
21.01258633309044,281.08984375
21.573833167087287,337.21484375
22.150734583148733,166.046875
22.70690429210663,193.421875
23.275886042043567,218.296875
1 elapsed_s rss_mb
2 0.006153792142868042 63.90234375
3 0.5778734171763062 63.90234375
4 1.147869042120874 63.90234375
5 1.7034579580649734 63.90234375
6 2.276205125031993 63.90234375
7 2.8752599169965833 194.703125
8 3.4416670829523355 230.83203125
9 3.9975552500691265 302.11328125
10 4.574312333017588 213.53515625
11 5.14147137501277 263.47265625
12 5.7091500831302255 110.90625
13 6.2797843751031905 220.01171875
14 6.835572292096913 304.51953125
15 7.403642917051911 289.5234375
16 7.963550708023831 196.44921875
17 8.53717741696164 258.8515625
18 9.096077542053536 186.86328125
19 9.66255066706799 292.37890625
20 10.221317625138909 277.51171875
21 10.789583917008713 333.26171875
22 11.363086167024449 317.81640625
23 11.92844095802866 303.1171875
24 12.479885042179376 243.38671875
25 13.050684583140537 94.1015625
26 13.61427737516351 182.80078125
27 14.231398458126932 240.96484375
28 14.796835124958307 197.81640625
29 15.373798333108425 231.953125
30 15.93250362505205 221.671875
31 16.498262125067413 263.9453125
32 17.066447000019252 319.6953125
33 17.630022125085816 269.69140625
34 18.18958625011146 206.35546875
35 18.755617167102173 245.66015625
36 19.313573125051335 277.03125
37 19.88116762507707 332.53125
38 20.441875083139166 269.94140625
39 21.01258633309044 281.08984375
40 21.573833167087287 337.21484375
41 22.150734583148733 166.046875
42 22.70690429210663 193.421875
43 23.275886042043567 218.296875
+25
View File
@@ -0,0 +1,25 @@
elapsed_s,rss_mb
0.0017751660197973251,63.90234375
0.5725909580942243,63.90234375
1.146317790960893,63.90234375
1.712353958049789,63.90234375
2.2800673749297857,63.90234375
2.886254500132054,342.91015625
3.4468102499376982,361.43359375
4.018739791121334,518.78515625
4.595743082929403,414.22265625
5.1572650000452995,385.99609375
5.722510583000258,270.40625
6.291970041114837,495.10546875
6.863853875081986,489.14453125
7.440310999983922,251.58203125
7.994805958122015,232.55078125
8.561554083134979,507.96484375
9.145290333079174,536.33984375
9.71233612508513,399.1328125
10.283997416030616,374.50390625
10.863315333146602,531.59375
11.457262207986787,337.00390625
12.02924433303997,245.5234375
12.59735562512651,383.62109375
13.175934208091348,467.24609375
1 elapsed_s rss_mb
2 0.0017751660197973251 63.90234375
3 0.5725909580942243 63.90234375
4 1.146317790960893 63.90234375
5 1.712353958049789 63.90234375
6 2.2800673749297857 63.90234375
7 2.886254500132054 342.91015625
8 3.4468102499376982 361.43359375
9 4.018739791121334 518.78515625
10 4.595743082929403 414.22265625
11 5.1572650000452995 385.99609375
12 5.722510583000258 270.40625
13 6.291970041114837 495.10546875
14 6.863853875081986 489.14453125
15 7.440310999983922 251.58203125
16 7.994805958122015 232.55078125
17 8.561554083134979 507.96484375
18 9.145290333079174 536.33984375
19 9.71233612508513 399.1328125
20 10.283997416030616 374.50390625
21 10.863315333146602 531.59375
22 11.457262207986787 337.00390625
23 12.02924433303997 245.5234375
24 12.59735562512651 383.62109375
25 13.175934208091348 467.24609375
+4
View File
@@ -0,0 +1,4 @@
config,total,ready,failed,duration_s,throughput,peak_processing,peak_rss_mb,mean_rss_mb
1/1,60,60,0,39.34,1.53,1.0,214.8,155.83
2/2,60,60,0,21.72,2.76,2.0,337.21,224.02
4/4,60,60,0,11.67,5.14,4.0,536.34,327.69
1 config total ready failed duration_s throughput peak_processing peak_rss_mb mean_rss_mb
2 1/1 60 60 0 39.34 1.53 1.0 214.8 155.83
3 2/2 60 60 0 21.72 2.76 2.0 337.21 224.02
4 4/4 60 60 0 11.67 5.14 4.0 536.34 327.69
+297
View File
@@ -0,0 +1,297 @@
import csv
import re
import subprocess
import time
from pathlib import Path
CONFIGS = [(1, 1), (2, 2), (4, 4)]
COUNT_PER_SIZE = 20
RESULTS_DIR = Path("benchmark_results")
RESULTS_DIR.mkdir(exist_ok=True)
RSS_CODE = """
import os
total_kb = 0
current_pid = os.getpid()
for name in os.listdir("/proc"):
if not name.isdigit():
continue
if int(name) == current_pid:
continue
try:
with open(f"/proc/{name}/status") as file:
for line in file:
if line.startswith("VmRSS:"):
total_kb += int(line.split()[1])
break
except (OSError, ValueError):
pass
print(total_kb / 1024)
"""
def command(args, check=True):
return subprocess.run(
args,
text=True,
capture_output=True,
check=check,
)
def remove_test_worker():
command(
["docker", "rm", "-f", "benchmark-worker"],
check=False,
)
def start_worker(concurrency, prefetch):
remove_test_worker()
result = command([
"docker", "compose", "run",
"-d",
"--name", "benchmark-worker",
"--no-deps",
"-e", f"WORKER_CONCURRENCY={concurrency}",
"-e", f"WORKER_PREFETCH={prefetch}",
"worker",
])
expected = (
f"[Worker] CONCURRENCY={concurrency}, "
f"PREFETCH={prefetch}"
)
for _ in range(30):
logs = command(
["docker", "logs", "benchmark-worker"],
check=False,
)
if expected in logs.stdout + logs.stderr:
print(expected)
return
time.sleep(1)
raise RuntimeError(
f"Worker {concurrency}/{prefetch} не запустился.\n"
+ command(
["docker", "logs", "benchmark-worker"],
check=False,
).stderr
)
def measure_rss():
result = command(
[
"docker", "exec",
"benchmark-worker",
"python", "-c", RSS_CODE,
],
check=False,
)
if result.returncode != 0:
return None
try:
return float(result.stdout.strip())
except ValueError:
return None
def extract_number(text, label):
match = re.search(
rf"^{re.escape(label)}:\s*([0-9.]+)",
text,
re.MULTILINE,
)
return float(match.group(1)) if match else None
def run_benchmark(concurrency, prefetch):
config = f"{concurrency}_{prefetch}"
print(f"\n{'=' * 55}")
print(f"ЭКСПЕРИМЕНТ: {concurrency}/{prefetch}")
print(f"{'=' * 55}")
start_worker(concurrency, prefetch)
log_path = RESULTS_DIR / f"load_{config}.log"
rss_samples = []
load_command = [
"docker", "compose", "exec",
"-T",
"-w", "/tmp",
"-e", "PYTHONPATH=/app",
"app1",
"python", "-m", "app.load_test",
"--count", str(COUNT_PER_SIZE),
"--timeout", "300",
]
print("Запускаем 60 фотографий...")
start_time = time.monotonic()
with log_path.open("w") as log_file:
process = subprocess.Popen(
load_command,
stdout=log_file,
stderr=subprocess.STDOUT,
)
while process.poll() is None:
elapsed = time.monotonic() - start_time
rss = measure_rss()
if rss is not None:
rss_samples.append((elapsed, rss))
time.sleep(0.5)
return_code = process.wait()
output = log_path.read_text()
if return_code != 0:
print(output)
raise RuntimeError(
f"Нагрузочный тест {concurrency}/{prefetch} завершился с ошибкой"
)
print(output)
if not rss_samples:
raise RuntimeError("Не удалось измерить RSS")
rss_values = [value for _, value in rss_samples]
peak_rss = max(rss_values)
mean_rss = sum(rss_values) / len(rss_values)
samples_path = RESULTS_DIR / f"rss_{config}.csv"
with samples_path.open("w", newline="") as file:
writer = csv.writer(file)
writer.writerow(["elapsed_s", "rss_mb"])
writer.writerows(rss_samples)
csv_match = re.search(
r"CSV:\s*(benchmark_[a-zA-Z0-9]+\.csv)",
output,
)
if csv_match:
filename = csv_match.group(1)
copy_result = command(
[
"docker", "compose", "cp",
f"app1:/tmp/{filename}",
str(RESULTS_DIR / filename),
],
check=False,
)
if copy_result.returncode != 0:
print("Не удалось скопировать CSV:", copy_result.stderr)
result = {
"config": f"{concurrency}/{prefetch}",
"total": int(extract_number(output, "Total jobs") or 0),
"ready": int(extract_number(output, "Ready") or 0),
"failed": int(extract_number(output, "Failed") or 0),
"duration_s": extract_number(output, "Duration"),
"throughput": extract_number(output, "Throughput"),
"peak_processing": extract_number(
output, "Peak observed processing"
),
"peak_rss_mb": round(peak_rss, 2),
"mean_rss_mb": round(mean_rss, 2),
}
print(f"\n[RSS] Пиковое значение: {peak_rss:.2f} MB")
print(f"[RSS] Среднее значение: {mean_rss:.2f} MB")
print(f"[RSS] Количество замеров: {len(rss_samples)}")
return result
def save_summary(results):
output = RESULTS_DIR / "summary.csv"
with output.open("w", newline="") as file:
writer = csv.DictWriter(
file,
fieldnames=[
"config",
"total",
"ready",
"failed",
"duration_s",
"throughput",
"peak_processing",
"peak_rss_mb",
"mean_rss_mb",
],
)
writer.writeheader()
writer.writerows(results)
print("\n" + "=" * 70)
print("ИТОГОВАЯ ТАБЛИЦА")
print("=" * 70)
for item in results:
print(
f"{item['config']:5} | "
f"Ready: {item['ready']:3} | "
f"Time: {item['duration_s']} s | "
f"Throughput: {item['throughput']} img/s | "
f"Peak RSS: {item['peak_rss_mb']} MB"
)
print(f"\nРезультаты сохранены в {RESULTS_DIR}/")
def main():
results = []
try:
print("Останавливаем обычный Worker...")
command(["docker", "compose", "stop", "worker"])
for concurrency, prefetch in CONFIGS:
result = run_benchmark(concurrency, prefetch)
results.append(result)
finally:
print("\nВосстанавливаем обычный Worker...")
remove_test_worker()
restore = command(
["docker", "compose", "up", "-d", "worker"],
check=False,
)
if restore.returncode != 0:
print("Ошибка восстановления Worker:")
print(restore.stderr)
if results:
save_summary(results)
if __name__ == "__main__":
main()
+197
View File
@@ -0,0 +1,197 @@
services:
db:
image: postgres:16-alpine
container_name: mtusi_messenger_db
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
command: postgres -c synchronous_commit=off
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
redis:
image: redis:7-alpine
container_name: mtusi_messenger_redis
command: redis-server --appendonly yes
volumes:
- redis_data:/data
ports:
- "6379:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
app1:
build: .
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS}
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
depends_on:
db:
condition: service_healthy
redis:
condition: service_healthy
minio:
condition: service_healthy
volumes:
- ./app:/app/app
- ./alembic:/app/alembic
- ./alembic.ini:/app/alembic.ini
app2:
build: .
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_REGION: ${S3_REGION}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS}
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
depends_on:
db:
condition: service_healthy
redis:
condition: service_healthy
minio:
condition: service_healthy
volumes:
- ./app:/app/app
- ./alembic:/app/alembic
- ./alembic.ini:/app/alembic.ini
rabbitmq:
image: rabbitmq:3.12-management-alpine
container_name: mtusi_messenger_rabbitmq
environment:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
ports:
- "127.0.0.1:5672:5672"
- "127.0.0.1:15672:15672"
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: [ "CMD", "rabbitmq-diagnostics", "-q", "ping" ]
interval: 10s
timeout: 5s
retries: 5
worker:
build: .
mem_limit: 512m
command: python -m app.worker
environment:
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
S3_BUCKET: ${S3_BUCKET}
RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/
depends_on:
db:
condition: service_healthy
minio:
condition: service_healthy
rabbitmq:
condition: service_healthy
volumes:
- ./app:/app/app
- ./scripts:/app/scripts
outbox:
build: .
command: python -m app.outbox_dispatcher
restart: unless-stopped
environment:
DATABASE_URL: ${DATABASE_URL}
RABBITMQ_URL: amqp://guest:guest@rabbitmq:5672/
depends_on:
db:
condition: service_healthy
rabbitmq:
condition: service_healthy
volumes:
- ./app:/app/app
nginx:
image: nginx:alpine
container_name: mtusi_messenger_nginx
ports:
- "127.0.0.1:8080:80"
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
- app1
- app2
minio:
image: pgsty/silo:latest
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
volumes:
- minio_data:/data
ports:
- "127.0.0.1:9000:9000"
- "127.0.0.1:9001:9001"
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
interval: 5s
timeout: 5s
retries: 5
minio-init:
image: pgsty/mc:latest
depends_on:
minio:
condition: service_healthy
entrypoint: >
/bin/sh -c "
/usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin;
/usr/bin/mc mb --ignore-existing myminio/${S3_BUCKET};
/usr/bin/mc anonymous set none myminio/${S3_BUCKET};
/usr/bin/mc admin user add myminio api-user api-user-secret-2026 || true;
/usr/bin/mc admin policy attach myminio readwrite --user api-user || true;
exit 0;
"
volumes:
postgres_data:
redis_data:
minio_data:
rabbitmq_data:
+27
View File
@@ -0,0 +1,27 @@
events { worker_connections 1024; }
http {
upstream api_backend {
least_conn;
server app1:8080 max_fails=2 fail_timeout=5s;
server app2:8080 max_fails=2 fail_timeout=5s;
}
server {
listen 80;
client_max_body_size 10M;
location / {
proxy_pass http://api_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_connect_timeout 2s;
proxy_send_timeout 5s;
proxy_read_timeout 5s;
}
}
}
+17
View File
@@ -1,2 +1,19 @@
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
openapi-spec-validator==0.7.2
pytest==8.3.4
fastapi==0.115.0
uvicorn[standard]==0.52.4
pydantic==2.13.5
asyncpg==0.31.0
sqlalchemy[asyncio]==2.1.0
redis==6.2.0
passlib==1.7.4
bcrypt==4.1.2
alembic==1.14.0
psycopg2-binary
filetype==1.2.0
Pillow==10.4.0
minio==7.2.9
python-multipart==0.0.9
aio-pika==9.4.0
Pillow==10.4.0
View File
+14
View File
@@ -0,0 +1,14 @@
from pydantic import BaseModel, Field, field_validator, ConfigDict
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("member_ids")
@classmethod
def unique_members(cls, v):
if len(set(v)) != len(v):
raise ValueError("member_ids должны быть уникальными")
return v
+6
View File
@@ -0,0 +1,6 @@
from pydantic import BaseModel, Field, ConfigDict
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: str = Field(min_length=1, max_length=4000)
+7
View File
@@ -0,0 +1,7 @@
from pydantic import BaseModel, Field, ConfigDict
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
View File
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
REPLICA="${1:-app1}"
echo "Останавливаю $REPLICA"
date +"%H:%M:%S.%3N"
./scripts/stop-one.sh "$REPLICA"
echo ""
echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id"
for i in {1..20}; do
TS=$(date +"%H:%M:%S.%3N")
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live)
INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r')
echo "$TS code=$CODE $INST"
sleep 0.5
done
echo ""
echo "Поднимаю $REPLICA обратно"
docker compose start "$REPLICA"
+76
View File
@@ -0,0 +1,76 @@
import io
import os
import time
import uuid
import requests
from PIL import Image
BASE_URL = "http://localhost:8080"
COUNT = 50
def make_png(width, height):
img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
buf = io.BytesIO()
img.save(buf, format="PNG", optimize=False)
return buf.getvalue()
def register_and_login(username):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "Test", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def create_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "Cost Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def upload(token, chat_id, content, filename):
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
headers=headers, data=content, timeout=60,
)
return resp.status_code
def main():
print(f"Загружаем {COUNT} файлов (по ~1 МБ)...")
token = register_and_login("cost_tester")
chat_id = create_chat(token)
print(f"Chat ID: {chat_id}")
content = make_png(200, 200)
print(f"Размер файла: {len(content) / 1024:.1f} КБ")
success = 0
errors = 0
start = time.monotonic()
for i in range(COUNT):
status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png")
if status == 201:
success += 1
else:
errors += 1
if (i + 1) % 10 == 0:
print(f" Загружено: {i + 1}/{COUNT}")
elapsed = time.monotonic() - start
print(f"\n=== Результаты ===")
print(f"Успешно: {success}")
print(f"Ошибок: {errors}")
print(f"Время: {elapsed:.2f}s")
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
import matplotlib.pyplot as plt
import os
replicas = [1, 2, 4, 6]
rps = [250, 255, 260, 257]
plt.figure(figsize=(8, 5))
plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8)
plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14)
plt.xlabel('Количество API-реплик', fontsize=12)
plt.ylabel('RPS (запросов в секунду)', fontsize=12)
plt.grid(True, linestyle='--', alpha=0.7)
plt.xticks(replicas)
plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252),
arrowprops=dict(facecolor='red', shrink=0.05),
fontsize=10, color='red')
os.makedirs('evidence', exist_ok=True)
plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight')
print("График сохранен в evidence/scaling.png")
+90
View File
@@ -0,0 +1,90 @@
import io
import os
import time
import uuid
import requests
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
BASE_URL = "http://localhost:8080"
CONCURRENCY = 10
def make_png(width, height):
img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3))
buf = io.BytesIO()
img.save(buf, format='PNG', optimize=False)
return buf.getvalue()
def register_and_login(username):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "Test", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def create_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "RAM Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def upload(token, chat_id, content, filename):
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
start = time.monotonic()
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
headers=headers, data=content, timeout=60,
)
elapsed = time.monotonic() - start
return resp.status_code, elapsed, len(content)
def main():
print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each")
token = register_and_login("ram_tester")
chat_id = create_chat(token)
print(f"Chat ID: {chat_id}")
print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...")
contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)]
sizes = [len(c) / (1024 * 1024) for c in contents]
print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ")
print(f"Суммарно: {sum(sizes):.2f} МБ")
print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...")
start = time.monotonic()
results, errors = [], []
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
futures = [
pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png")
for i in range(CONCURRENCY)
]
for f in futures:
try:
results.append(f.result())
except Exception as e:
errors.append(str(e))
total = time.monotonic() - start
print(f"\n=== Результаты ===")
print(f"Всего: {len(results) + len(errors)}")
print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}")
print(f"Ошибок: {len(errors)}")
if results:
lat = [r[1] for r in results]
print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s")
print(f"Общее время: {total:.2f}s")
if errors:
print(f"Ошибки: {errors[:5]}")
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from minio import Minio
from sqlalchemy import select, cast, Text
from app.models.chat import Chat, ChatMember
from app.models.user import User
from app.models.message import Message
from app.database import async_session_maker
from app.models.attachment import Attachment
from app.config import (
S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
)
GRACE_PERIOD = timedelta(seconds=1)
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
minio_client = Minio(
endpoint,
access_key=S3_ACCESS_KEY_ID,
secret_key=S3_SECRET_ACCESS_KEY,
secure=False,
)
BUCKET = S3_BUCKET
async def find_orphan_s3_objects(dry_run: bool) -> int:
print("Поиск orphan объектов в S3, файлы без записи в БД")
async with async_session_maker() as session:
result = await session.execute(select(Attachment.object_key))
db_keys = {row[0] for row in result.all()}
print(f"Файлов в БД: {len(db_keys)}")
objects = list(minio_client.list_objects(BUCKET, recursive=True))
print(f"Объектов в MinIO: {len(objects)}")
now = datetime.now(timezone.utc)
orphans = []
for obj in objects:
if obj.object_name not in db_keys:
age = now - obj.last_modified if obj.last_modified else timedelta(0)
if age > GRACE_PERIOD:
orphans.append(obj.object_name)
print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)")
else:
print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)")
if not orphans:
print("Orphan-объектов не найдено.")
return 0
if dry_run:
print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.")
else:
for key in orphans:
minio_client.remove_object(BUCKET, key)
print(f" [DELETED] {key}")
print(f"Удалено {len(orphans)} orphan-объектов.")
return len(orphans)
async def find_missing_s3_objects(dry_run: bool) -> int:
print("\nПоиск missing объектов, записи в БД без файла в MinIO")
async with async_session_maker() as session:
result = await session.execute(select(Attachment.id, Attachment.object_key))
attachments = result.all()
print(f"Записей в БД: {len(attachments)}")
missing = []
used_missing = []
for att_id, key in attachments:
try:
minio_client.stat_object(BUCKET, key)
except Exception:
async with async_session_maker() as session:
result_msg = await session.execute(
select(Message).where(
cast(Message.attachment_ids, Text).contains(f'"{att_id}"')
)
)
is_used = result_msg.scalar_one_or_none() is not None
if is_used:
used_missing.append((att_id, key))
print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем")
else:
missing.append((att_id, key))
print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить")
total = len(missing) + len(used_missing)
if total == 0:
print("Missing-записей не найдено.")
return 0
if dry_run:
print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.")
else:
if missing:
async with async_session_maker() as session:
for att_id, _ in missing:
att = await session.get(Attachment, att_id)
if att:
await session.delete(att)
await session.commit()
print(f"Удалено {len(missing)} неиспользуемых missing из БД.")
if used_missing:
print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).")
return len(missing)
async def main():
parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД")
parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять")
parser.add_argument("--execute", action="store_true", help="Удалить найденное")
args = parser.parse_args()
if not args.dry_run and not args.execute:
print("Укажите --dry-run или --execute")
return
dry_run = args.dry_run and not args.execute
orphans = await find_orphan_s3_objects(dry_run)
missing = await find_missing_s3_objects(dry_run)
print("\nИтог")
print(f"Orphan S3 без БД: {orphans}")
print(f"Missing БД без S3 (удалено): {missing}")
if dry_run:
print("Режим: DRY-RUN (ничего не удалено)")
else:
print("Режим: EXECUTE (изменения применены)")
if __name__ == "__main__":
asyncio.run(main())
+12
View File
@@ -0,0 +1,12 @@
echo "Нагрузка: 30 запросов с интервалом 0.5 сек"
for i in {1..30}; do
TS=$(date +"%H:%M:%S")
RESP=$(curl -s -i http://localhost:8080/health/live)
CODE=$(echo "$RESP" | head -1 | awk '{print $2}')
INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}')
echo "$TS code=$CODE instance=${INST:-<нет>}"
sleep 0.5
done
+20
View File
@@ -0,0 +1,20 @@
set -e
echo "Rolling update: app1"
docker compose stop app1
docker compose up -d --build app1
echo "Жду, пока app1 станет healthy..."
sleep 5
echo ""
echo "Rolling update: app2"
docker compose stop app2
docker compose up -d --build app2
echo "Жду, пока app2 станет healthy..."
sleep 5
echo ""
echo "Проверка: обе реплики живы"
curl -i http://localhost:8080/health/live
curl -i http://localhost:8080/health/live
+78
View File
@@ -0,0 +1,78 @@
set -e
mkdir -p evidence
{
echo "=== GRACE PERIOD TEST — $(date) ==="
echo ""
curl -s -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
-u "grace_tester:secret1234567" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
echo "Токен получен: ${TOKEN:0:20}..."
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Attachment ID: $ATT_ID"
echo "Object key: $OBJECT_KEY"
docker compose exec -T db psql -U messenger -d messenger_db -c \
"DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null
echo "Запись удалена из БД — ORPHAN"
echo ""
echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
echo ""
echo "=== Пауза 11 секунд ==="
sleep 11
echo ""
echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
echo ""
echo "=== EXECUTE (удаляем созревший orphan) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов"
echo ""
echo "=== Проверка: orphan удалён из MinIO ==="
STAT=$(docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1)
echo "Orphan в MinIO: $STAT"
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется"
echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»"
echo "3. Execute → [DELETED] — удалён из MinIO"
echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются"
} 2>&1 | tee evidence/track2-grace-period.txt
echo ""
echo "Лог сохранён в evidence/track2-grace-period.txt"
+153
View File
@@ -0,0 +1,153 @@
set -e
mkdir -p evidence
{
echo "================================================================"
echo " RECONCILER TEST — $(date)"
echo "================================================================"
echo ""
TEST_USER="reconcile_tester_$(date +%s)"
TEST_PASS="secret1234567"
echo "=== 1. Создание пользователя ($TEST_USER) ==="
REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}")
REG_BODY=$(echo "$REG_RESP" | sed '$d')
REG_CODE=$(echo "$REG_RESP" | tail -n 1)
echo "HTTP $REG_CODE"
echo "Body: $REG_BODY"
if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then
echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)"
exit 1
fi
echo ""
echo "=== 2. Логин ==="
LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \
-u "$TEST_USER:$TEST_PASS")
LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d')
LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1)
echo "HTTP $LOGIN_CODE"
if [ "$LOGIN_CODE" != "201" ]; then
echo "FAIL: логин не удался (HTTP $LOGIN_CODE)"
echo "Body: $LOGIN_BODY"
exit 1
fi
TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "")
if [ -z "$TOKEN" ]; then
echo "FAIL: токен не получен из ответа"
echo "Body: $LOGIN_BODY"
exit 1
fi
echo "Токен получен: ${TOKEN:0:20}..."
echo ""
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "User ID: $USER_ID"
echo ""
echo "=== 3. Создание чата ==="
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
echo "=== 4. Создаём ORPHAN ==="
RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n')
echo "Orphan attachment_id: $ORPHAN_ATT_ID"
echo "Orphan object_key: $ORPHAN_KEY"
docker compose exec -T db psql -U messenger -d messenger_db -c \
"DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null
echo "Запись удалена из БД — ORPHAN"
echo ""
echo "=== 5. Создаём MISSING ==="
RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
echo "Missing attachment_id: $MISSING_ATT_ID"
echo "Missing object_key: $MISSING_KEY"
docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY"
echo "Файл удалён из MinIO — MISSING"
echo ""
echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ==="
sleep 11
echo ""
echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --dry-run
echo ""
echo "=== 7. EXECUTE ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --execute
echo ""
echo "=== 8. Проверка: orphan удалён из MinIO ==="
ORPHAN_STAT=$(docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1)
echo "Orphan в MinIO: $ORPHAN_STAT"
echo ""
echo "=== 9. Проверка: missing удалён из БД ==="
MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT"
echo ""
echo "=== 10. Идемпотентность: повторный DRY-RUN ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --dry-run
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
ORPHAN_GONE=false
if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then
ORPHAN_GONE=true
fi
if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then
echo "PASS:"
echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен"
echo " - ORPHAN удалён из MinIO"
echo " - MISSING удалён из БД"
echo " - Повторный dry-run: 0/0 (идемпотентность)"
else
echo "FAIL:"
echo " - Orphan в MinIO: $ORPHAN_STAT"
echo " - Missing count в БД: $MISSING_COUNT"
fi
} 2>&1 | tee evidence/reconciler-full.txt
echo ""
echo "Лог сохранён в evidence/reconciler-full.txt"
+76
View File
@@ -0,0 +1,76 @@
set -e
mkdir -p evidence
{
echo "=== USED ATTACHMENT TEST — $(date) ==="
echo ""
curl -s -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
-u "used_tester:secret1234567" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Attachment ID: $ATT_ID"
echo "Object key: $KEY"
# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ
IDEM_KEY="used-test-$(date +%s)"
curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \
-H "Authorization: Bearer $TOKEN" \
-H "Idempotency-Key: $IDEM_KEY" \
-H "Content-Type: application/json" \
-d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null
echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ"
echo ""
docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY"
echo "Файл удалён из MinIO — missing (используется)"
echo ""
echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей"
echo ""
echo "=== EXECUTE (используемый не удаляется) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено"
echo ""
COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Записей с id=$ATT_ID в БД: $COUNT"
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
if [ "$COUNT" = "1" ]; then
echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])"
else
echo "FAIL: используемый missing удалён (count=$COUNT)"
fi
} 2>&1 | tee evidence/track2-used-attachment.txt
+61
View File
@@ -0,0 +1,61 @@
import time
import requests
BASE_URL = "http://localhost:8080"
def get_token(username="stream_tester"):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "T", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def get_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "Stream Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def main():
print("=" * 60)
print(" STREAM LIMIT TEST")
print("=" * 60)
token = get_token()
chat_id = get_chat(token)
print(f"Chat ID: {chat_id}")
print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===")
def gen():
for _ in range(100):
yield b"0" * (1024 * 1024)
start = time.monotonic()
try:
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png",
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"},
data=gen(), timeout=60,
)
status = resp.status_code
except Exception as e:
print(f"Ошибка: {e}")
status = None
elapsed = time.monotonic() - start
print(f"Status: {status} (ожидаем 413)")
print(f"Время: {elapsed:.2f}s")
print("\n" + "=" * 60)
if status == 413:
print("PASS: потоковый лимит работает, 413")
else:
print(f"FAIL: ожидался 413, получен {status}")
print("=" * 60)
if __name__ == "__main__":
main()
+29
View File
@@ -0,0 +1,29 @@
import asyncio
import sys
from uuid import UUID
from app.worker import async_session_maker, engine
from app.models.attachment import Attachment
async def main():
attachment_id = UUID(sys.argv[1])
try:
async with async_session_maker() as session:
att = await session.get(Attachment, attachment_id)
if att is None:
print("Attachment not found")
return
print("Attachment ID:", att.id)
print("Status:", att.status)
print("Error code:", att.error_code)
finally:
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
+29
View File
@@ -0,0 +1,29 @@
import asyncio
import sys
from uuid import UUID
from app.worker import async_session_maker, engine
from app.models.attachment import Attachment
async def main():
attachment_id = UUID(sys.argv[1])
try:
async with async_session_maker() as session:
att = await session.get(Attachment, attachment_id)
if att is None:
print("Attachment not found")
return
print("Attachment ID:", att.id)
print("Status:", att.status)
print("Error code:", att.error_code)
finally:
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
+154
View File
@@ -0,0 +1,154 @@
import asyncio
import json
import subprocess
import sys
import uuid
from pathlib import Path
def test_worker_retry():
result = subprocess.run(
[
"docker", "compose", "exec",
"-T", "-w", "/app",
"worker", "python", "-"
],
input=Path(__file__).read_text(encoding="utf-8"),
text=True,
capture_output=True,
timeout=60,
)
print(result.stdout, end="")
if result.stderr:
print(result.stderr, file=sys.stderr)
assert result.returncode == 0, "Проверка Worker не прошла"
async def main():
import aio_pika
from sqlalchemy import select
from app.worker import (
async_session_maker,
engine,
RABBITMQ_URL,
QUEUE_NAME,
)
from app.models.attachment import Attachment
try:
async with async_session_maker() as session:
example = (
await session.execute(
select(Attachment).limit(1)
)
).scalars().first()
if example is None:
print(
"FAIL: нет вложений. Сначала запусти make test."
)
return False
test_id = uuid.uuid4()
test_att = Attachment(
id=test_id,
object_key=f"retry-test-{test_id}.jpg",
filename="retry-test.jpg",
mime_type="image/jpeg",
size_bytes=1,
sha256="0" * 64,
chat_id=example.chat_id,
user_id=example.user_id,
status="queued",
)
session.add(test_att)
await session.commit()
print(f"Тестовое вложение: {test_id}", flush=True)
connection = await aio_pika.connect_robust(
RABBITMQ_URL
)
async with connection:
channel = await connection.channel(
publisher_confirms=True
)
await channel.declare_queue(
QUEUE_NAME,
durable=True
)
await channel.default_exchange.publish(
aio_pika.Message(
body=json.dumps({
"attachment_id": str(test_id)
}).encode(),
delivery_mode=(
aio_pika.DeliveryMode.PERSISTENT
),
),
routing_key=QUEUE_NAME,
)
success = False
for _ in range(60):
await asyncio.sleep(0.5)
async with async_session_maker() as session:
att = await session.get(
Attachment,
test_id
)
if att and att.status == "failed":
print("Финальный статус:", att.status)
print(
"Безопасный error_code:",
att.error_code
)
success = (
att.error_code == "s3_download_error"
)
break
if success:
async with async_session_maker() as session:
att = await session.get(
Attachment,
test_id
)
if att:
await session.delete(att)
await session.commit()
print("Тестовая запись удалена.")
print("PASS: failed и безопасный error_code")
else:
print("FAIL: ожидаемый результат не получен.")
print(
"Тестовая запись оставлена для диагностики."
)
print(f"ID для проверки логов: {test_id}")
return success
finally:
await engine.dispose()
if __name__ == "__main__":
success = asyncio.run(main())
sys.exit(0 if success else 1)