Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b92fa218ba |
+18
-12
@@ -1,25 +1,31 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
# Скопируйте в .env. Файл .env не коммитить.
|
||||
# Пароли ниже — учебные заглушки локального стенда, не боевые секреты.
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
SESSION_STORE=valkey
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=local-only-change-me
|
||||
POSTGRES_DB=messenger
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
DRAIN_SECONDS=3
|
||||
# Не задавайте INSTANCE_ID: иначе все реплики получат один X-Instance-Id.
|
||||
# DATABASE_URL, REDIS_URL и BROKER_URL собирает compose.yaml.
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=LAB5LOCALACCESS
|
||||
S3_SECRET_ACCESS_KEY=local-only-change-me-s3
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=local-only-change-me-rabbit
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_BACKOFF_BASE_SECONDS=1
|
||||
JOB_BACKOFF_CAP_SECONDS=30
|
||||
JOB_BACKOFF_JITTER_RATIO=0.2
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
JPEG_QUALITY=85
|
||||
WEBP_QUALITY=80
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
*.sh text eol=lf
|
||||
*.cfg text eol=lf
|
||||
compose.yaml text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
.env.example text eol=lf
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
FROM python:3.12.6-slim-bookworm
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY --chown=app:app src/ /app/
|
||||
COPY --chown=app:app migrations/ /app/migrations/
|
||||
|
||||
USER app
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
LABEL lab.build-id=$BUILD_ID
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=8s --retries=5 \
|
||||
CMD python3 -c "import urllib.request,sys; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2); sys.exit(0)"
|
||||
|
||||
CMD ["python3", "/app/server.py"]
|
||||
@@ -1,37 +1,117 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Самсонов Александр, БВТ2303
|
||||
- Целевая оценка: 4
|
||||
- Commit/PR: локально, до публикации
|
||||
- Источники и помощь ИИ: контракт лабораторной, [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), перенос стенда lab4. Реализацию и прогон проверок помог сделать ИИ в Cursor.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose v2 и Python 3.9+ для публичных тестов. Образ API и worker — Python 3.12.6. Проверка выполнена в WSL Ubuntu 22.04: Docker 29.1.3, Compose 2.40.3.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp .env.example .env
|
||||
chmod +x scripts/up.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh scripts/stop-worker.sh scripts/start-worker.sh
|
||||
docker compose --env-file .env -f compose.yaml up -d --build --scale api=2 --remove-orphans
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Тот же запуск: `sh scripts/up.sh`. Дождитесь `GET http://127.0.0.1:8080/health/ready` → `{"status":"ok"}`.
|
||||
|
||||
## Доказательства критериев
|
||||
Публичный порт только у `lb`: `127.0.0.1:8080`. У API, worker, PostgreSQL, Valkey, RustFS и RabbitMQ порты наружу не опубликованы. Миграции выполняет одноразовый `migrate`, приватный bucket создаёт одноразовый `s3-init`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
Остановка без удаления данных:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env -f compose.yaml stop
|
||||
```
|
||||
|
||||
Удаление учебных volumes:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env -f compose.yaml down -v
|
||||
```
|
||||
|
||||
После `scripts/stop-one.sh` остановленная реплика не поднимается сама. Вернуть её: `sh scripts/restore-api.sh`. Остановить и снова запустить только worker: `scripts/stop-worker.sh` и `scripts/start-worker.sh`.
|
||||
|
||||
Образы: `python:3.12.6-slim-bookworm`, `postgres:16.6-bookworm`, `valkey/valkey:8.0.2`, `traefik:v3.7.16`, `rustfs/rustfs:1.0.4`, `rabbitmq:3.13.7-alpine`.
|
||||
|
||||
Healthcheck RabbitMQ вызывает `rabbitmq-diagnostics --timeout 5 -q ping`, а timeout самой проверки Docker — 10 с. Короткий timeout здесь нельзя ставить: Docker обрывает diagnostics, и этот обрыв останавливает брокер.
|
||||
|
||||
## Архитектура
|
||||
|
||||
```
|
||||
Клиент → Traefik → API ×2 → PostgreSQL (пользователи, чаты, сообщения, метаданные, jobs, outbox)
|
||||
→ Valkey (сессии, AOF)
|
||||
→ RustFS (staging, оригинал, варианты)
|
||||
→ RabbitMQ quorum (wake-up, не байты)
|
||||
→ worker (отдельный процесс: lease, resize, ack)
|
||||
```
|
||||
|
||||
Из lab4 сохранены две реплики, общий PostgreSQL и Valkey, Traefik без sticky, `X-Instance-Id`, идемпотентность POST сообщения, drain по SIGTERM, приватный bucket и проверка PNG/JPEG по заголовку до распаковки. Upload больше не делает resize и не кладёт оригинал в конечный ключ.
|
||||
|
||||
`/health/ready` отвечает 200, только если живы PostgreSQL, Valkey и bucket. Недоступность worker или RabbitMQ не переводит API в синхронный resize и не обязана ронять ready: намерение уже лежит в outbox. Период проверки Traefik — 1 с, timeout — 1 с.
|
||||
|
||||
Что устойчиво в момент 202:
|
||||
|
||||
1. Полные байты оригинала в приватном ключе `staging/<attachment_id>`.
|
||||
2. Строка `attachments` со статусом `queued`, checksum и размерами.
|
||||
3. Строка `attachment_jobs` и строка `outbox` в той же транзакции PostgreSQL.
|
||||
|
||||
Ответ 202 — снимок этого коммита (`status=queued`), даже если worker успеет закончить работу сразу после него. В сообщение брокера входят только поля `contracts/jobs.schema.json`: `job_id`, `attachment_id`, `pipeline_version=1`, `attempt`, `created_at`. Байтов, имени файла, ключа S3 и credentials там нет. Worker читает ключ staging из PostgreSQL.
|
||||
|
||||
Publisher confirm означает, что брокер принял durable-сообщение. Consumer ack ставится только после устойчивого эффекта: `ready` или финальный `failed` уже закоммичены, повторная доставка готовой job — no-op. Это не exactly-once доставка.
|
||||
|
||||
Разрыв между COMMIT и publish закрыт outbox. Релей крутится и в API, и в worker. Строка помечается опубликованной только после confirm. Если API умер после COMMIT, worker при старте публикует неопубликованные строки. Повторная публикация не создаёт второй job: ключ эффекта — `(attachment_id, pipeline_version)`.
|
||||
|
||||
Ключ идемпотентности варианта — `(attachment_id, pipeline_version, name)`. Повтор пишет те же ключи `originals/<id>` и `variants/<id>/{thumbnail,crop,optimized}`.
|
||||
|
||||
### Варианты
|
||||
|
||||
| Вариант | Формат | Правило | Параметры |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| thumbnail | JPEG | вписать в 256×256, без увеличения, пол вниз, минимум 1 px | quality 85, optimize, EXIF orientation до масштаба, альфа на белый, EXIF/ICC не пишутся |
|
||||
| crop | JPEG | центральный квадрат, затем ровно 128×128, увеличение маленького исходника разрешено | те же параметры JPEG |
|
||||
| optimized | WebP | вписать в 1280×1280 без увеличения | quality 80, method 6, EXIF/ICC/XMP не пишутся; меньший размер каждого маленького файла не обещается |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Фикстура 320×200 даёт thumbnail 256×160, crop 128×128 и optimized 320×200. `ready` ставится одной транзакцией только когда записаны оригинал и все три варианта.
|
||||
|
||||
## Эксперименты и отказы
|
||||
### Ограничения обработки
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
- Байты: 10 MiB, отказ 413 до чтения лишнего тела.
|
||||
- Пиксели: 40 000 000 по заголовку в API и повторно в worker до `load()`. Pillow `MAX_IMAGE_PIXELS` тот же. Decompression bomb — постоянная ошибка `payload_too_large`.
|
||||
- Время: lease `JOB_TIMEOUT_SECONDS` (60). Истёкший `processing` sweeper возвращает в очередь и тратит попытку.
|
||||
- Память worker: лимит контейнера 512 MiB. В логе старта это `memory_limit=536870912`.
|
||||
- Конкуренция: `WORKER_CONCURRENCY=2`, у каждого канала `prefetch_count=1`. Worker не вычитывает очередь целиком.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
### Повторы, DLQ, replay, staging
|
||||
|
||||
Постоянные ошибки (`invalid_image`, `payload_too_large`) сразу дают `failed` и копию сообщения в quorum-очередь `attachment.failed`. Реестр — таблица `job_failures`.
|
||||
|
||||
Временные ошибки (S3, обрыв, таймаут) делают backoff `min(30, 1 * 2^(attempt-1))` секунд плюс jitter до 20% и публикуют тот же `job_id` с новым `attempt`. После `JOB_MAX_ATTEMPTS=5` статус `failed`, код `retry_exhausted`.
|
||||
|
||||
`POST /api/v1/attachments/{id}/replay` доступен участнику чата. Если статус уже `ready`, ответ 200 и новая работа не создаётся (запись `job_replays.outcome=ignored_ready`). Если `failed`, тот же job снова становится `queued`. Пока job в `queued` или `processing`, replay отвечает 409.
|
||||
|
||||
Staging удаляется после коммита `ready`. Отдельный проход стирает оставшийся staging готовых объектов и staging `failed` старше `STAGING_RETENTION_SECONDS` (86400). Текущую очередь этот проход не трогает.
|
||||
|
||||
В `APP_ENV=lab` имена `__lab_fault_transient*` и `__lab_fault_permanent*` — учебные инъекции временной и постоянной ошибки. `CRASH_AFTER_S3=1` останавливает процесс после записи объектов и до ack.
|
||||
|
||||
## Доказательства
|
||||
|
||||
| Критерий | Команда | Результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Публичный smoke: 202, ready, JPEG thumbnail 256×160, оригинал | `python3 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS | evidence/smoke.txt |
|
||||
| Restart API, worker, PostgreSQL, Valkey, RustFS и RabbitMQ, тот же токен и тот же файл | `python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080` | PASS. Скрипт перезапускает `api worker postgres valkey s3 rabbitmq`, volumes не удаляет | evidence/persistence.txt, evidence/restart-times.txt |
|
||||
| Остановка обслужившей реплики | `python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080` | 2 реплики, 140 успешных записей, 4 ошибки, первое восстановление 3.658 с, PASS | evidence/failover.txt |
|
||||
| Worker остановлен, upload 202, content 409, чат жив, рестарт RabbitMQ с volume сохраняет 2 сообщения, затем ready | `python3 scripts/grade4_demo.py` | `broker restart kept 2 then 2 process messages`, id `59f221a0-7fe3-47ba-9b87-97ac966357b7` | evidence/grade4.txt |
|
||||
| Crop 128×128 и WebP 320×200, replay ready не плодит варианты | тот же прогон | id `113efcbd-9dcf-46bd-9af8-17d1c927f517`, `replay=ignored_ready`, вариантов 3 | evidence/grade4.txt |
|
||||
| Crash между COMMIT и publish: брокер и API остановлены после 202, worker поднимает outbox без нового upload | тот же прогон | ready в PostgreSQL до старта API, id `50db09ac-d826-47a3-899a-ada58734390a` | evidence/grade4.txt |
|
||||
| Постоянная ошибка, DLQ, явный replay не создаёт успешный дубль | тот же прогон | `invalid_image`, очередь `attachment.failed` есть, id `8e345bcd-8900-4287-9a06-4bb504ee6ae2` | evidence/grade4.txt |
|
||||
| Backoff и две временные ошибки, затем один набор вариантов | тот же прогон | `retries=2 final_attempt=3`, id `c266f9bb-1d43-453e-b4b3-f1f0ef29a112` | evidence/grade4.txt |
|
||||
| Остановка после записи S3 и до ack; повтор доводит тот же результат | тот же прогон | статус после падения `processing`, затем ready, вариантов 3, staging очищен, id `e2c19231-f4ec-4d67-a89b-f94c5325d91f` | evidence/grade4.txt |
|
||||
| Истёкший lease зависшего processing | тот же прогон | `lease_expired`, затем ready, id `17531de7-4616-45fb-ac94-9d45355f846d` | evidence/grade4.txt |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Контейнеры, которые реально перезапускает `scripts/restart.sh`: `api`, `worker`, `postgres`, `valkey`, `s3`, `rabbitmq`. `lb` скрипт не трогает. После прогона failover остановленная реплика была возвращена `scripts/restore-api.sh`, чтобы стенд не остался с одной копией.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Ограничения
|
||||
|
||||
Уровень 5 не делался: нет сравнения prefetch под нагрузкой и нет versioned pipeline. Один worker, внутри него два потребителя. Пока worker лежит дольше lease, сверка может положить в очередь ещё одно wake-up сообщение; эффект от этого не дублируется. Учебные имена `__lab_fault_*` работают только при `APP_ENV=lab`. Один и тот же ключ из `.env` является корневым ключом RustFS и ключом API. Это локальный стенд.
|
||||
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
# Локальный стенд lab5: клиент → Traefik → API ×2 → PostgreSQL + Valkey + RustFS.
|
||||
# Задания: transactional outbox → RabbitMQ → отдельный worker.
|
||||
# Публичный порт только у lb.
|
||||
name: mtusi-lab5
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6-bookworm
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
shm_size: 128mb
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.75"
|
||||
memory: 512M
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.2
|
||||
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"]
|
||||
volumes:
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 3s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.4
|
||||
environment:
|
||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_CHECK_UPDATE: "false"
|
||||
volumes:
|
||||
- s3data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
# diagnostics по умолчанию ждёт до 70 с. Короткий timeout Docker убивает
|
||||
# этот процесс и вместе с ним останавливает сервер брокера.
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "--timeout", "5", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 8
|
||||
start_period: 40s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 768M
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python3", "/app/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
|
||||
s3-init:
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python3", "/app/s3_init.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
api:
|
||||
image: mtusi-messenger:lab5
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: valkey
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
REDIS_URL: redis://valkey:6379/0
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
BROKER_URL: amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F
|
||||
OUTBOX_PUBLISH_ENABLED: "1"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
expose:
|
||||
- "8080"
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.docker.network=mtusi-lab5-net
|
||||
- traefik.http.routers.messenger.rule=PathPrefix(`/`)
|
||||
- traefik.http.routers.messenger.entrypoints=web
|
||||
- traefik.http.services.messenger.loadbalancer.server.port=8080
|
||||
- traefik.http.services.messenger.loadbalancer.healthcheck.path=/health/ready
|
||||
- traefik.http.services.messenger.loadbalancer.healthcheck.interval=1s
|
||||
- traefik.http.services.messenger.loadbalancer.healthcheck.timeout=1s
|
||||
- traefik.http.services.messenger.loadbalancer.healthcheck.scheme=http
|
||||
- traefik.http.services.messenger.loadbalancer.passhostheader=true
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 8
|
||||
start_period: 8s
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
stop_signal: SIGTERM
|
||||
networks: [lab]
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
|
||||
worker:
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python3", "/app/worker.py"]
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
BROKER_URL: amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-1}
|
||||
JOB_BACKOFF_CAP_SECONDS: ${JOB_BACKOFF_CAP_SECONDS:-30}
|
||||
JOB_BACKOFF_JITTER_RATIO: ${JOB_BACKOFF_JITTER_RATIO:-0.2}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
JPEG_QUALITY: ${JPEG_QUALITY:-85}
|
||||
WEBP_QUALITY: ${WEBP_QUALITY:-80}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
init: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
stop_signal: SIGTERM
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.75"
|
||||
memory: 512M
|
||||
|
||||
lb:
|
||||
image: traefik:v3.7.16
|
||||
command: ["--configFile=/etc/traefik/traefik.yml"]
|
||||
volumes:
|
||||
- ./infra/traefik.yml:/etc/traefik/traefik.yml:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 8
|
||||
start_period: 3s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 5s
|
||||
networks: [lab]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
networks:
|
||||
lab:
|
||||
name: mtusi-lab5-net
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
s3data:
|
||||
rabbitmqdata:
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 4, "first_recovery_seconds": 3.658}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,8 @@
|
||||
happy-path ready id=113efcbd-9dcf-46bd-9af8-17d1c927f517 variants=3 replay=ignored_ready
|
||||
broker restart kept 2 then 2 process messages
|
||||
worker-down then broker restart kept queue; ready id=59f221a0-7fe3-47ba-9b87-97ac966357b7
|
||||
commit/publish gap recovered by worker without a new upload id=50db09ac-d826-47a3-899a-ada58734390a
|
||||
permanent invalid_image + replay id=8e345bcd-8900-4287-9a06-4bb504ee6ae2 dlq present
|
||||
backoff retries=2 final_attempt=3 id=c266f9bb-1d43-453e-b4b3-f1f0ef29a112
|
||||
crash after s3 before ack redelivery id=e2c19231-f4ec-4d67-a89b-f94c5325d91f variants=3 staging cleared
|
||||
expired processing lease recovered id=17531de7-4616-45fb-ac94-9d45355f846d
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,20 @@
|
||||
BEFORE
|
||||
mtusi-lab5-rabbitmq-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-lb-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-api-2 Up Less than a second (health: starting)
|
||||
mtusi-lab5-api-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-worker-1 Up Less than a second
|
||||
mtusi-lab5-postgres-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-s3-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-valkey-1 Up Less than a second (health: starting)
|
||||
worker concurrency=2 prefetch_per_channel=1 max_attempts=5 timeout=60s memory_limit=536870912
|
||||
AFTER
|
||||
mtusi-lab5-rabbitmq-1 Up 2 seconds (health: starting)
|
||||
mtusi-lab5-lb-1 Up 4 seconds (health: starting)
|
||||
mtusi-lab5-api-2 Up Less than a second (health: starting)
|
||||
mtusi-lab5-api-1 Up Less than a second (health: starting)
|
||||
mtusi-lab5-worker-1 Up 3 seconds
|
||||
mtusi-lab5-postgres-1 Up 3 seconds (health: starting)
|
||||
mtusi-lab5-s3-1 Up 3 seconds (health: starting)
|
||||
mtusi-lab5-valkey-1 Up 3 seconds (health: starting)
|
||||
READY
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -45,9 +45,14 @@ Content-Type: application/json
|
||||
GET {{baseUrl}}/api/v1/chats/{{chatId}}/messages?limit=50
|
||||
Authorization: Bearer {{sessionToken}}
|
||||
|
||||
### Binary upload; HTTP file-client syntax for reading local fixture
|
||||
### Binary upload. Lab5 отвечает 202 queued; готовность смотреть отдельным GET
|
||||
POST {{baseUrl}}/api/v1/chats/{{chatId}}/attachments?filename=sample.png
|
||||
Authorization: Bearer {{sessionToken}}
|
||||
Content-Type: image/png
|
||||
|
||||
< ../tests/fixtures/sample.png
|
||||
|
||||
### Явный replay. Для ready это no-op и 200, для failed — снова queued и 202
|
||||
# @attachmentId = REPLACE_WITH_ATTACHMENT_UUID
|
||||
POST {{baseUrl}}/api/v1/attachments/REPLACE_WITH_ATTACHMENT_UUID/replay
|
||||
Authorization: Bearer {{sessionToken}}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Traefik 3.7.16. Без sticky cookie.
|
||||
# Адреса API Traefik берёт из Docker и обновляет по событиям start/die,
|
||||
# поэтому новая реплика и остановленная не зависят от кэша одного DNS-ответа.
|
||||
# Healthcheck: период 1s, timeout 1s, одна неуспешная проверка исключает backend.
|
||||
# Верхняя оценка обнаружения отказа: 1s до следующей проверки + 1s timeout = 2s.
|
||||
# Таймауты чтения запроса и ответа — 60s, чтобы прошёл оригинал до 10 MiB.
|
||||
# Обнаружение мёртвой реплики от этого не зависит: healthcheck по-прежнему 1s+1s.
|
||||
|
||||
global:
|
||||
checkNewVersion: false
|
||||
sendAnonymousUsage: false
|
||||
|
||||
entryPoints:
|
||||
web:
|
||||
address: ":8080"
|
||||
http:
|
||||
aliasHeadersStrategy: reject
|
||||
transport:
|
||||
respondingTimeouts:
|
||||
readTimeout: 60s
|
||||
writeTimeout: 60s
|
||||
idleTimeout: 30s
|
||||
ping:
|
||||
address: "127.0.0.1:8082"
|
||||
http:
|
||||
aliasHeadersStrategy: delete
|
||||
|
||||
ping:
|
||||
entryPoint: ping
|
||||
|
||||
providers:
|
||||
docker:
|
||||
endpoint: unix:///var/run/docker.sock
|
||||
exposedByDefault: false
|
||||
network: mtusi-lab5-net
|
||||
watch: true
|
||||
|
||||
serversTransport:
|
||||
forwardingTimeouts:
|
||||
dialTimeout: 1s
|
||||
responseHeaderTimeout: 60s
|
||||
idleConnTimeout: 1s
|
||||
|
||||
log:
|
||||
level: INFO
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Пользователи, чаты, участники, сообщения.
|
||||
-- Сессии здесь не хранятся: их источник истины — Valkey (AOF + volume).
|
||||
-- Идемпотентность отправки — в 002_idempotency.sql.
|
||||
--
|
||||
-- Транзакции приложения:
|
||||
-- регистрация — один INSERT, уникальность username держит UNIQUE
|
||||
-- создание чата — одна транзакция: чат и все chat_members, иначе откат
|
||||
-- сообщение и запись идемпотентности — одна транзакция
|
||||
--
|
||||
-- Индекс истории messages_history_idx покрывает keyset-запрос
|
||||
-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
-- ORDER BY created_at, id
|
||||
-- LIMIT $4
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT users_username_unique UNIQUE (username)
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX chats_created_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,18 @@
|
||||
-- Общий результат POST сообщения.
|
||||
-- Область: user_id + http_method + path + idem_key.
|
||||
-- Строка пишется в той же транзакции, что и messages, поэтому 5xx
|
||||
-- до COMMIT не оставляет ни сообщения, ни «окончательного» ключа.
|
||||
-- Повтор с другим request_hash в пределах TTL — отказ без второй вставки.
|
||||
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
http_method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
idem_key text NOT NULL,
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, http_method, path, idem_key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Метаданные вложения. Байты лежат в приватном S3-bucket, не в этой таблице.
|
||||
-- object_key назначает сервер. filename только для отображения.
|
||||
-- Связь сообщения и вложения — message_attachments, порядок задаёт position.
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT attachments_object_key_unique UNIQUE (object_key)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
@@ -0,0 +1,92 @@
|
||||
-- Lab5: состояние обработки, варианты и transactional outbox.
|
||||
-- Байты в сообщения брокера не кладутся. Источник намерения — эта база.
|
||||
-- Уже сохранённые вложения lab4 остаются ready: worker их не пересчитывает.
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN status text NOT NULL DEFAULT 'ready',
|
||||
ADD COLUMN error_code text,
|
||||
ADD COLUMN updated_at timestamptz,
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
|
||||
|
||||
UPDATE attachments SET updated_at = created_at WHERE updated_at IS NULL;
|
||||
|
||||
ALTER TABLE attachments ALTER COLUMN updated_at SET NOT NULL;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
attempt integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
error_code text,
|
||||
lease_token uuid,
|
||||
lease_until timestamptz,
|
||||
next_attempt_at timestamptz,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_attempt_check CHECK (attempt >= 1)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_due_idx
|
||||
ON attachment_jobs (status, next_attempt_at, lease_until);
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (id),
|
||||
payload jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
available_at timestamptz NOT NULL,
|
||||
published_at timestamptz,
|
||||
publish_attempts integer NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX outbox_one_unpublished
|
||||
ON outbox (job_id)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE INDEX outbox_due_idx
|
||||
ON outbox (available_at)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE job_failures (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
attempt integer NOT NULL,
|
||||
error_code text NOT NULL,
|
||||
permanent boolean NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX job_failures_attachment_idx
|
||||
ON job_failures (attachment_id, created_at);
|
||||
|
||||
CREATE TABLE job_replays (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
outcome text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,6 @@
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg-pool==3.2.3
|
||||
redis==5.1.1
|
||||
minio==7.2.15
|
||||
pika==1.3.2
|
||||
Pillow==10.4.0
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сценарии уровня 4. Стенд уже запущен. Секреты и токены не печатаются."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def compose(*args, check_code=True):
|
||||
proc = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
if check_code and proc.returncode != 0:
|
||||
raise Failure(proc.stdout[-2000:])
|
||||
return proc
|
||||
|
||||
|
||||
def psql(sql):
|
||||
proc = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql)
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def queues():
|
||||
proc = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def queue_depth(name):
|
||||
text = queues()
|
||||
for line in text.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and parts[0] == name and parts[-1].isdigit():
|
||||
return int(parts[-1])
|
||||
raise Failure("queue %s missing:\n%s" % (name, text))
|
||||
|
||||
|
||||
def wait_depth(name, minimum, seconds=20):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last = queue_depth(name)
|
||||
except Failure:
|
||||
last = None
|
||||
if last is not None and last >= minimum:
|
||||
return last
|
||||
time.sleep(0.5)
|
||||
raise Failure("queue %s depth %s, want >= %s" % (name, last, minimum))
|
||||
|
||||
|
||||
def wait_status(client, actor, att, wanted, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = att
|
||||
while time.monotonic() < deadline:
|
||||
last = client.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
|
||||
if last["status"] == wanted:
|
||||
return last
|
||||
if wanted != "failed" and last["status"] == "failed":
|
||||
raise Failure("attachment failed: " + str(last.get("error_code")))
|
||||
time.sleep(0.4)
|
||||
raise Failure("status %s did not become %s" % (last.get("status"), wanted))
|
||||
|
||||
|
||||
def wait_postgres_ready(att_id, seconds=70):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = ""
|
||||
while time.monotonic() < deadline:
|
||||
last = psql("SELECT status FROM attachments WHERE id = '%s'" % att_id)
|
||||
if last == "ready":
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure("postgres status stayed " + last)
|
||||
|
||||
|
||||
def register(client):
|
||||
username = "u_g4_" + uuid.uuid4().hex[:12]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
user = client.call(
|
||||
"POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "Проверка 4", "password": password},
|
||||
schema="User",
|
||||
)["json"]
|
||||
raw = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = client.call(
|
||||
"POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + raw},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
return {"id": user["id"], "token": session["token"], "user": user}
|
||||
|
||||
|
||||
def upload(client, actor, chat_id, filename, content, content_type="image/png"):
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
|
||||
202,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={"Content-Type": content_type},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
|
||||
|
||||
def bad_png():
|
||||
ihdr = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
||||
chunk = b"IHDR" + ihdr
|
||||
crc = zlib.crc32(chunk) & 0xFFFFFFFF
|
||||
return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def webp_size(data):
|
||||
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
||||
index = 12
|
||||
while index + 8 <= len(data):
|
||||
kind = data[index:index + 4]
|
||||
size = int.from_bytes(data[index + 4:index + 8], "little")
|
||||
chunk = data[index + 8:index + 8 + size]
|
||||
if kind == b"VP8X" and len(chunk) >= 10:
|
||||
return (1 + int.from_bytes(chunk[4:7], "little"), 1 + int.from_bytes(chunk[7:10], "little"))
|
||||
if kind == b"VP8 " and len(chunk) >= 10:
|
||||
return (int.from_bytes(chunk[6:8], "little") & 0x3FFF, int.from_bytes(chunk[8:10], "little") & 0x3FFF)
|
||||
if kind == b"VP8L" and len(chunk) >= 5:
|
||||
bits = int.from_bytes(chunk[1:5], "little")
|
||||
return ((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1)
|
||||
index += 8 + size + (size & 1)
|
||||
raise Failure("WebP dimensions not found")
|
||||
|
||||
|
||||
def variant_count(att_id):
|
||||
return int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att_id))
|
||||
|
||||
|
||||
def main():
|
||||
sample = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
client = Client(BASE, None)
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
alice = register(client)
|
||||
bob_name = "u_g4b_" + uuid.uuid4().hex[:12]
|
||||
bob_password = "T3st-" + uuid.uuid4().hex
|
||||
bob_user = client.call(
|
||||
"POST", "/api/v1/users", 201,
|
||||
body={"username": bob_name, "display_name": "Боб", "password": bob_password},
|
||||
schema="User",
|
||||
)["json"]
|
||||
chat = client.call(
|
||||
"POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Очередь", "member_ids": [alice["id"], bob_user["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
notes = []
|
||||
|
||||
att = upload(client, alice, chat["id"], "sample.png", sample)
|
||||
check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload snapshot")
|
||||
ready = wait_status(client, alice, att, "ready")
|
||||
names = [item["name"] for item in ready["variants"]]
|
||||
check(names == ["thumbnail", "crop", "optimized"], "variant set " + ",".join(names))
|
||||
thumb = next(item for item in ready["variants"] if item["name"] == "thumbnail")
|
||||
crop = next(item for item in ready["variants"] if item["name"] == "crop")
|
||||
optimized = next(item for item in ready["variants"] if item["name"] == "optimized")
|
||||
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail meta")
|
||||
check((crop["width"], crop["height"]) == (128, 128) and crop["content_type"] == "image/jpeg", "crop meta")
|
||||
check(optimized["content_type"] == "image/webp" and (optimized["width"], optimized["height"]) == (320, 200), "webp meta")
|
||||
original = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice)
|
||||
check(original["bytes"] == sample, "original bytes changed")
|
||||
thumb_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=alice)
|
||||
crop_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=alice)
|
||||
webp_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=alice)
|
||||
check(jpeg_size(thumb_bytes["bytes"]) == (256, 160), "thumbnail jpeg")
|
||||
check(jpeg_size(crop_bytes["bytes"]) == (128, 128), "crop jpeg")
|
||||
check(webp_size(webp_bytes["bytes"]) == (320, 200), "webp size")
|
||||
check(hashlib.sha256(thumb_bytes["bytes"]).hexdigest() == thumb["sha256"], "thumb sha")
|
||||
replay = client.call("POST", "/api/v1/attachments/%s/replay" % att["id"], 200, actor=alice, schema="Attachment")["json"]
|
||||
check(replay["status"] == "ready" and replay["variants"] == ready["variants"], "ready replay changed variants")
|
||||
check(variant_count(att["id"]) == 3, "replay created another variant set")
|
||||
check(psql("SELECT outcome FROM job_replays WHERE attachment_id = '%s' ORDER BY created_at DESC LIMIT 1" % att["id"]) == "ignored_ready", "replay audit")
|
||||
notes.append("happy-path ready id=%s variants=3 replay=ignored_ready" % att["id"])
|
||||
|
||||
compose("stop", "worker")
|
||||
held = upload(client, alice, chat["id"], "while-worker-down.png", sample)
|
||||
client.call("GET", "/api/v1/attachments/%s/content" % held["id"], 409, actor=alice)
|
||||
client.call(
|
||||
"POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice,
|
||||
body={"text": "очередь не блокирует чат", "attachment_ids": [held["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
schema="Message",
|
||||
)
|
||||
before = wait_depth("attachment.process", 1)
|
||||
compose("restart", "rabbitmq")
|
||||
for _ in range(40):
|
||||
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
||||
if probe.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("rabbitmq did not come back")
|
||||
after = wait_depth("attachment.process", 1, seconds=30)
|
||||
notes.append("broker restart kept %s then %s process messages" % (before, after))
|
||||
compose("start", "worker")
|
||||
held_ready = wait_status(client, alice, held, "ready")
|
||||
check(variant_count(held["id"]) == 3, "restarted job duplicated variants")
|
||||
notes.append("worker-down then broker restart kept queue; ready id=%s" % held_ready["id"])
|
||||
|
||||
compose("stop", "worker")
|
||||
compose("stop", "rabbitmq")
|
||||
gap = upload(client, alice, chat["id"], "outbox-gap.png", sample)
|
||||
unpublished = psql(
|
||||
"SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id "
|
||||
"WHERE j.attachment_id = '%s' AND o.published_at IS NULL" % gap["id"]
|
||||
)
|
||||
check(unpublished == "1", "outbox was published while broker was down: " + unpublished)
|
||||
compose("stop", "api")
|
||||
compose("start", "rabbitmq")
|
||||
for _ in range(40):
|
||||
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
||||
if probe.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
compose("start", "worker")
|
||||
wait_postgres_ready(gap["id"])
|
||||
ready_before_api = psql("SELECT status FROM attachments WHERE id = '%s'" % gap["id"])
|
||||
check(ready_before_api == "ready", "worker did not recover unpublished outbox")
|
||||
compose("start", "api")
|
||||
for _ in range(40):
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("api did not return")
|
||||
gap_body = client.call("GET", "/api/v1/attachments/%s" % gap["id"], actor=alice, schema="Attachment")["json"]
|
||||
check(gap_body["status"] == "ready" and len(gap_body["variants"]) == 3, "gap attachment not ready via api")
|
||||
notes.append("commit/publish gap recovered by worker without a new upload id=%s" % gap["id"])
|
||||
|
||||
broken = upload(client, alice, chat["id"], "broken.png", bad_png())
|
||||
failed = wait_status(client, alice, broken, "failed")
|
||||
check(failed["error_code"] == "invalid_image", "permanent code " + str(failed.get("error_code")))
|
||||
check(variant_count(broken["id"]) == 0, "failed job stored variants")
|
||||
client.call(
|
||||
"POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
|
||||
body={"text": "", "attachment_ids": [broken["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
again = client.call("POST", "/api/v1/attachments/%s/replay" % broken["id"], 202, actor=alice, schema="Attachment")["json"]
|
||||
check(again["status"] == "queued", "replay of failed did not queue")
|
||||
failed_again = wait_status(client, alice, broken, "failed")
|
||||
check(failed_again["error_code"] == "invalid_image", "replay did not finish as permanent failure")
|
||||
check(variant_count(broken["id"]) == 0, "replay of failed created variants")
|
||||
dlq = queues()
|
||||
check("attachment.failed" in dlq, "dlq missing")
|
||||
notes.append("permanent invalid_image + replay id=%s dlq present" % broken["id"])
|
||||
|
||||
flaky = upload(client, alice, chat["id"], "__lab_fault_transient.png", sample)
|
||||
flaky_ready = wait_status(client, alice, flaky, "ready", seconds=40)
|
||||
failures = psql(
|
||||
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND permanent = false AND error_code = 'unavailable'"
|
||||
% flaky["id"]
|
||||
)
|
||||
check(int(failures) >= 2, "expected two transient failures, got " + failures)
|
||||
check(variant_count(flaky["id"]) == 3, "transient retries duplicated variants")
|
||||
attempt = psql("SELECT attempt FROM attachment_jobs WHERE attachment_id = '%s'" % flaky["id"])
|
||||
notes.append("backoff retries=%s final_attempt=%s id=%s" % (failures, attempt, flaky_ready["id"]))
|
||||
|
||||
compose("stop", "worker")
|
||||
crash = upload(client, alice, chat["id"], "crash-before-ack.png", sample)
|
||||
wait_depth("attachment.process", 1)
|
||||
crashed = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "run", "--rm", "--no-deps", "--name", "lab5-crash-once", "-e", "CRASH_AFTER_S3=1", "worker"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=45,
|
||||
)
|
||||
check(crashed.returncode != 0, "crash hook did not stop the worker")
|
||||
status_after_crash = psql("SELECT status FROM attachments WHERE id = '%s'" % crash["id"])
|
||||
check(status_after_crash == "processing", "crash did not stop between claim and ready: " + status_after_crash)
|
||||
compose("start", "worker")
|
||||
crash_ready = wait_status(client, alice, crash, "ready")
|
||||
check(variant_count(crash["id"]) == 3, "redelivery created a second variant set")
|
||||
original_after = client.call("GET", "/api/v1/attachments/%s/content" % crash["id"], actor=alice)
|
||||
check(original_after["bytes"] == sample, "redelivery changed the original")
|
||||
staging_left = psql("SELECT staging_key IS NULL FROM attachments WHERE id = '%s'" % crash["id"])
|
||||
check(staging_left == "t", "staging was not cleared after ready")
|
||||
notes.append("crash after s3 before ack redelivery id=%s variants=3 staging cleared" % crash_ready["id"])
|
||||
|
||||
compose("stop", "worker")
|
||||
compose("stop", "rabbitmq")
|
||||
leased = upload(client, alice, chat["id"], "expired-lease.png", sample)
|
||||
psql(
|
||||
"UPDATE attachment_jobs SET status = 'processing', attempt = 1, "
|
||||
"lease_token = gen_random_uuid(), lease_until = now() - interval '2 minutes', "
|
||||
"updated_at = now() WHERE attachment_id = '%s'" % leased["id"]
|
||||
)
|
||||
psql("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '%s'" % leased["id"])
|
||||
psql(
|
||||
"UPDATE outbox SET published_at = now() WHERE job_id = "
|
||||
"(SELECT id FROM attachment_jobs WHERE attachment_id = '%s')" % leased["id"]
|
||||
)
|
||||
compose("start", "rabbitmq")
|
||||
for _ in range(40):
|
||||
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
||||
if probe.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
compose("start", "worker")
|
||||
lease_ready = wait_status(client, alice, leased, "ready")
|
||||
lease_rows = psql(
|
||||
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND error_code = 'lease_expired'"
|
||||
% leased["id"]
|
||||
)
|
||||
check(int(lease_rows) >= 1, "expired lease was not recovered")
|
||||
check(variant_count(leased["id"]) == 3, "lease recovery duplicated variants")
|
||||
notes.append("expired processing lease recovered id=%s" % lease_ready["id"])
|
||||
|
||||
text = "\n".join(notes)
|
||||
print(text)
|
||||
evidence = ROOT / "evidence" / "grade4.txt"
|
||||
evidence.write_text(text + "\n", encoding="utf-8")
|
||||
print("PASS: grade 4 fault scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, subprocess.SubprocessError) as exc:
|
||||
print("FAIL:", exc)
|
||||
raise SystemExit(1)
|
||||
Executable → Regular
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Перезапуск API, worker и всех хранилищ, включая брокер. Volumes не удаляются.
|
||||
docker compose --env-file .env -f compose.yaml restart api worker postgres valkey s3 rabbitmq
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
|
||||
if [ -z "${replicas}" ]; then
|
||||
replicas=2
|
||||
fi
|
||||
docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env -f compose.yaml start worker
|
||||
Executable → Regular
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
found=""
|
||||
for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$cid")
|
||||
short=$(printf '%s' "$cid" | cut -c1-12)
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$cid
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
echo "API replica was not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# docker stop оставляет реплику выключенной: restart policy unless-stopped.
|
||||
docker stop -t 8 "$found" >/dev/null
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env -f compose.yaml stop worker
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ ! -f .env ]; then
|
||||
echo "Copy .env.example to .env" >&2
|
||||
exit 1
|
||||
fi
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
|
||||
if [ -z "${replicas}" ]; then
|
||||
replicas=2
|
||||
fi
|
||||
docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""RabbitMQ: durable quorum-очереди и publisher confirms.
|
||||
|
||||
Сообщение содержит только идентификаторы задания. Байты и ключи S3 сюда не входят.
|
||||
Ack потребителя и confirm издателя — разные границы.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pika
|
||||
|
||||
EXCHANGE = "attachments"
|
||||
PROCESS_QUEUE = "attachment.process"
|
||||
FAILED_QUEUE = "attachment.failed"
|
||||
PROCESS_KEY = "process"
|
||||
FAILED_KEY = "failed"
|
||||
|
||||
_lock = threading.Lock()
|
||||
_connection: pika.BlockingConnection | None = None
|
||||
_channel: pika.adapters.blocking_connection.BlockingChannel | None = None
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
path = os.environ.get("BROKER_URL_FILE", "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("BROKER_URL", "").strip()
|
||||
|
||||
|
||||
def declare(channel) -> None:
|
||||
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
|
||||
arguments = {"x-queue-type": "quorum"}
|
||||
channel.queue_declare(PROCESS_QUEUE, durable=True, arguments=arguments)
|
||||
channel.queue_declare(FAILED_QUEUE, durable=True, arguments=arguments)
|
||||
channel.queue_bind(PROCESS_QUEUE, EXCHANGE, PROCESS_KEY)
|
||||
channel.queue_bind(FAILED_QUEUE, EXCHANGE, FAILED_KEY)
|
||||
|
||||
|
||||
def connect() -> tuple[pika.BlockingConnection, pika.adapters.blocking_connection.BlockingChannel]:
|
||||
url = broker_url()
|
||||
if not url:
|
||||
raise RuntimeError("broker is not configured")
|
||||
params = pika.URLParameters(url)
|
||||
params.heartbeat = 60
|
||||
params.blocked_connection_timeout = 15
|
||||
params.socket_timeout = 10
|
||||
params.connection_attempts = 3
|
||||
params.retry_delay = 1
|
||||
connection = pika.BlockingConnection(params)
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
declare(channel)
|
||||
return connection, channel
|
||||
|
||||
|
||||
def _ensure():
|
||||
global _connection, _channel
|
||||
if _connection is not None and _connection.is_open and _channel is not None and _channel.is_open:
|
||||
return _channel
|
||||
if _connection is not None:
|
||||
try:
|
||||
_connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
_connection, _channel = connect()
|
||||
return _channel
|
||||
|
||||
|
||||
def publish_job(payload: dict, routing_key: str = PROCESS_KEY) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
if len(body) > 4096:
|
||||
raise RuntimeError("job message exceeds 4096 bytes")
|
||||
properties = pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(payload.get("job_id", "")),
|
||||
type=str(payload.get("type", "")),
|
||||
)
|
||||
with _lock:
|
||||
channel = _ensure()
|
||||
try:
|
||||
channel.basic_publish(
|
||||
EXCHANGE,
|
||||
routing_key,
|
||||
body,
|
||||
properties,
|
||||
mandatory=True,
|
||||
)
|
||||
except Exception:
|
||||
global _connection, _channel
|
||||
_connection = None
|
||||
_channel = None
|
||||
raise
|
||||
|
||||
|
||||
def publish_on(channel, payload: dict, routing_key: str) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
channel.basic_publish(
|
||||
EXCHANGE,
|
||||
routing_key,
|
||||
body,
|
||||
pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(payload.get("job_id", "")),
|
||||
type=str(payload.get("type", "")),
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Ошибки API. Текст не содержит SQL, паролей и токенов."""
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей."""
|
||||
from __future__ import annotations
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
||||
# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC.
|
||||
_JPEG_SOF = {
|
||||
0xC0, 0xC1, 0xC2, 0xC3,
|
||||
0xC5, 0xC6, 0xC7,
|
||||
0xC9, 0xCA, 0xCB,
|
||||
0xCD, 0xCE, 0xCF,
|
||||
}
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str:
|
||||
if data.startswith(PNG_SIG):
|
||||
return "image/png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
|
||||
|
||||
|
||||
def dimensions(data: bytes, content_type: str) -> tuple[int, int]:
|
||||
if content_type == "image/png":
|
||||
return _png_size(data)
|
||||
if content_type == "image/jpeg":
|
||||
return _jpeg_size(data)
|
||||
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
|
||||
|
||||
|
||||
def _png_size(data: bytes) -> tuple[int, int]:
|
||||
if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR":
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
|
||||
width = int.from_bytes(data[16:20], "big")
|
||||
height = int.from_bytes(data[20:24], "big")
|
||||
if width < 1 or height < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
|
||||
return width, height
|
||||
|
||||
|
||||
def _jpeg_size(data: bytes) -> tuple[int, int]:
|
||||
if len(data) < 4 or data[:2] != b"\xff\xd8":
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
index = 2
|
||||
size = len(data)
|
||||
while index < size:
|
||||
if data[index] != 0xFF:
|
||||
break
|
||||
while index < size and data[index] == 0xFF:
|
||||
index += 1
|
||||
if index >= size:
|
||||
break
|
||||
marker = data[index]
|
||||
index += 1
|
||||
if marker in (0xD8, 0xD9) or marker == 0xDA:
|
||||
break
|
||||
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
if index + 2 > size:
|
||||
break
|
||||
segment = int.from_bytes(data[index:index + 2], "big")
|
||||
if segment < 2 or index + segment > size:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
if marker in _JPEG_SOF:
|
||||
if segment < 7:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
height = int.from_bytes(data[index + 3:index + 5], "big")
|
||||
width = int.from_bytes(data[index + 5:index + 7], "big")
|
||||
if width < 1 or height < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
return width, height
|
||||
index += segment
|
||||
raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров")
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Однократные миграции. Реплики API этот файл не запускают."""
|
||||
import sys
|
||||
|
||||
import db
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db.migrate_with_retry()
|
||||
sys.stderr.write("migrations applied\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
N = 2**14
|
||||
R = 8
|
||||
P = 1
|
||||
DKLEN = 32
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN,
|
||||
maxmem=64 * 1024 * 1024)
|
||||
return "scrypt$%d$%d$%d$%s$%s" % (
|
||||
N, R, P,
|
||||
base64.b64encode(salt).decode("ascii"),
|
||||
base64.b64encode(digest).decode("ascii"),
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
algo, n, r, p, salt_b64, digest_b64 = stored.split("$")
|
||||
if algo != "scrypt":
|
||||
return False
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(digest_b64)
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(expected), maxmem=64 * 1024 * 1024)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(actual, expected)
|
||||
|
||||
|
||||
# Одинаковая стоимость проверки, когда пользователя нет.
|
||||
DUMMY_HASH = hash_password("dummy-password-not-used")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Доставка outbox в брокер. Строка остаётся неопубликованной, пока нет confirm."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import broker
|
||||
import db
|
||||
|
||||
|
||||
def publish_batch(limit: int = 20) -> int:
|
||||
if os.environ.get("OUTBOX_PUBLISH_ENABLED", "1") == "0":
|
||||
return 0
|
||||
rows = db.take_outbox(limit)
|
||||
published = 0
|
||||
for outbox_id, payload in rows:
|
||||
try:
|
||||
broker.publish_job(payload)
|
||||
except Exception:
|
||||
sys.stderr.write("broker did not confirm job %s\n" % payload.get("job_id", ""))
|
||||
return published
|
||||
db.mark_outbox_published(outbox_id)
|
||||
published += 1
|
||||
return published
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Однократное создание приватного bucket. Повторный запуск ничего не публикует."""
|
||||
import sys
|
||||
import time
|
||||
|
||||
import storage
|
||||
|
||||
|
||||
def main() -> None:
|
||||
storage.init()
|
||||
last = None
|
||||
for _ in range(30):
|
||||
try:
|
||||
storage.ensure_bucket()
|
||||
sys.stderr.write("bucket ready\n")
|
||||
return
|
||||
except Exception as exc:
|
||||
last = exc
|
||||
time.sleep(1)
|
||||
sys.stderr.write("bucket bootstrap failed\n")
|
||||
raise SystemExit(1) from last
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+730
@@ -0,0 +1,730 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Messenger lab5. Приём файла отделён от обработки: staging, outbox, брокер, worker."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import signal
|
||||
import socket
|
||||
import socketserver
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import db
|
||||
import images
|
||||
import passwords
|
||||
import sessions
|
||||
import storage
|
||||
from errors import ApiError
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z")
|
||||
UUID_RE = re.compile(
|
||||
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
||||
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z")
|
||||
TIMESTAMP_RE = re.compile(
|
||||
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z")
|
||||
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z")
|
||||
MAX_BODY = 1024 * 1024
|
||||
|
||||
_draining = threading.Event()
|
||||
_inflight = 0
|
||||
_inflight_lock = threading.Lock()
|
||||
|
||||
|
||||
def drain_seconds() -> float:
|
||||
raw = os.environ.get("DRAIN_SECONDS", "3")
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 3.0
|
||||
if value < 0:
|
||||
return 0.0
|
||||
if value > 6:
|
||||
return 6.0
|
||||
return value
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
current = datetime.now(timezone.utc)
|
||||
return current.replace(microsecond=(current.microsecond // 1000) * 1000)
|
||||
|
||||
|
||||
def is_blank(value: str) -> bool:
|
||||
return value.strip() == ""
|
||||
|
||||
|
||||
def require_object(body):
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом")
|
||||
|
||||
|
||||
def reject_extra(body: dict, allowed: set):
|
||||
extra = set(body) - allowed
|
||||
if extra:
|
||||
raise ApiError(400, "invalid_request",
|
||||
"Лишние поля: " + ", ".join(sorted(extra)))
|
||||
|
||||
|
||||
def parse_uuid(value, field: str) -> str:
|
||||
if not isinstance(value, str) or not UUID_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}")
|
||||
return value.lower()
|
||||
|
||||
|
||||
def parse_display_name(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "display_name должен быть строкой")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "display_name: 1..100 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "display_name не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def parse_username(value) -> str:
|
||||
if not isinstance(value, str) or not USERNAME_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$")
|
||||
return value
|
||||
|
||||
|
||||
def parse_password(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "password должен быть строкой")
|
||||
if not (12 <= len(value) <= 128):
|
||||
raise ApiError(400, "invalid_request", "password: 12..128 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "password не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def parse_title(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "title должен быть строкой")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "title: 1..100 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "title не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def max_upload_bytes() -> int:
|
||||
raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 10485760
|
||||
return value if value >= 1 else 10485760
|
||||
|
||||
|
||||
def max_image_pixels() -> int:
|
||||
raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 40000000
|
||||
return value if value >= 1 else 40000000
|
||||
|
||||
|
||||
def parse_text(value, has_attachments: bool) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "text должен быть строкой")
|
||||
if len(value) > 4000:
|
||||
raise ApiError(400, "invalid_request", "text: не больше 4000 code points")
|
||||
if is_blank(value) and not has_attachments:
|
||||
raise ApiError(400, "invalid_request", "text не может быть пустым без вложений")
|
||||
return value
|
||||
|
||||
|
||||
def parse_attachment_ids(value) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом")
|
||||
if len(value) > 10:
|
||||
raise ApiError(400, "invalid_request", "attachment_ids: не больше 10")
|
||||
seen = set()
|
||||
result = []
|
||||
for item in value:
|
||||
att_id = parse_uuid(item, "attachment_ids[]")
|
||||
if att_id in seen:
|
||||
raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты")
|
||||
seen.add(att_id)
|
||||
result.append(att_id)
|
||||
return result
|
||||
|
||||
|
||||
def parse_filename(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
|
||||
if not (1 <= len(value) <= 255):
|
||||
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
|
||||
for char in value:
|
||||
if char in "/\\" or unicodedata.category(char) == "Cc":
|
||||
raise ApiError(400, "invalid_request", "filename содержит запрещённый символ")
|
||||
return value
|
||||
|
||||
|
||||
def parse_member_ids(value) -> list:
|
||||
if not isinstance(value, list):
|
||||
raise ApiError(400, "invalid_request", "member_ids должен быть массивом")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "member_ids: 1..100")
|
||||
seen = set()
|
||||
result = []
|
||||
for item in value:
|
||||
uid = parse_uuid(item, "member_ids[]")
|
||||
if uid in seen:
|
||||
raise ApiError(400, "invalid_request", "member_ids содержит дубликаты")
|
||||
seen.add(uid)
|
||||
result.append(uid)
|
||||
return result
|
||||
|
||||
|
||||
def parse_limit(raw, default=50) -> int:
|
||||
if raw is None:
|
||||
return default
|
||||
if isinstance(raw, str) and raw.strip() == "":
|
||||
raise ApiError(400, "invalid_request", "limit должен быть целым")
|
||||
try:
|
||||
value = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise ApiError(400, "invalid_request", "limit должен быть целым")
|
||||
if not (1 <= value <= 100):
|
||||
raise ApiError(400, "invalid_request", "limit: 1..100")
|
||||
return value
|
||||
|
||||
|
||||
def parse_optional_cursor(raw):
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, str) or raw.strip() == "":
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
return raw
|
||||
|
||||
|
||||
def parse_cursor_timestamp(value: str) -> datetime:
|
||||
if not isinstance(value, str) or not TIMESTAMP_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_idempotency_key(headers) -> str:
|
||||
raw = headers.get("Idempotency-Key")
|
||||
if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw):
|
||||
raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key")
|
||||
return raw
|
||||
|
||||
|
||||
def canonical_hash(text: str, attachment_ids: list) -> str:
|
||||
# Порядок ключей JSON не важен. Отсутствующий attachment_ids равен [].
|
||||
# Порядок элементов массива сохраняется: это часть тела запроса.
|
||||
payload = {"attachment_ids": attachment_ids, "text": text}
|
||||
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def challenge_for(path: str) -> str:
|
||||
if path == "/api/v1/auth/sessions":
|
||||
return 'Basic realm="messenger", charset="UTF-8"'
|
||||
return 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def bearer_token(headers) -> str:
|
||||
auth = headers.get("Authorization") or ""
|
||||
parts = auth.split(None, 1)
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
|
||||
raise ApiError(401, "unauthorized", "Требуется Bearer")
|
||||
return parts[1].strip()
|
||||
|
||||
|
||||
def current_user(headers) -> dict:
|
||||
token = bearer_token(headers)
|
||||
user = db.get_user(sessions.user_id(token))
|
||||
if user is None:
|
||||
raise ApiError(401, "unauthorized", "Сессия недействительна")
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(headers) -> tuple[str, str]:
|
||||
auth = headers.get("Authorization") or ""
|
||||
parts = auth.split(None, 1)
|
||||
if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip():
|
||||
raise ApiError(401, "unauthorized", "Требуется Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc
|
||||
if ":" not in decoded:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные")
|
||||
username, password = decoded.split(":", 1)
|
||||
return username, password
|
||||
|
||||
|
||||
def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str:
|
||||
raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, chat_id: str):
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
parts = raw.split("|")
|
||||
if len(parts) != 3:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
c_chat, c_created, c_id = parts
|
||||
if not UUID_RE.match(c_chat):
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
if c_chat.lower() != chat_id:
|
||||
raise ApiError(400, "invalid_request", "cursor от другого чата")
|
||||
return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor")
|
||||
|
||||
|
||||
def encode_chat_cursor(created_at: str, chat_id: str) -> str:
|
||||
raw = f"chat|{created_at}|{chat_id}".encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_chat_cursor(cursor: str):
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
parts = raw.split("|")
|
||||
if len(parts) != 3 or parts[0] != "chat":
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor")
|
||||
|
||||
|
||||
def create_user(body) -> dict:
|
||||
require_object(body)
|
||||
reject_extra(body, {"username", "display_name", "password"})
|
||||
username = parse_username(body.get("username"))
|
||||
display_name = parse_display_name(body.get("display_name"))
|
||||
password = parse_password(body.get("password"))
|
||||
return db.create_user(username, display_name, passwords.hash_password(password), now_utc())
|
||||
|
||||
|
||||
def create_session(headers) -> dict:
|
||||
username, password = basic_credentials(headers)
|
||||
row = None
|
||||
try:
|
||||
row = db.user_for_login(username)
|
||||
except ApiError:
|
||||
raise
|
||||
stored = row[3] if row is not None else passwords.DUMMY_HASH
|
||||
password_ok = passwords.verify_password(password, stored)
|
||||
if row is None or not password_ok:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные")
|
||||
token = secrets.token_urlsafe(32)
|
||||
ttl = sessions.session_ttl()
|
||||
expires = now_utc() + timedelta(seconds=ttl)
|
||||
sessions.create(token, str(row[0]), ttl)
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": db.format_ts(expires),
|
||||
"user": db.user_json((row[0], row[1], row[2], row[4])),
|
||||
}
|
||||
|
||||
|
||||
def logout(headers) -> None:
|
||||
token = bearer_token(headers)
|
||||
sessions.user_id(token)
|
||||
sessions.revoke(token)
|
||||
|
||||
|
||||
def create_chat(body, actor: dict) -> dict:
|
||||
require_object(body)
|
||||
reject_extra(body, {"title", "member_ids"})
|
||||
title = parse_title(body.get("title"))
|
||||
member_ids = parse_member_ids(body.get("member_ids"))
|
||||
if actor["id"] not in member_ids:
|
||||
raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids")
|
||||
return db.create_chat(title, member_ids, now_utc())
|
||||
|
||||
|
||||
def list_chats(actor: dict, limit: int, cursor) -> dict:
|
||||
decoded = decode_chat_cursor(cursor) if cursor else None
|
||||
page = db.list_chats(actor["id"], limit, decoded)
|
||||
next_cursor = None
|
||||
if page["next_cursor_parts"] is not None:
|
||||
created, chat_id = page["next_cursor_parts"]
|
||||
next_cursor = encode_chat_cursor(db.format_ts(created), chat_id)
|
||||
return {"items": page["items"], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict:
|
||||
cid = parse_uuid(chat_id, "chat_id")
|
||||
require_object(body)
|
||||
reject_extra(body, {"text", "attachment_ids"})
|
||||
attachment_ids = parse_attachment_ids(body.get("attachment_ids"))
|
||||
text = parse_text(body.get("text"), bool(attachment_ids))
|
||||
path = "/api/v1/chats/%s/messages" % cid
|
||||
return db.create_message_idempotent(
|
||||
cid,
|
||||
actor["id"],
|
||||
text,
|
||||
attachment_ids,
|
||||
now_utc(),
|
||||
"POST",
|
||||
path,
|
||||
idem_key,
|
||||
canonical_hash(text, attachment_ids),
|
||||
)
|
||||
|
||||
|
||||
def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict:
|
||||
detected = images.sniff(data)
|
||||
if detected != declared:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type")
|
||||
width, height = images.dimensions(data, detected)
|
||||
if width * height > max_image_pixels():
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
||||
att_id = str(uuid.uuid4())
|
||||
created = now_utc()
|
||||
staging_key = "staging/" + att_id
|
||||
object_key = "originals/" + att_id
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(uuid.uuid4()),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": att_id,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": db.format_ts(created),
|
||||
}
|
||||
storage.put_bytes(staging_key, data, detected)
|
||||
try:
|
||||
return db.insert_queued_attachment(
|
||||
att_id, chat_id, actor["id"], filename, detected, len(data),
|
||||
hashlib.sha256(data).hexdigest(), object_key, staging_key,
|
||||
width, height, created, payload,
|
||||
)
|
||||
except Exception:
|
||||
storage.delete_quiet(staging_key)
|
||||
raise
|
||||
|
||||
|
||||
def read_attachment(attachment_id: str, actor: dict) -> dict:
|
||||
return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"])
|
||||
|
||||
|
||||
def read_attachment_bytes(attachment_id: str, actor: dict, variant: str):
|
||||
if variant not in ("original", "thumbnail", "crop", "optimized"):
|
||||
raise ApiError(400, "invalid_request", "Некорректный variant")
|
||||
record = db.get_download(parse_uuid(attachment_id, "attachment_id"), actor["id"], variant)
|
||||
data = storage.get_bytes(record["object_key"])
|
||||
if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
|
||||
raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными")
|
||||
return record["content_type"], data
|
||||
|
||||
|
||||
def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict:
|
||||
cid = parse_uuid(chat_id, "chat_id")
|
||||
decoded = decode_cursor(cursor, cid) if cursor else None
|
||||
page = db.list_messages(cid, actor["id"], limit, decoded)
|
||||
next_cursor = None
|
||||
if page["next_cursor_parts"] is not None:
|
||||
created, message_id = page["next_cursor_parts"]
|
||||
next_cursor = encode_cursor(cid, db.format_ts(created), message_id)
|
||||
return {"items": page["items"], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
allow_reuse_address = True
|
||||
request_queue_size = 256
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "mtusi-messenger/5.0"
|
||||
protocol_version = "HTTP/1.0"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
def _request_id(self) -> str:
|
||||
return self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
|
||||
def _send_json(self, status: int, payload, request_id: str, extra_headers=None):
|
||||
body = b""
|
||||
if status != 204:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
if status != 204:
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("Connection", "close")
|
||||
for key, value in (extra_headers or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_bytes(self, content_type: str, payload: bytes, request_id: str):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if payload:
|
||||
self.wfile.write(payload)
|
||||
|
||||
def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
|
||||
def _read_body(self):
|
||||
length = self.headers.get("Content-Length")
|
||||
if not length:
|
||||
return None
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
|
||||
if size > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело слишком большое")
|
||||
raw = self.rfile.read(size)
|
||||
ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip()
|
||||
if ctype and ctype != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Пустое тело")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON") from exc
|
||||
|
||||
def _read_upload(self) -> tuple[str, bytes]:
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Нужен Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
|
||||
if size > max_upload_bytes():
|
||||
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
|
||||
if size < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Пустое тело")
|
||||
declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
|
||||
if declared not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
|
||||
data = self.rfile.read(size)
|
||||
if len(data) != size:
|
||||
raise ApiError(400, "invalid_request", "Тело короче Content-Length")
|
||||
return declared, data
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
global _inflight
|
||||
with _inflight_lock:
|
||||
_inflight += 1
|
||||
rid = self._request_id()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
try:
|
||||
self._route(method, path, query, rid)
|
||||
except ApiError as exc:
|
||||
headers = {}
|
||||
if exc.status == 401:
|
||||
headers["WWW-Authenticate"] = challenge_for(path)
|
||||
self._send_error(exc.status, exc.code, exc.message, rid, headers)
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
except Exception:
|
||||
sys.stderr.write("internal error on %s %s\n" % (method, path))
|
||||
self._send_error(500, "internal_error", "Внутренняя ошибка", rid)
|
||||
finally:
|
||||
with _inflight_lock:
|
||||
_inflight -= 1
|
||||
|
||||
def _route(self, method: str, path: str, query: dict, rid: str):
|
||||
if method == "GET" and path == "/health/live":
|
||||
return self._send_json(200, {"status": "ok"}, rid)
|
||||
if method == "GET" and path == "/health/ready":
|
||||
if _draining.is_set():
|
||||
raise ApiError(503, "unavailable", "Реплика останавливается")
|
||||
if db.ping() and sessions.ping() and storage.ping():
|
||||
return self._send_json(200, {"status": "ok"}, rid)
|
||||
raise ApiError(503, "unavailable", "Зависимость недоступна")
|
||||
if _draining.is_set():
|
||||
raise ApiError(503, "unavailable", "Реплика останавливается")
|
||||
|
||||
if method == "POST" and path == "/api/v1/users":
|
||||
return self._send_json(201, create_user(self._read_body()), rid)
|
||||
if method == "POST" and path == "/api/v1/auth/sessions":
|
||||
return self._send_json(201, create_session(self.headers), rid)
|
||||
if method == "DELETE" and path == "/api/v1/auth/sessions/current":
|
||||
logout(self.headers)
|
||||
return self._send_json(204, None, rid)
|
||||
if method == "GET" and path == "/api/v1/users/me":
|
||||
return self._send_json(200, current_user(self.headers), rid)
|
||||
|
||||
if method == "POST" and path == "/api/v1/chats":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(201, create_chat(self._read_body(), actor), rid)
|
||||
if method == "GET" and path == "/api/v1/chats":
|
||||
actor = current_user(self.headers)
|
||||
limit = parse_limit(query.get("limit", [None])[0])
|
||||
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
|
||||
return self._send_json(200, list_chats(actor, limit, cursor), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
chat_id = parse_uuid(match.group(1), "chat_id")
|
||||
if not db.is_member(chat_id, actor["id"]):
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
names = query.get("filename") or []
|
||||
if len(names) != 1:
|
||||
raise ApiError(400, "invalid_request", "Нужен один filename")
|
||||
filename = parse_filename(names[0])
|
||||
declared, data = self._read_upload()
|
||||
body = upload_attachment(chat_id, actor, filename, declared, data)
|
||||
return self._send_json(202, body, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/replay", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
att_id = parse_uuid(match.group(1), "attachment_id")
|
||||
body = db.replay_attachment(att_id, actor["id"])
|
||||
status = 200 if body["status"] == "ready" else 202
|
||||
return self._send_json(status, body, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
variants = query.get("variant") or ["original"]
|
||||
if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "":
|
||||
raise ApiError(400, "invalid_request", "Некорректный variant")
|
||||
content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0])
|
||||
return self._send_bytes(content_type, payload, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(200, read_attachment(match.group(1), actor), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
idem_key = parse_idempotency_key(self.headers)
|
||||
body = create_message(match.group(1), self._read_body(), actor, idem_key)
|
||||
return self._send_json(201, body, rid)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
limit = parse_limit(query.get("limit", [None])[0])
|
||||
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
|
||||
return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid)
|
||||
|
||||
raise ApiError(404, "not_found", "Маршрут не найден")
|
||||
|
||||
|
||||
def _inflight_now() -> int:
|
||||
with _inflight_lock:
|
||||
return _inflight
|
||||
|
||||
|
||||
def _publish_outbox() -> None:
|
||||
import relay
|
||||
while not _draining.is_set():
|
||||
try:
|
||||
relay.publish_batch()
|
||||
except Exception:
|
||||
sys.stderr.write("outbox publish failed\n")
|
||||
_draining.wait(0.4)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
db.init()
|
||||
except Exception:
|
||||
sys.stderr.write("database is not configured\n")
|
||||
try:
|
||||
sessions.init()
|
||||
except Exception:
|
||||
sys.stderr.write("session store is not configured\n")
|
||||
try:
|
||||
storage.init()
|
||||
except Exception:
|
||||
sys.stderr.write("object storage is not configured\n")
|
||||
threading.Thread(target=_publish_outbox, name="outbox", daemon=True).start()
|
||||
|
||||
server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler)
|
||||
|
||||
def drain_and_stop():
|
||||
time.sleep(drain_seconds())
|
||||
deadline = time.monotonic() + 2
|
||||
while time.monotonic() < deadline and _inflight_now() > 0:
|
||||
time.sleep(0.05)
|
||||
server.shutdown()
|
||||
|
||||
def shutdown(signum, frame):
|
||||
if _draining.is_set():
|
||||
return
|
||||
_draining.set()
|
||||
sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID)
|
||||
threading.Thread(target=drain_and_stop, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, shutdown)
|
||||
signal.signal(signal.SIGINT, shutdown)
|
||||
sys.stderr.write(
|
||||
"listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID)
|
||||
)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
db.close()
|
||||
sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии.
|
||||
|
||||
Источник истины сессий — общий Valkey, не память процесса API.
|
||||
AOF и volume сохраняют ключ при перезапуске хранилища.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
_client: redis.Redis | None = None
|
||||
|
||||
|
||||
def session_ttl() -> int:
|
||||
raw = os.environ.get("SESSION_TTL_SECONDS", "3600")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 3600
|
||||
return value if value >= 1 else 1
|
||||
|
||||
|
||||
def redis_url() -> str:
|
||||
path = os.environ.get("CACHE_URL_FILE", "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("REDIS_URL", "").strip()
|
||||
|
||||
|
||||
def init() -> None:
|
||||
global _client
|
||||
url = redis_url()
|
||||
if not url:
|
||||
_client = None
|
||||
return
|
||||
_client = redis.Redis.from_url(
|
||||
url, socket_connect_timeout=0.5, socket_timeout=0.5,
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
def _key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
return "session:" + digest
|
||||
|
||||
|
||||
def _require() -> redis.Redis:
|
||||
if _client is None:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно")
|
||||
return _client
|
||||
|
||||
|
||||
def ping() -> bool:
|
||||
try:
|
||||
return bool(_require().ping())
|
||||
except (ApiError, redis.RedisError):
|
||||
return False
|
||||
|
||||
|
||||
def create(token: str, user_id: str, ttl: int) -> None:
|
||||
try:
|
||||
_require().set(_key(token), user_id, ex=ttl)
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
|
||||
|
||||
def user_id(token: str) -> str:
|
||||
try:
|
||||
value = _require().get(_key(token))
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
if not value:
|
||||
raise ApiError(401, "unauthorized", "Сессия недействительна")
|
||||
return value
|
||||
|
||||
|
||||
def revoke(token: str) -> None:
|
||||
try:
|
||||
_require().delete(_key(token))
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Численные пределы worker. Секреты здесь не читаются."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
|
||||
def _int(name: str, default: int, low: int, high: int) -> int:
|
||||
try:
|
||||
value = int(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
if value < low:
|
||||
return low
|
||||
if value > high:
|
||||
return high
|
||||
return value
|
||||
|
||||
|
||||
def _float(name: str, default: float, low: float, high: float) -> float:
|
||||
try:
|
||||
value = float(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
if value < low:
|
||||
return low
|
||||
if value > high:
|
||||
return high
|
||||
return value
|
||||
|
||||
|
||||
def max_attempts() -> int:
|
||||
return _int("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
|
||||
|
||||
def job_timeout() -> int:
|
||||
return _int("JOB_TIMEOUT_SECONDS", 60, 5, 600)
|
||||
|
||||
|
||||
def worker_concurrency() -> int:
|
||||
return _int("WORKER_CONCURRENCY", 2, 1, 8)
|
||||
|
||||
|
||||
def max_image_pixels() -> int:
|
||||
return _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000)
|
||||
|
||||
|
||||
def max_upload_bytes() -> int:
|
||||
return _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760)
|
||||
|
||||
|
||||
def staging_retention() -> int:
|
||||
return _int("STAGING_RETENTION_SECONDS", 86400, 60, 7 * 86400)
|
||||
|
||||
|
||||
def jpeg_quality() -> int:
|
||||
return _int("JPEG_QUALITY", 85, 40, 95)
|
||||
|
||||
|
||||
def webp_quality() -> int:
|
||||
return _int("WEBP_QUALITY", 80, 40, 95)
|
||||
|
||||
|
||||
def backoff_delay(attempt: int) -> float:
|
||||
"""Экспоненциальная пауза с jitter. attempt — номер уже совершённой попытки."""
|
||||
base = _float("JOB_BACKOFF_BASE_SECONDS", 1.0, 0.1, 60.0)
|
||||
cap = _float("JOB_BACKOFF_CAP_SECONDS", 30.0, 0.1, 300.0)
|
||||
ratio = _float("JOB_BACKOFF_JITTER_RATIO", 0.2, 0.0, 1.0)
|
||||
delay = min(cap, base * (2 ** max(0, attempt - 1)))
|
||||
return delay + random.uniform(0, delay * ratio)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
_client: Minio | None = None
|
||||
|
||||
|
||||
def _read_setting(env_name: str, file_env: str) -> str:
|
||||
path = os.environ.get(file_env, "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get(env_name, "").strip()
|
||||
|
||||
|
||||
def endpoint() -> str:
|
||||
return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
|
||||
|
||||
|
||||
def bucket() -> str:
|
||||
return os.environ.get("S3_BUCKET", "messenger-attachments").strip()
|
||||
|
||||
|
||||
def _host_and_secure() -> tuple[str, bool]:
|
||||
raw = endpoint()
|
||||
secure = raw.startswith("https://")
|
||||
host = raw.split("://", 1)[-1].rstrip("/")
|
||||
return host, secure
|
||||
|
||||
|
||||
def init() -> None:
|
||||
global _client
|
||||
host, secure = _host_and_secure()
|
||||
access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
if not host or not access or not secret or not bucket():
|
||||
raise RuntimeError("object storage is not configured")
|
||||
_client = Minio(
|
||||
host,
|
||||
access_key=access,
|
||||
secret_key=secret,
|
||||
secure=secure,
|
||||
region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
)
|
||||
|
||||
|
||||
def _api() -> Minio:
|
||||
if _client is None:
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно")
|
||||
return _client
|
||||
|
||||
|
||||
def ping() -> bool:
|
||||
try:
|
||||
return _api().bucket_exists(bucket())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_bucket() -> None:
|
||||
api = _api()
|
||||
name = bucket()
|
||||
if not api.bucket_exists(name):
|
||||
api.make_bucket(name)
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str) -> None:
|
||||
try:
|
||||
_api().put_object(
|
||||
bucket(),
|
||||
key,
|
||||
io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type=content_type,
|
||||
)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
sys.stderr.write("s3 put failed\n")
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
|
||||
|
||||
|
||||
def get_bytes(key: str) -> bytes:
|
||||
response = None
|
||||
try:
|
||||
response = _api().get_object(bucket(), key)
|
||||
return response.read()
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
sys.stderr.write("s3 get failed\n")
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
|
||||
def exists(key: str) -> bool:
|
||||
try:
|
||||
_api().stat_object(bucket(), key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def delete_quiet(key: str) -> None:
|
||||
try:
|
||||
_api().remove_object(bucket(), key)
|
||||
except Exception:
|
||||
sys.stderr.write("s3 delete failed\n")
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
"""Варианты изображения. Оригинал этот модуль не перекодирует.
|
||||
|
||||
thumbnail: JPEG, вписать в 256×256 без увеличения, пропорции сохраняются,
|
||||
размеры округляются вниз до минимума 1 px. EXIF orientation применяется до
|
||||
масштаба. Альфа кладётся на белый. Качество JPEG_QUALITY (по умолчанию 85),
|
||||
optimize, без EXIF/ICC.
|
||||
|
||||
crop: JPEG тех же параметров, центральный квадрат, затем ровно 128×128.
|
||||
Маленький исходник можно увеличить.
|
||||
|
||||
optimized: WebP, вписать в 1280×1280 без увеличения. Качество WEBP_QUALITY
|
||||
(по умолчанию 80), method=6. EXIF, ICC и XMP в файл не пишутся. Меньший размер
|
||||
маленького файла не обещается.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
import settings
|
||||
|
||||
THUMBNAIL_LIMIT = 256
|
||||
CROP_SIZE = 128
|
||||
OPTIMIZED_LIMIT = 1280
|
||||
|
||||
|
||||
class PermanentJobError(Exception):
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class TransientJobError(Exception):
|
||||
def __init__(self, code: str = "unavailable"):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _open(data: bytes) -> Image.Image:
|
||||
if len(data) > settings.max_upload_bytes():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
Image.MAX_IMAGE_PIXELS = settings.max_image_pixels()
|
||||
try:
|
||||
image = Image.open(io.BytesIO(data))
|
||||
width, height = image.size
|
||||
if width < 1 or height < 1 or width * height > settings.max_image_pixels():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.load()
|
||||
except PermanentJobError:
|
||||
raise
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise PermanentJobError("payload_too_large") from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise PermanentJobError("invalid_image") from exc
|
||||
return image
|
||||
|
||||
|
||||
def _on_white(image: Image.Image) -> Image.Image:
|
||||
if image.mode == "RGB":
|
||||
return image
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
|
||||
|
||||
def _fit(image: Image.Image, limit: int) -> Image.Image:
|
||||
width, height = image.size
|
||||
if width <= limit and height <= limit:
|
||||
return image
|
||||
scale = min(limit / width, limit / height)
|
||||
resized = (max(1, int(width * scale)), max(1, int(height * scale)))
|
||||
return image.resize(resized, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def _crop(image: Image.Image) -> Image.Image:
|
||||
width, height = image.size
|
||||
side = min(width, height)
|
||||
left = (width - side) // 2
|
||||
top = (height - side) // 2
|
||||
cropped = image.crop((left, top, left + side, top + side))
|
||||
if cropped.size != (CROP_SIZE, CROP_SIZE):
|
||||
cropped = cropped.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)
|
||||
return cropped
|
||||
|
||||
|
||||
def _jpeg(image: Image.Image) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
_on_white(image).save(
|
||||
buffer,
|
||||
format="JPEG",
|
||||
quality=settings.jpeg_quality(),
|
||||
optimize=True,
|
||||
progressive=False,
|
||||
)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _webp(image: Image.Image) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
_on_white(image).save(
|
||||
buffer,
|
||||
format="WEBP",
|
||||
quality=settings.webp_quality(),
|
||||
method=6,
|
||||
)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def render(data: bytes) -> dict:
|
||||
image = _open(data)
|
||||
thumb = _fit(image, THUMBNAIL_LIMIT)
|
||||
crop = _crop(image)
|
||||
optimized = _fit(image, OPTIMIZED_LIMIT)
|
||||
thumb_bytes = _jpeg(thumb)
|
||||
crop_bytes = _jpeg(crop)
|
||||
optimized_bytes = _webp(optimized)
|
||||
return {
|
||||
"thumbnail": (thumb_bytes, "image/jpeg", thumb.size),
|
||||
"crop": (crop_bytes, "image/jpeg", crop.size),
|
||||
"optimized": (optimized_bytes, "image/webp", optimized.size),
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import broker
|
||||
import db
|
||||
import relay
|
||||
import settings
|
||||
import storage
|
||||
import variants
|
||||
from variants import PermanentJobError, TransientJobError
|
||||
|
||||
_stop = threading.Event()
|
||||
|
||||
|
||||
def _validate(payload) -> dict:
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("payload")
|
||||
allowed = {
|
||||
"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at", "traceparent", "tracestate",
|
||||
}
|
||||
if set(payload) - allowed:
|
||||
raise ValueError("extra")
|
||||
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
|
||||
raise ValueError("type")
|
||||
for field in ("job_id", "attachment_id"):
|
||||
uuid.UUID(str(payload.get(field)))
|
||||
if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1:
|
||||
raise ValueError("attempt")
|
||||
if not isinstance(payload.get("created_at"), str):
|
||||
raise ValueError("created_at")
|
||||
return payload
|
||||
|
||||
|
||||
def _lab_fault(source: dict, attempt: int) -> None:
|
||||
if os.environ.get("APP_ENV", "lab") != "lab":
|
||||
return
|
||||
name = source["filename"]
|
||||
if name.startswith("__lab_fault_permanent"):
|
||||
raise PermanentJobError("invalid_image")
|
||||
if name.startswith("__lab_fault_transient") and attempt < 3:
|
||||
raise TransientJobError("unavailable")
|
||||
|
||||
|
||||
def _produce(source: dict, attempt: int) -> list:
|
||||
_lab_fault(source, attempt)
|
||||
if source["size_bytes"] > settings.max_upload_bytes():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
if source["width"] * source["height"] > settings.max_image_pixels():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
staging_key = source["staging_key"]
|
||||
if not staging_key:
|
||||
if storage.exists(source["object_key"]):
|
||||
original = storage.get_bytes(source["object_key"])
|
||||
else:
|
||||
raise TransientJobError("unavailable")
|
||||
else:
|
||||
original = storage.get_bytes(staging_key)
|
||||
if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]:
|
||||
raise TransientJobError("unavailable")
|
||||
rendered = variants.render(original)
|
||||
storage.put_bytes(source["object_key"], original, source["content_type"])
|
||||
stored = []
|
||||
for name, (data, content_type, size) in rendered.items():
|
||||
key = "variants/%s/%s" % (source["attachment_id"], name)
|
||||
storage.put_bytes(key, data, content_type)
|
||||
stored.append({
|
||||
"name": name,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"width": size[0],
|
||||
"height": size[1],
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"object_key": key,
|
||||
})
|
||||
if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"):
|
||||
sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"])
|
||||
os._exit(86)
|
||||
return stored
|
||||
|
||||
|
||||
def _retry(payload: dict, claim: dict, code: str) -> str:
|
||||
delay = settings.backoff_delay(claim["attempt"])
|
||||
available = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
||||
sys.stderr.write(
|
||||
"retry job=%s attempt=%s delay=%.3f code=%s\n"
|
||||
% (payload["job_id"], claim["attempt"], delay, code)
|
||||
)
|
||||
return db.retry_job(
|
||||
payload["job_id"],
|
||||
claim["lease_token"],
|
||||
code,
|
||||
payload,
|
||||
available,
|
||||
settings.max_attempts(),
|
||||
)
|
||||
|
||||
|
||||
def _dlq(channel, payload: dict) -> None:
|
||||
try:
|
||||
broker.publish_on(channel, payload, broker.FAILED_KEY)
|
||||
except Exception:
|
||||
sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id"))
|
||||
|
||||
|
||||
def _cleanup(source: dict) -> None:
|
||||
key = source.get("staging_key")
|
||||
if not key:
|
||||
return
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(source["attachment_id"])
|
||||
|
||||
|
||||
def _work(channel, payload: dict) -> None:
|
||||
claim = db.claim_job(payload["job_id"], settings.job_timeout())
|
||||
if claim["action"] == "ack":
|
||||
return
|
||||
if claim["action"] != "work":
|
||||
time.sleep(0.4)
|
||||
raise TransientJobError("lease_held")
|
||||
source = db.load_job_source(payload["attachment_id"])
|
||||
if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready":
|
||||
return
|
||||
if int(payload["pipeline_version"]) != int(source["pipeline_version"]):
|
||||
return
|
||||
try:
|
||||
produced = _produce(source, claim["attempt"])
|
||||
outcome = db.finish_job(
|
||||
payload["job_id"], claim["lease_token"], produced, source["pipeline_version"],
|
||||
)
|
||||
except PermanentJobError as exc:
|
||||
outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"])
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
except TransientJobError as exc:
|
||||
if exc.code == "lease_held":
|
||||
raise
|
||||
outcome = _retry(payload, claim, exc.code)
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
except Exception as exc:
|
||||
sys.stderr.write("transient processing error %s\n" % type(exc).__name__)
|
||||
outcome = _retry(payload, claim, "unavailable")
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
if outcome == "lost":
|
||||
raise TransientJobError("lease_lost")
|
||||
if outcome == "ready":
|
||||
_cleanup(source)
|
||||
|
||||
|
||||
def _on_message(channel, method, _properties, body: bytes) -> None:
|
||||
try:
|
||||
payload = _validate(json.loads(body.decode("utf-8")))
|
||||
except Exception:
|
||||
sys.stderr.write("dropping invalid job message\n")
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
return
|
||||
try:
|
||||
_work(channel, payload)
|
||||
except TransientJobError:
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
return
|
||||
except Exception:
|
||||
sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id"))
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
return
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
|
||||
|
||||
def _consume() -> None:
|
||||
connection, channel = broker.connect()
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False)
|
||||
channel.start_consuming()
|
||||
|
||||
|
||||
def _consume_loop() -> None:
|
||||
while not _stop.is_set():
|
||||
try:
|
||||
_consume()
|
||||
except Exception:
|
||||
if _stop.is_set():
|
||||
return
|
||||
sys.stderr.write("consumer reconnect\n")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _sweep() -> None:
|
||||
while not _stop.is_set():
|
||||
try:
|
||||
expired = db.recover_expired_leases(settings.max_attempts())
|
||||
restored = db.reconcile_queued(settings.job_timeout())
|
||||
if expired or restored:
|
||||
sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored))
|
||||
for attachment_id, key in db.staging_ready_for_cleanup(40):
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(attachment_id)
|
||||
for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40):
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(attachment_id)
|
||||
relay.publish_batch()
|
||||
except Exception:
|
||||
sys.stderr.write("sweeper failed\n")
|
||||
_stop.wait(2)
|
||||
|
||||
|
||||
def _memory_limit() -> str:
|
||||
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
except OSError:
|
||||
continue
|
||||
return "unknown"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db.init()
|
||||
storage.init()
|
||||
sys.stderr.write(
|
||||
"worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n"
|
||||
% (
|
||||
settings.worker_concurrency(),
|
||||
settings.max_attempts(),
|
||||
settings.job_timeout(),
|
||||
_memory_limit(),
|
||||
)
|
||||
)
|
||||
threading.Thread(target=_sweep, name="sweep", daemon=True).start()
|
||||
threads = []
|
||||
for index in range(settings.worker_concurrency()):
|
||||
thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
try:
|
||||
while any(thread.is_alive() for thread in threads):
|
||||
time.sleep(0.5)
|
||||
except KeyboardInterrupt:
|
||||
_stop.set()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user