diff --git a/.env.example b/.env.example index 119384b5..a8550d0f 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,31 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 +# Скопируйте в .env. Файл .env не коммитить. +# Пароли ниже — учебные заглушки локального стенда, не боевые секреты. APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_STORE=valkey +POSTGRES_USER=messenger +POSTGRES_PASSWORD=local-only-change-me +POSTGRES_DB=messenger API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +DRAIN_SECONDS=3 +# Не задавайте INSTANCE_ID: иначе все реплики получат один X-Instance-Id. +# DATABASE_URL, REDIS_URL и BROKER_URL собирает compose.yaml. S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=LAB5LOCALACCESS +S3_SECRET_ACCESS_KEY=local-only-change-me-s3 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=local-only-change-me-rabbit WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_BACKOFF_BASE_SECONDS=1 +JOB_BACKOFF_CAP_SECONDS=30 +JOB_BACKOFF_JITTER_RATIO=0.2 STAGING_RETENTION_SECONDS=86400 +JPEG_QUALITY=85 +WEBP_QUALITY=80 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..04901310 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.cfg text eol=lf +compose.yaml text eol=lf +Dockerfile text eol=lf +.env.example text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..476acdd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12.6-slim-bookworm + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY --chown=app:app src/ /app/ +COPY --chown=app:app migrations/ /app/migrations/ + +USER app + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab + +EXPOSE 8080 + +ARG BUILD_ID=dev +LABEL lab.build-id=$BUILD_ID + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=8s --retries=5 \ + CMD python3 -c "import urllib.request,sys; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2); sys.exit(0)" + +CMD ["python3", "/app/server.py"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..d5bed78c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,117 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Самсонов Александр, БВТ2303 +- Целевая оценка: 4 +- Commit/PR: локально, до публикации +- Источники и помощь ИИ: контракт лабораторной, [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), перенос стенда lab4. Реализацию и прогон проверок помог сделать ИИ в Cursor. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ для публичных тестов. Образ API и worker — Python 3.12.6. Проверка выполнена в WSL Ubuntu 22.04: Docker 29.1.3, Compose 2.40.3. -## Архитектура и решения +```sh +cp .env.example .env +chmod +x scripts/up.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh scripts/stop-worker.sh scripts/start-worker.sh +docker compose --env-file .env -f compose.yaml up -d --build --scale api=2 --remove-orphans +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Тот же запуск: `sh scripts/up.sh`. Дождитесь `GET http://127.0.0.1:8080/health/ready` → `{"status":"ok"}`. -## Доказательства критериев +Публичный порт только у `lb`: `127.0.0.1:8080`. У API, worker, PostgreSQL, Valkey, RustFS и RabbitMQ порты наружу не опубликованы. Миграции выполняет одноразовый `migrate`, приватный bucket создаёт одноразовый `s3-init`. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Остановка без удаления данных: + +```sh +docker compose --env-file .env -f compose.yaml stop +``` + +Удаление учебных volumes: + +```sh +docker compose --env-file .env -f compose.yaml down -v +``` + +После `scripts/stop-one.sh` остановленная реплика не поднимается сама. Вернуть её: `sh scripts/restore-api.sh`. Остановить и снова запустить только worker: `scripts/stop-worker.sh` и `scripts/start-worker.sh`. + +Образы: `python:3.12.6-slim-bookworm`, `postgres:16.6-bookworm`, `valkey/valkey:8.0.2`, `traefik:v3.7.16`, `rustfs/rustfs:1.0.4`, `rabbitmq:3.13.7-alpine`. + +Healthcheck RabbitMQ вызывает `rabbitmq-diagnostics --timeout 5 -q ping`, а timeout самой проверки Docker — 10 с. Короткий timeout здесь нельзя ставить: Docker обрывает diagnostics, и этот обрыв останавливает брокер. + +## Архитектура + +``` +Клиент → Traefik → API ×2 → PostgreSQL (пользователи, чаты, сообщения, метаданные, jobs, outbox) + → Valkey (сессии, AOF) + → RustFS (staging, оригинал, варианты) + → RabbitMQ quorum (wake-up, не байты) + → worker (отдельный процесс: lease, resize, ack) +``` + +Из lab4 сохранены две реплики, общий PostgreSQL и Valkey, Traefik без sticky, `X-Instance-Id`, идемпотентность POST сообщения, drain по SIGTERM, приватный bucket и проверка PNG/JPEG по заголовку до распаковки. Upload больше не делает resize и не кладёт оригинал в конечный ключ. + +`/health/ready` отвечает 200, только если живы PostgreSQL, Valkey и bucket. Недоступность worker или RabbitMQ не переводит API в синхронный resize и не обязана ронять ready: намерение уже лежит в outbox. Период проверки Traefik — 1 с, timeout — 1 с. + +Что устойчиво в момент 202: + +1. Полные байты оригинала в приватном ключе `staging/`. +2. Строка `attachments` со статусом `queued`, checksum и размерами. +3. Строка `attachment_jobs` и строка `outbox` в той же транзакции PostgreSQL. + +Ответ 202 — снимок этого коммита (`status=queued`), даже если worker успеет закончить работу сразу после него. В сообщение брокера входят только поля `contracts/jobs.schema.json`: `job_id`, `attachment_id`, `pipeline_version=1`, `attempt`, `created_at`. Байтов, имени файла, ключа S3 и credentials там нет. Worker читает ключ staging из PostgreSQL. + +Publisher confirm означает, что брокер принял durable-сообщение. Consumer ack ставится только после устойчивого эффекта: `ready` или финальный `failed` уже закоммичены, повторная доставка готовой job — no-op. Это не exactly-once доставка. + +Разрыв между COMMIT и publish закрыт outbox. Релей крутится и в API, и в worker. Строка помечается опубликованной только после confirm. Если API умер после COMMIT, worker при старте публикует неопубликованные строки. Повторная публикация не создаёт второй job: ключ эффекта — `(attachment_id, pipeline_version)`. + +Ключ идемпотентности варианта — `(attachment_id, pipeline_version, name)`. Повтор пишет те же ключи `originals/` и `variants//{thumbnail,crop,optimized}`. + +### Варианты + +| Вариант | Формат | Правило | Параметры | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| thumbnail | JPEG | вписать в 256×256, без увеличения, пол вниз, минимум 1 px | quality 85, optimize, EXIF orientation до масштаба, альфа на белый, EXIF/ICC не пишутся | +| crop | JPEG | центральный квадрат, затем ровно 128×128, увеличение маленького исходника разрешено | те же параметры JPEG | +| optimized | WebP | вписать в 1280×1280 без увеличения | quality 80, method 6, EXIF/ICC/XMP не пишутся; меньший размер каждого маленького файла не обещается | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Фикстура 320×200 даёт thumbnail 256×160, crop 128×128 и optimized 320×200. `ready` ставится одной транзакцией только когда записаны оригинал и все три варианта. -## Эксперименты и отказы +### Ограничения обработки -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- Байты: 10 MiB, отказ 413 до чтения лишнего тела. +- Пиксели: 40 000 000 по заголовку в API и повторно в worker до `load()`. Pillow `MAX_IMAGE_PIXELS` тот же. Decompression bomb — постоянная ошибка `payload_too_large`. +- Время: lease `JOB_TIMEOUT_SECONDS` (60). Истёкший `processing` sweeper возвращает в очередь и тратит попытку. +- Память worker: лимит контейнера 512 MiB. В логе старта это `memory_limit=536870912`. +- Конкуренция: `WORKER_CONCURRENCY=2`, у каждого канала `prefetch_count=1`. Worker не вычитывает очередь целиком. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +### Повторы, DLQ, replay, staging + +Постоянные ошибки (`invalid_image`, `payload_too_large`) сразу дают `failed` и копию сообщения в quorum-очередь `attachment.failed`. Реестр — таблица `job_failures`. + +Временные ошибки (S3, обрыв, таймаут) делают backoff `min(30, 1 * 2^(attempt-1))` секунд плюс jitter до 20% и публикуют тот же `job_id` с новым `attempt`. После `JOB_MAX_ATTEMPTS=5` статус `failed`, код `retry_exhausted`. + +`POST /api/v1/attachments/{id}/replay` доступен участнику чата. Если статус уже `ready`, ответ 200 и новая работа не создаётся (запись `job_replays.outcome=ignored_ready`). Если `failed`, тот же job снова становится `queued`. Пока job в `queued` или `processing`, replay отвечает 409. + +Staging удаляется после коммита `ready`. Отдельный проход стирает оставшийся staging готовых объектов и staging `failed` старше `STAGING_RETENTION_SECONDS` (86400). Текущую очередь этот проход не трогает. + +В `APP_ENV=lab` имена `__lab_fault_transient*` и `__lab_fault_permanent*` — учебные инъекции временной и постоянной ошибки. `CRASH_AFTER_S3=1` останавливает процесс после записи объектов и до ack. + +## Доказательства + +| Критерий | Команда | Результат | Файл | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Публичный smoke: 202, ready, JPEG thumbnail 256×160, оригинал | `python3 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS | evidence/smoke.txt | +| Restart API, worker, PostgreSQL, Valkey, RustFS и RabbitMQ, тот же токен и тот же файл | `python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080` | PASS. Скрипт перезапускает `api worker postgres valkey s3 rabbitmq`, volumes не удаляет | evidence/persistence.txt, evidence/restart-times.txt | +| Остановка обслужившей реплики | `python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080` | 2 реплики, 140 успешных записей, 4 ошибки, первое восстановление 3.658 с, PASS | evidence/failover.txt | +| Worker остановлен, upload 202, content 409, чат жив, рестарт RabbitMQ с volume сохраняет 2 сообщения, затем ready | `python3 scripts/grade4_demo.py` | `broker restart kept 2 then 2 process messages`, id `59f221a0-7fe3-47ba-9b87-97ac966357b7` | evidence/grade4.txt | +| Crop 128×128 и WebP 320×200, replay ready не плодит варианты | тот же прогон | id `113efcbd-9dcf-46bd-9af8-17d1c927f517`, `replay=ignored_ready`, вариантов 3 | evidence/grade4.txt | +| Crash между COMMIT и publish: брокер и API остановлены после 202, worker поднимает outbox без нового upload | тот же прогон | ready в PostgreSQL до старта API, id `50db09ac-d826-47a3-899a-ada58734390a` | evidence/grade4.txt | +| Постоянная ошибка, DLQ, явный replay не создаёт успешный дубль | тот же прогон | `invalid_image`, очередь `attachment.failed` есть, id `8e345bcd-8900-4287-9a06-4bb504ee6ae2` | evidence/grade4.txt | +| Backoff и две временные ошибки, затем один набор вариантов | тот же прогон | `retries=2 final_attempt=3`, id `c266f9bb-1d43-453e-b4b3-f1f0ef29a112` | evidence/grade4.txt | +| Остановка после записи S3 и до ack; повтор доводит тот же результат | тот же прогон | статус после падения `processing`, затем ready, вариантов 3, staging очищен, id `e2c19231-f4ec-4d67-a89b-f94c5325d91f` | evidence/grade4.txt | +| Истёкший lease зависшего processing | тот же прогон | `lease_expired`, затем ready, id `17531de7-4616-45fb-ac94-9d45355f846d` | evidence/grade4.txt | -## Ограничения и следующие шаги +Контейнеры, которые реально перезапускает `scripts/restart.sh`: `api`, `worker`, `postgres`, `valkey`, `s3`, `rabbitmq`. `lb` скрипт не трогает. После прогона failover остановленная реплика была возвращена `scripts/restore-api.sh`, чтобы стенд не остался с одной копией. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +Уровень 5 не делался: нет сравнения prefetch под нагрузкой и нет versioned pipeline. Один worker, внутри него два потребителя. Пока worker лежит дольше lease, сверка может положить в очередь ещё одно wake-up сообщение; эффект от этого не дублируется. Учебные имена `__lab_fault_*` работают только при `APP_ENV=lab`. Один и тот же ключ из `.env` является корневым ключом RustFS и ключом API. Это локальный стенд. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..04268a18 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,281 @@ +# Локальный стенд lab5: клиент → Traefik → API ×2 → PostgreSQL + Valkey + RustFS. +# Задания: transactional outbox → RabbitMQ → отдельный worker. +# Публичный порт только у lb. +name: mtusi-lab5 + +services: + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - pgdata:/var/lib/postgresql/data + shm_size: 128mb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + stop_grace_period: 20s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + valkey: + image: valkey/valkey:8.0.2 + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 3s + restart: unless-stopped + stop_grace_period: 10s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + s3: + image: rustfs/rustfs:1.0.4 + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_VOLUMES: /data + RUSTFS_CHECK_UPDATE: "false" + volumes: + - s3data:/data + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + stop_grace_period: 15s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + + rabbitmq: + image: rabbitmq:3.13.7-alpine + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD in .env} + volumes: + - rabbitmqdata:/var/lib/rabbitmq + # diagnostics по умолчанию ждёт до 70 с. Короткий timeout Docker убивает + # этот процесс и вместе с ним останавливает сервер брокера. + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "--timeout", "5", "-q", "ping"] + interval: 10s + timeout: 10s + retries: 8 + start_period: 40s + restart: unless-stopped + stop_grace_period: 20s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.50" + memory: 768M + + migrate: + build: . + image: mtusi-messenger:lab5 + command: ["python3", "/app/migrate.py"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + restart: "no" + networks: [lab] + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + s3-init: + image: mtusi-messenger:lab5 + command: ["python3", "/app/s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + depends_on: + s3: + condition: service_healthy + restart: "no" + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + api: + image: mtusi-messenger:lab5 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: valkey + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://valkey:6379/0 + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + BROKER_URL: amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F + OUTBOX_PUBLISH_ENABLED: "1" + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + rabbitmq: + condition: service_healthy + expose: + - "8080" + labels: + - traefik.enable=true + - traefik.docker.network=mtusi-lab5-net + - traefik.http.routers.messenger.rule=PathPrefix(`/`) + - traefik.http.routers.messenger.entrypoints=web + - traefik.http.services.messenger.loadbalancer.server.port=8080 + - traefik.http.services.messenger.loadbalancer.healthcheck.path=/health/ready + - traefik.http.services.messenger.loadbalancer.healthcheck.interval=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.timeout=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.scheme=http + - traefik.http.services.messenger.loadbalancer.passhostheader=true + healthcheck: + test: + - CMD + - python3 + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) + interval: 2s + timeout: 2s + retries: 8 + start_period: 8s + init: true + restart: unless-stopped + stop_grace_period: 10s + stop_signal: SIGTERM + networks: [lab] + deploy: + replicas: 2 + resources: + limits: + cpus: "0.50" + memory: 512M + + worker: + image: mtusi-messenger:lab5 + command: ["python3", "/app/worker.py"] + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + BROKER_URL: amqp://${RABBITMQ_USER:-messenger}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-1} + JOB_BACKOFF_CAP_SECONDS: ${JOB_BACKOFF_CAP_SECONDS:-30} + JOB_BACKOFF_JITTER_RATIO: ${JOB_BACKOFF_JITTER_RATIO:-0.2} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + JPEG_QUALITY: ${JPEG_QUALITY:-85} + WEBP_QUALITY: ${WEBP_QUALITY:-80} + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + init: true + healthcheck: + disable: true + restart: unless-stopped + stop_grace_period: 20s + stop_signal: SIGTERM + networks: [lab] + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + lb: + image: traefik:v3.7.16 + command: ["--configFile=/etc/traefik/traefik.yml"] + volumes: + - ./infra/traefik.yml:/etc/traefik/traefik.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/ping"] + interval: 2s + timeout: 2s + retries: 8 + start_period: 3s + restart: unless-stopped + stop_grace_period: 5s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + +networks: + lab: + name: mtusi-lab5-net + driver: bridge + +volumes: + pgdata: + valkeydata: + s3data: + rabbitmqdata: diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..1b00e307 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 4, "first_recovery_seconds": 3.658} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/grade4.txt b/evidence/grade4.txt new file mode 100644 index 00000000..5240542c --- /dev/null +++ b/evidence/grade4.txt @@ -0,0 +1,8 @@ +happy-path ready id=113efcbd-9dcf-46bd-9af8-17d1c927f517 variants=3 replay=ignored_ready +broker restart kept 2 then 2 process messages +worker-down then broker restart kept queue; ready id=59f221a0-7fe3-47ba-9b87-97ac966357b7 +commit/publish gap recovered by worker without a new upload id=50db09ac-d826-47a3-899a-ada58734390a +permanent invalid_image + replay id=8e345bcd-8900-4287-9a06-4bb504ee6ae2 dlq present +backoff retries=2 final_attempt=3 id=c266f9bb-1d43-453e-b4b3-f1f0ef29a112 +crash after s3 before ack redelivery id=e2c19231-f4ec-4d67-a89b-f94c5325d91f variants=3 staging cleared +expired processing lease recovered id=17531de7-4616-45fb-ac94-9d45355f846d diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/restart-times.txt b/evidence/restart-times.txt new file mode 100644 index 00000000..b76e6fec --- /dev/null +++ b/evidence/restart-times.txt @@ -0,0 +1,20 @@ +BEFORE +mtusi-lab5-rabbitmq-1 Up Less than a second (health: starting) +mtusi-lab5-lb-1 Up Less than a second (health: starting) +mtusi-lab5-api-2 Up Less than a second (health: starting) +mtusi-lab5-api-1 Up Less than a second (health: starting) +mtusi-lab5-worker-1 Up Less than a second +mtusi-lab5-postgres-1 Up Less than a second (health: starting) +mtusi-lab5-s3-1 Up Less than a second (health: starting) +mtusi-lab5-valkey-1 Up Less than a second (health: starting) +worker concurrency=2 prefetch_per_channel=1 max_attempts=5 timeout=60s memory_limit=536870912 +AFTER +mtusi-lab5-rabbitmq-1 Up 2 seconds (health: starting) +mtusi-lab5-lb-1 Up 4 seconds (health: starting) +mtusi-lab5-api-2 Up Less than a second (health: starting) +mtusi-lab5-api-1 Up Less than a second (health: starting) +mtusi-lab5-worker-1 Up 3 seconds +mtusi-lab5-postgres-1 Up 3 seconds (health: starting) +mtusi-lab5-s3-1 Up 3 seconds (health: starting) +mtusi-lab5-valkey-1 Up 3 seconds (health: starting) +READY diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/examples/requests.http b/examples/requests.http index 052497b1..33bf88f6 100644 --- a/examples/requests.http +++ b/examples/requests.http @@ -45,9 +45,14 @@ Content-Type: application/json GET {{baseUrl}}/api/v1/chats/{{chatId}}/messages?limit=50 Authorization: Bearer {{sessionToken}} -### Binary upload; HTTP file-client syntax for reading local fixture +### Binary upload. Lab5 отвечает 202 queued; готовность смотреть отдельным GET POST {{baseUrl}}/api/v1/chats/{{chatId}}/attachments?filename=sample.png Authorization: Bearer {{sessionToken}} Content-Type: image/png < ../tests/fixtures/sample.png + +### Явный replay. Для ready это no-op и 200, для failed — снова queued и 202 +# @attachmentId = REPLACE_WITH_ATTACHMENT_UUID +POST {{baseUrl}}/api/v1/attachments/REPLACE_WITH_ATTACHMENT_UUID/replay +Authorization: Bearer {{sessionToken}} diff --git a/infra/traefik.yml b/infra/traefik.yml new file mode 100644 index 00000000..bcf20c19 --- /dev/null +++ b/infra/traefik.yml @@ -0,0 +1,45 @@ +# Traefik 3.7.16. Без sticky cookie. +# Адреса API Traefik берёт из Docker и обновляет по событиям start/die, +# поэтому новая реплика и остановленная не зависят от кэша одного DNS-ответа. +# Healthcheck: период 1s, timeout 1s, одна неуспешная проверка исключает backend. +# Верхняя оценка обнаружения отказа: 1s до следующей проверки + 1s timeout = 2s. +# Таймауты чтения запроса и ответа — 60s, чтобы прошёл оригинал до 10 MiB. +# Обнаружение мёртвой реплики от этого не зависит: healthcheck по-прежнему 1s+1s. + +global: + checkNewVersion: false + sendAnonymousUsage: false + +entryPoints: + web: + address: ":8080" + http: + aliasHeadersStrategy: reject + transport: + respondingTimeouts: + readTimeout: 60s + writeTimeout: 60s + idleTimeout: 30s + ping: + address: "127.0.0.1:8082" + http: + aliasHeadersStrategy: delete + +ping: + entryPoint: ping + +providers: + docker: + endpoint: unix:///var/run/docker.sock + exposedByDefault: false + network: mtusi-lab5-net + watch: true + +serversTransport: + forwardingTimeouts: + dialTimeout: 1s + responseHeaderTimeout: 60s + idleConnTimeout: 1s + +log: + level: INFO diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..3c2cc42f --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,49 @@ +-- Пользователи, чаты, участники, сообщения. +-- Сессии здесь не хранятся: их источник истины — Valkey (AOF + volume). +-- Идемпотентность отправки — в 002_idempotency.sql. +-- +-- Транзакции приложения: +-- регистрация — один INSERT, уникальность username держит UNIQUE +-- создание чата — одна транзакция: чат и все chat_members, иначе откат +-- сообщение и запись идемпотентности — одна транзакция +-- +-- Индекс истории messages_history_idx покрывает keyset-запрос +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX chats_created_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..bab92309 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,18 @@ +-- Общий результат POST сообщения. +-- Область: user_id + http_method + path + idem_key. +-- Строка пишется в той же транзакции, что и messages, поэтому 5xx +-- до COMMIT не оставляет ни сообщения, ни «окончательного» ключа. +-- Повтор с другим request_hash в пределах TTL — отказ без второй вставки. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, path, idem_key) +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..c21e8dcb --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,27 @@ +-- Метаданные вложения. Байты лежат в приватном S3-bucket, не в этой таблице. +-- object_key назначает сервер. filename только для отображения. +-- Связь сообщения и вложения — message_attachments, порядок задаёт position. + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT attachments_object_key_unique UNIQUE (object_key) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 00000000..6380c913 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,92 @@ +-- Lab5: состояние обработки, варианты и transactional outbox. +-- Байты в сообщения брокера не кладутся. Источник намерения — эта база. +-- Уже сохранённые вложения lab4 остаются ready: worker их не пересчитывает. + +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'ready', + ADD COLUMN error_code text, + ADD COLUMN updated_at timestamptz, + ADD COLUMN staging_key text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; + +UPDATE attachments SET updated_at = created_at WHERE updated_at IS NULL; + +ALTER TABLE attachments ALTER COLUMN updated_at SET NOT NULL; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + attempt integer NOT NULL, + status text NOT NULL, + error_code text, + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_attempt_check CHECK (attempt >= 1) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (status, next_attempt_at, lease_until); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0 +); + +CREATE UNIQUE INDEX outbox_one_unpublished + ON outbox (job_id) + WHERE published_at IS NULL; + +CREATE INDEX outbox_due_idx + ON outbox (available_at) + WHERE published_at IS NULL; + +CREATE TABLE job_failures ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + attempt integer NOT NULL, + error_code text NOT NULL, + permanent boolean NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX job_failures_attachment_idx + ON job_failures (attachment_id, created_at); + +CREATE TABLE job_replays ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + outcome text NOT NULL, + created_at timestamptz NOT NULL +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..f92e1c8c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +psycopg[binary]==3.2.3 +psycopg-pool==3.2.3 +redis==5.1.1 +minio==7.2.15 +pika==1.3.2 +Pillow==10.4.0 diff --git a/scripts/grade4_demo.py b/scripts/grade4_demo.py new file mode 100644 index 00000000..1df37627 --- /dev/null +++ b/scripts/grade4_demo.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Сценарии уровня 4. Стенд уже запущен. Секреты и токены не печатаются.""" +import base64 +import hashlib +import json +import struct +import subprocess +import sys +import time +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +BASE = "http://127.0.0.1:8080" + + +def compose(*args, check_code=True): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if check_code and proc.returncode != 0: + raise Failure(proc.stdout[-2000:]) + return proc + + +def psql(sql): + proc = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql) + return proc.stdout.strip() + + +def queues(): + proc = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages") + return proc.stdout + + +def queue_depth(name): + text = queues() + for line in text.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] == name and parts[-1].isdigit(): + return int(parts[-1]) + raise Failure("queue %s missing:\n%s" % (name, text)) + + +def wait_depth(name, minimum, seconds=20): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + last = queue_depth(name) + except Failure: + last = None + if last is not None and last >= minimum: + return last + time.sleep(0.5) + raise Failure("queue %s depth %s, want >= %s" % (name, last, minimum)) + + +def wait_status(client, actor, att, wanted, seconds=60): + deadline = time.monotonic() + seconds + last = att + while time.monotonic() < deadline: + last = client.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"] + if last["status"] == wanted: + return last + if wanted != "failed" and last["status"] == "failed": + raise Failure("attachment failed: " + str(last.get("error_code"))) + time.sleep(0.4) + raise Failure("status %s did not become %s" % (last.get("status"), wanted)) + + +def wait_postgres_ready(att_id, seconds=70): + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + last = psql("SELECT status FROM attachments WHERE id = '%s'" % att_id) + if last == "ready": + return + time.sleep(0.5) + raise Failure("postgres status stayed " + last) + + +def register(client): + username = "u_g4_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + user = client.call( + "POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Проверка 4", "password": password}, + schema="User", + )["json"] + raw = base64.b64encode((username + ":" + password).encode()).decode() + session = client.call( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + raw}, + schema="Session", + )["json"] + return {"id": user["id"], "token": session["token"], "user": user} + + +def upload(client, actor, chat_id, filename, content, content_type="image/png"): + return client.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename), + 202, + actor=actor, + raw=content, + headers={"Content-Type": content_type}, + schema="Attachment", + )["json"] + + +def bad_png(): + ihdr = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0) + chunk = b"IHDR" + ihdr + crc = zlib.crc32(chunk) & 0xFFFFFFFF + return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc) + + +def webp_size(data): + check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP") + index = 12 + while index + 8 <= len(data): + kind = data[index:index + 4] + size = int.from_bytes(data[index + 4:index + 8], "little") + chunk = data[index + 8:index + 8 + size] + if kind == b"VP8X" and len(chunk) >= 10: + return (1 + int.from_bytes(chunk[4:7], "little"), 1 + int.from_bytes(chunk[7:10], "little")) + if kind == b"VP8 " and len(chunk) >= 10: + return (int.from_bytes(chunk[6:8], "little") & 0x3FFF, int.from_bytes(chunk[8:10], "little") & 0x3FFF) + if kind == b"VP8L" and len(chunk) >= 5: + bits = int.from_bytes(chunk[1:5], "little") + return ((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1) + index += 8 + size + (size & 1) + raise Failure("WebP dimensions not found") + + +def variant_count(att_id): + return int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att_id)) + + +def main(): + sample = (ROOT / "tests/fixtures/sample.png").read_bytes() + client = Client(BASE, None) + client.call("GET", "/health/ready", schema="Health") + alice = register(client) + bob_name = "u_g4b_" + uuid.uuid4().hex[:12] + bob_password = "T3st-" + uuid.uuid4().hex + bob_user = client.call( + "POST", "/api/v1/users", 201, + body={"username": bob_name, "display_name": "Боб", "password": bob_password}, + schema="User", + )["json"] + chat = client.call( + "POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Очередь", "member_ids": [alice["id"], bob_user["id"]]}, + schema="Chat", + )["json"] + notes = [] + + att = upload(client, alice, chat["id"], "sample.png", sample) + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload snapshot") + ready = wait_status(client, alice, att, "ready") + names = [item["name"] for item in ready["variants"]] + check(names == ["thumbnail", "crop", "optimized"], "variant set " + ",".join(names)) + thumb = next(item for item in ready["variants"] if item["name"] == "thumbnail") + crop = next(item for item in ready["variants"] if item["name"] == "crop") + optimized = next(item for item in ready["variants"] if item["name"] == "optimized") + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail meta") + check((crop["width"], crop["height"]) == (128, 128) and crop["content_type"] == "image/jpeg", "crop meta") + check(optimized["content_type"] == "image/webp" and (optimized["width"], optimized["height"]) == (320, 200), "webp meta") + original = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice) + check(original["bytes"] == sample, "original bytes changed") + thumb_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=alice) + crop_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=alice) + webp_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=alice) + check(jpeg_size(thumb_bytes["bytes"]) == (256, 160), "thumbnail jpeg") + check(jpeg_size(crop_bytes["bytes"]) == (128, 128), "crop jpeg") + check(webp_size(webp_bytes["bytes"]) == (320, 200), "webp size") + check(hashlib.sha256(thumb_bytes["bytes"]).hexdigest() == thumb["sha256"], "thumb sha") + replay = client.call("POST", "/api/v1/attachments/%s/replay" % att["id"], 200, actor=alice, schema="Attachment")["json"] + check(replay["status"] == "ready" and replay["variants"] == ready["variants"], "ready replay changed variants") + check(variant_count(att["id"]) == 3, "replay created another variant set") + check(psql("SELECT outcome FROM job_replays WHERE attachment_id = '%s' ORDER BY created_at DESC LIMIT 1" % att["id"]) == "ignored_ready", "replay audit") + notes.append("happy-path ready id=%s variants=3 replay=ignored_ready" % att["id"]) + + compose("stop", "worker") + held = upload(client, alice, chat["id"], "while-worker-down.png", sample) + client.call("GET", "/api/v1/attachments/%s/content" % held["id"], 409, actor=alice) + client.call( + "POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice, + body={"text": "очередь не блокирует чат", "attachment_ids": [held["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Message", + ) + before = wait_depth("attachment.process", 1) + compose("restart", "rabbitmq") + for _ in range(40): + probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) + if probe.returncode == 0: + break + time.sleep(1) + else: + raise Failure("rabbitmq did not come back") + after = wait_depth("attachment.process", 1, seconds=30) + notes.append("broker restart kept %s then %s process messages" % (before, after)) + compose("start", "worker") + held_ready = wait_status(client, alice, held, "ready") + check(variant_count(held["id"]) == 3, "restarted job duplicated variants") + notes.append("worker-down then broker restart kept queue; ready id=%s" % held_ready["id"]) + + compose("stop", "worker") + compose("stop", "rabbitmq") + gap = upload(client, alice, chat["id"], "outbox-gap.png", sample) + unpublished = psql( + "SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id " + "WHERE j.attachment_id = '%s' AND o.published_at IS NULL" % gap["id"] + ) + check(unpublished == "1", "outbox was published while broker was down: " + unpublished) + compose("stop", "api") + compose("start", "rabbitmq") + for _ in range(40): + probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) + if probe.returncode == 0: + break + time.sleep(1) + compose("start", "worker") + wait_postgres_ready(gap["id"]) + ready_before_api = psql("SELECT status FROM attachments WHERE id = '%s'" % gap["id"]) + check(ready_before_api == "ready", "worker did not recover unpublished outbox") + compose("start", "api") + for _ in range(40): + try: + client.call("GET", "/health/ready", schema="Health") + break + except Failure: + time.sleep(1) + else: + raise Failure("api did not return") + gap_body = client.call("GET", "/api/v1/attachments/%s" % gap["id"], actor=alice, schema="Attachment")["json"] + check(gap_body["status"] == "ready" and len(gap_body["variants"]) == 3, "gap attachment not ready via api") + notes.append("commit/publish gap recovered by worker without a new upload id=%s" % gap["id"]) + + broken = upload(client, alice, chat["id"], "broken.png", bad_png()) + failed = wait_status(client, alice, broken, "failed") + check(failed["error_code"] == "invalid_image", "permanent code " + str(failed.get("error_code"))) + check(variant_count(broken["id"]) == 0, "failed job stored variants") + client.call( + "POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, + body={"text": "", "attachment_ids": [broken["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + again = client.call("POST", "/api/v1/attachments/%s/replay" % broken["id"], 202, actor=alice, schema="Attachment")["json"] + check(again["status"] == "queued", "replay of failed did not queue") + failed_again = wait_status(client, alice, broken, "failed") + check(failed_again["error_code"] == "invalid_image", "replay did not finish as permanent failure") + check(variant_count(broken["id"]) == 0, "replay of failed created variants") + dlq = queues() + check("attachment.failed" in dlq, "dlq missing") + notes.append("permanent invalid_image + replay id=%s dlq present" % broken["id"]) + + flaky = upload(client, alice, chat["id"], "__lab_fault_transient.png", sample) + flaky_ready = wait_status(client, alice, flaky, "ready", seconds=40) + failures = psql( + "SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND permanent = false AND error_code = 'unavailable'" + % flaky["id"] + ) + check(int(failures) >= 2, "expected two transient failures, got " + failures) + check(variant_count(flaky["id"]) == 3, "transient retries duplicated variants") + attempt = psql("SELECT attempt FROM attachment_jobs WHERE attachment_id = '%s'" % flaky["id"]) + notes.append("backoff retries=%s final_attempt=%s id=%s" % (failures, attempt, flaky_ready["id"])) + + compose("stop", "worker") + crash = upload(client, alice, chat["id"], "crash-before-ack.png", sample) + wait_depth("attachment.process", 1) + crashed = subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "run", "--rm", "--no-deps", "--name", "lab5-crash-once", "-e", "CRASH_AFTER_S3=1", "worker"], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=45, + ) + check(crashed.returncode != 0, "crash hook did not stop the worker") + status_after_crash = psql("SELECT status FROM attachments WHERE id = '%s'" % crash["id"]) + check(status_after_crash == "processing", "crash did not stop between claim and ready: " + status_after_crash) + compose("start", "worker") + crash_ready = wait_status(client, alice, crash, "ready") + check(variant_count(crash["id"]) == 3, "redelivery created a second variant set") + original_after = client.call("GET", "/api/v1/attachments/%s/content" % crash["id"], actor=alice) + check(original_after["bytes"] == sample, "redelivery changed the original") + staging_left = psql("SELECT staging_key IS NULL FROM attachments WHERE id = '%s'" % crash["id"]) + check(staging_left == "t", "staging was not cleared after ready") + notes.append("crash after s3 before ack redelivery id=%s variants=3 staging cleared" % crash_ready["id"]) + + compose("stop", "worker") + compose("stop", "rabbitmq") + leased = upload(client, alice, chat["id"], "expired-lease.png", sample) + psql( + "UPDATE attachment_jobs SET status = 'processing', attempt = 1, " + "lease_token = gen_random_uuid(), lease_until = now() - interval '2 minutes', " + "updated_at = now() WHERE attachment_id = '%s'" % leased["id"] + ) + psql("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '%s'" % leased["id"]) + psql( + "UPDATE outbox SET published_at = now() WHERE job_id = " + "(SELECT id FROM attachment_jobs WHERE attachment_id = '%s')" % leased["id"] + ) + compose("start", "rabbitmq") + for _ in range(40): + probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False) + if probe.returncode == 0: + break + time.sleep(1) + compose("start", "worker") + lease_ready = wait_status(client, alice, leased, "ready") + lease_rows = psql( + "SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND error_code = 'lease_expired'" + % leased["id"] + ) + check(int(lease_rows) >= 1, "expired lease was not recovered") + check(variant_count(leased["id"]) == 3, "lease recovery duplicated variants") + notes.append("expired processing lease recovered id=%s" % lease_ready["id"]) + + text = "\n".join(notes) + print(text) + evidence = ROOT / "evidence" / "grade4.txt" + evidence.write_text(text + "\n", encoding="utf-8") + print("PASS: grade 4 fault scenarios") + + +if __name__ == "__main__": + try: + main() + except (Failure, OSError, subprocess.SubprocessError) as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh old mode 100755 new mode 100644 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..1de6fd0d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Перезапуск API, worker и всех хранилищ, включая брокер. Volumes не удаляются. +docker compose --env-file .env -f compose.yaml restart api worker postgres valkey s3 rabbitmq diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100644 index 00000000..902ae572 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r') +if [ -z "${replicas}" ]; then + replicas=2 +fi +docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100644 index 00000000..15392e96 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env -f compose.yaml start worker diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh old mode 100755 new mode 100644 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..30465da0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +cd "$(dirname "$0")/.." + +found="" +for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$cid") + short=$(printf '%s' "$cid" | cut -c1-12) + if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then + found=$cid + break + fi +done + +if [ -z "$found" ]; then + echo "API replica was not found" >&2 + exit 1 +fi + +# docker stop оставляет реплику выключенной: restart policy unless-stopped. +docker stop -t 8 "$found" >/dev/null diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100644 index 00000000..8d869f86 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env -f compose.yaml stop worker diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100644 index 00000000..5d111aa2 --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if [ ! -f .env ]; then + echo "Copy .env.example to .env" >&2 + exit 1 +fi +replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r') +if [ -z "${replicas}" ]; then + replicas=2 +fi +docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..ef0f9e93 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,112 @@ +"""RabbitMQ: durable quorum-очереди и publisher confirms. + +Сообщение содержит только идентификаторы задания. Байты и ключи S3 сюда не входят. +Ack потребителя и confirm издателя — разные границы. +""" +from __future__ import annotations + +import json +import os +import threading +from pathlib import Path + +import pika + +EXCHANGE = "attachments" +PROCESS_QUEUE = "attachment.process" +FAILED_QUEUE = "attachment.failed" +PROCESS_KEY = "process" +FAILED_KEY = "failed" + +_lock = threading.Lock() +_connection: pika.BlockingConnection | None = None +_channel: pika.adapters.blocking_connection.BlockingChannel | None = None + + +def broker_url() -> str: + path = os.environ.get("BROKER_URL_FILE", "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("BROKER_URL", "").strip() + + +def declare(channel) -> None: + channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True) + arguments = {"x-queue-type": "quorum"} + channel.queue_declare(PROCESS_QUEUE, durable=True, arguments=arguments) + channel.queue_declare(FAILED_QUEUE, durable=True, arguments=arguments) + channel.queue_bind(PROCESS_QUEUE, EXCHANGE, PROCESS_KEY) + channel.queue_bind(FAILED_QUEUE, EXCHANGE, FAILED_KEY) + + +def connect() -> tuple[pika.BlockingConnection, pika.adapters.blocking_connection.BlockingChannel]: + url = broker_url() + if not url: + raise RuntimeError("broker is not configured") + params = pika.URLParameters(url) + params.heartbeat = 60 + params.blocked_connection_timeout = 15 + params.socket_timeout = 10 + params.connection_attempts = 3 + params.retry_delay = 1 + connection = pika.BlockingConnection(params) + channel = connection.channel() + channel.confirm_delivery() + declare(channel) + return connection, channel + + +def _ensure(): + global _connection, _channel + if _connection is not None and _connection.is_open and _channel is not None and _channel.is_open: + return _channel + if _connection is not None: + try: + _connection.close() + except Exception: + pass + _connection, _channel = connect() + return _channel + + +def publish_job(payload: dict, routing_key: str = PROCESS_KEY) -> None: + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(body) > 4096: + raise RuntimeError("job message exceeds 4096 bytes") + properties = pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(payload.get("job_id", "")), + type=str(payload.get("type", "")), + ) + with _lock: + channel = _ensure() + try: + channel.basic_publish( + EXCHANGE, + routing_key, + body, + properties, + mandatory=True, + ) + except Exception: + global _connection, _channel + _connection = None + _channel = None + raise + + +def publish_on(channel, payload: dict, routing_key: str) -> None: + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + channel.basic_publish( + EXCHANGE, + routing_key, + body, + pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(payload.get("job_id", "")), + type=str(payload.get("type", "")), + ), + mandatory=True, + ) diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..912e1379 --- /dev/null +++ b/src/db.py @@ -0,0 +1,1258 @@ +"""PostgreSQL: пользователи, чаты, участники, сообщения, идемпотентность.""" +from __future__ import annotations + +import hashlib +import os +import time +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import psycopg +from psycopg.errors import UniqueViolation +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import ApiError + +_pool: ConnectionPool | None = None + + +def migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here / "migrations", here.parent / "migrations"): + if candidate.is_dir(): + return candidate + return here.parent / "migrations" + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE", "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "").strip() + + +def idempotency_ttl() -> int: + raw = os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400") + try: + value = int(raw) + except ValueError: + return 86400 + return value if value >= 1 else 86400 + + +def init() -> None: + global _pool + url = database_url() + if not url: + raise RuntimeError("database is not configured") + _pool = ConnectionPool( + url, + min_size=0, + max_size=12, + timeout=3, + open=True, + kwargs={ + "connect_timeout": 3, + "autocommit": False, + "options": "-c statement_timeout=5000", + }, + ) + + +def migrate_with_retry() -> None: + last = None + for _ in range(30): + try: + migrate() + return + except Exception as exc: + last = exc + time.sleep(1) + raise RuntimeError("migrations failed") from last + + +def migrate() -> None: + url = database_url() + if not url: + raise RuntimeError("database is not configured") + with psycopg.connect(url, connect_timeout=3, autocommit=False) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (842021,)) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + applied = { + row[0] + for row in conn.execute("SELECT version FROM schema_migrations").fetchall() + } + for path in sorted(migrations_dir().glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + with conn.transaction(): + for statement in sql.split(";"): + if statement.strip(): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (path.name,), + ) + + +@contextmanager +def connection(): + if _pool is None: + raise ApiError(503, "unavailable", "База данных недоступна") + try: + with _pool.connection() as conn: + yield conn + except ApiError: + raise + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc: + raise ApiError(503, "unavailable", "База данных недоступна") from exc + + +def ping() -> bool: + try: + with connection() as conn: + conn.execute("SELECT 1") + return True + except ApiError: + return False + + +def close() -> None: + if _pool is not None: + _pool.close() + + +def user_json(row) -> dict: + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": format_ts(row[3]), + } + + +def format_ts(value: datetime) -> str: + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def create_user(username: str, display_name: str, password_hash: str, created: datetime) -> dict: + uid = uuid.uuid4() + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uid, username, display_name, password_hash, created), + ).fetchone() + except UniqueViolation as exc: + raise ApiError(409, "conflict", "username занят") from exc + return user_json(row) + + +def user_for_login(username: str): + with connection() as conn: + return conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + + +def get_user(user_id: str) -> dict | None: + with connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (uuid.UUID(user_id),), + ).fetchone() + if row is None: + return None + return user_json(row) + + +def create_chat(title: str, member_ids: list[str], created: datetime) -> dict: + cid = uuid.uuid4() + members = [uuid.UUID(item) for item in member_ids] + with connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (cid, title, created), + ) + found = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", + (members,), + ).fetchall() + if {row[0] for row in found} != set(members): + raise ApiError(400, "invalid_request", "неизвестный участник") + for position, member in enumerate(members): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES (%s, %s, %s) + """, + (cid, member, position), + ) + return { + "id": str(cid), + "title": title, + "member_ids": list(member_ids), + "created_at": format_ts(created), + } + + +def _chat_row(row) -> dict: + return { + "id": str(row[0]), + "title": row[1], + "created_at": format_ts(row[2]), + "member_ids": [str(item) for item in row[3]], + } + + +def get_chat(chat_id: str, user_id: str) -> dict: + with connection() as conn: + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id ORDER BY cm.position) + FROM chats c + JOIN chat_members mine + ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (uuid.UUID(user_id), uuid.UUID(chat_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + chat = _chat_row(row) + return { + "id": chat["id"], + "title": chat["title"], + "member_ids": chat["member_ids"], + "created_at": chat["created_at"], + } + + +def list_chats(user_id: str, limit: int, cursor) -> dict: + params: list = [uuid.UUID(user_id)] + where = "" + if cursor is not None: + created, chat_id = cursor + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params.extend([created, uuid.UUID(chat_id)]) + params.append(limit + 1) + sql = f""" + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id ORDER BY cm.position) + FROM chats c + JOIN chat_members mine + ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members cm ON cm.chat_id = c.id + {where} + GROUP BY c.id + ORDER BY c.created_at, c.id + LIMIT %s + """ + with connection() as conn: + rows = conn.execute(sql, params).fetchall() + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = (last[2], str(last[0])) + return { + "items": [ + { + "id": str(row[0]), + "title": row[1], + "member_ids": [str(item) for item in row[3]], + "created_at": format_ts(row[2]), + } + for row in page + ], + "next_cursor_parts": next_cursor, + } + + +def _lock_key(scope: str) -> int: + digest = hashlib.sha256(scope.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +def _is_member(conn, chat_id: str, user_id: str) -> bool: + row = conn.execute( + """ + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + """, + (uuid.UUID(chat_id), uuid.UUID(user_id)), + ).fetchone() + return row is not None + + +def _attachment_ids(value) -> list: + if not value: + return [] + return [str(item) for item in value] + + +def _load_message(conn, message_id) -> dict: + row = conn.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + '{}'::uuid[] + ) + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.id = %s + GROUP BY m.id + """, + (message_id,), + ).fetchone() + if row is None: + raise ApiError(500, "internal_error", "Внутренняя ошибка") + return message_json(row) + + +def _require_chat_attachments(conn, chat_id: str, attachment_ids: list[str]) -> None: + if not attachment_ids: + return + rows = conn.execute( + """ + SELECT id::text, chat_id::text, status + FROM attachments + WHERE id = ANY(%s::uuid[]) + """, + (attachment_ids,), + ).fetchall() + found = {row[0]: (row[1], row[2]) for row in rows} + for att_id in attachment_ids: + item = found.get(att_id) + if item is None or item[0] != chat_id: + raise ApiError(404, "not_found", "Вложение не найдено") + if item[1] == "failed": + raise ApiError(409, "conflict", "Нельзя прикрепить необработанное вложение") + + +def create_message_idempotent( + chat_id: str, + sender_id: str, + text: str, + attachment_ids: list[str], + created: datetime, + method: str, + path: str, + idem_key: str, + request_hash: str, +) -> dict: + """Один ключ — одно сообщение, в том числе при гонке двух реплик. + + pg_advisory_xact_lock сериализует одинаковую область. + INSERT сообщения, связей и ключа коммитятся вместе. + """ + scope = "%s|%s|%s|%s" % (sender_id, method, path, idem_key) + ttl = idempotency_ttl() + with connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s::bigint)", (_lock_key(scope),)) + existing = conn.execute( + """ + SELECT request_hash, message_id, created_at + FROM idempotency_keys + WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s + """, + (uuid.UUID(sender_id), method, path, idem_key), + ).fetchone() + if existing is not None: + age = conn.execute("SELECT now()").fetchone()[0] - existing[2] + if age.total_seconds() > ttl: + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s + """, + (uuid.UUID(sender_id), method, path, idem_key), + ) + existing = None + if existing is not None: + if existing[0] != request_hash: + raise ApiError(409, "conflict", "повторный ключ с другим сообщением") + if not _is_member(conn, chat_id, sender_id): + raise ApiError(404, "not_found", "Чат не найден") + return _load_message(conn, existing[1]) + if not _is_member(conn, chat_id, sender_id): + raise ApiError(404, "not_found", "Чат не найден") + _require_chat_attachments(conn, chat_id, attachment_ids) + mid = uuid.uuid4() + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (mid, uuid.UUID(chat_id), uuid.UUID(sender_id), text, created), + ) + for position, att_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + (mid, uuid.UUID(att_id), position), + ) + conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, http_method, path, idem_key, request_hash, message_id, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + (uuid.UUID(sender_id), method, path, idem_key, request_hash, mid, created), + ) + return _load_message(conn, mid) + + +def list_messages(chat_id: str, user_id: str, limit: int, cursor) -> dict: + params: list = [uuid.UUID(chat_id), uuid.UUID(chat_id), uuid.UUID(user_id)] + cursor_sql = "" + if cursor is not None: + created, message_id = cursor + cursor_sql = "AND (m.created_at, m.id) > (%s, %s)" + params.extend([created, uuid.UUID(message_id)]) + params.append(limit + 1) + sql = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + '{{}}'::uuid[] + ) + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.chat_id = %s + AND EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + ) + {cursor_sql} + GROUP BY m.id + ORDER BY m.created_at, m.id + LIMIT %s + """ + with connection() as conn: + rows = conn.execute(sql, params).fetchall() + if not rows: + with connection() as conn: + member = conn.execute( + """ + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + """, + (uuid.UUID(chat_id), uuid.UUID(user_id)), + ).fetchone() + if member is None: + raise ApiError(404, "not_found", "Чат не найден") + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = (last[4], str(last[0])) + return { + "items": [message_json(row) for row in page], + "next_cursor_parts": next_cursor, + } + + +def message_json(row) -> dict: + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": format_ts(row[4]), + "attachment_ids": _attachment_ids(row[5]), + } + + +def is_member(chat_id: str, user_id: str) -> bool: + try: + with connection() as conn: + return _is_member(conn, chat_id, user_id) + except ApiError: + raise + + +def insert_queued_attachment( + att_id: str, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, + digest: str, + object_key: str, + staging_key: str, + width: int, + height: int, + created: datetime, + job_payload: dict, +) -> dict: + """Оригинал уже в staging. Строка, job и outbox коммитятся одной транзакцией.""" + job_id = uuid.UUID(job_payload["job_id"]) + with connection() as conn: + with conn.transaction(): + if not _is_member(conn, chat_id, owner_id): + raise ApiError(404, "not_found", "Чат не найден") + conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, width, height, created_at, updated_at, status, + error_code, staging_key, pipeline_version + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + 'queued', NULL, %s, 1) + """, + ( + uuid.UUID(att_id), + uuid.UUID(chat_id), + uuid.UUID(owner_id), + filename, + content_type, + size_bytes, + digest, + object_key, + width, + height, + created, + created, + staging_key, + ), + ) + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, attempt, status, + created_at, updated_at + ) + VALUES (%s, %s, 1, %s, 'queued', %s, %s) + """, + (job_id, uuid.UUID(att_id), int(job_payload["attempt"]), created, created), + ) + _insert_outbox(conn, job_id, job_payload, created) + created_text = format_ts(created) + return { + "id": att_id, + "chat_id": chat_id, + "owner_id": owner_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": digest, + "status": "queued", + "variants": [], + "error_code": None, + "created_at": created_text, + "updated_at": created_text, + } + + +def get_attachment(att_id: str, user_id: str) -> dict: + with connection() as conn: + row = _attachment_row(conn, att_id, user_id) + variants = _variants(conn, att_id, row[11]) + return _attachment_public(row, variants) + + +def get_download(att_id: str, user_id: str, variant: str) -> dict: + with connection() as conn: + row = _attachment_row(conn, att_id, user_id) + status = row[7] + if status != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + if variant == "original": + return { + "content_type": row[4], + "size_bytes": row[5], + "sha256": row[6], + "object_key": row[12], + } + found = conn.execute( + """ + SELECT content_type, size_bytes, sha256, object_key + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (uuid.UUID(att_id), row[11], variant), + ).fetchone() + if found is None: + raise ApiError(404, "not_found", "Вариант не найден") + return { + "content_type": found[0], + "size_bytes": found[1], + "sha256": found[2], + "object_key": found[3], + } + + +def load_job_source(attachment_id: str) -> dict | None: + with connection() as conn: + row = conn.execute( + """ + SELECT a.id::text, a.filename, a.content_type, a.size_bytes, a.sha256, + a.width, a.height, a.object_key, a.staging_key, a.status, + a.pipeline_version, j.id::text, j.attempt, j.status + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = %s + """, + (uuid.UUID(attachment_id),), + ).fetchone() + if row is None: + return None + return { + "attachment_id": row[0], + "filename": row[1], + "content_type": row[2], + "size_bytes": row[3], + "sha256": row[4], + "width": row[5], + "height": row[6], + "object_key": row[7], + "staging_key": row[8], + "attachment_status": row[9], + "pipeline_version": row[10], + "job_id": row[11], + "attempt": row[12], + "job_status": row[13], + } + + +def _attachment_row(conn, att_id: str, user_id: str): + row = conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, + a.updated_at, a.pipeline_version, a.object_key + FROM attachments a + WHERE a.id = %s + AND EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = a.chat_id AND user_id = %s + ) + """, + (uuid.UUID(att_id), uuid.UUID(user_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +def _variants(conn, att_id: str, pipeline_version: int) -> list: + rows = conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY CASE name + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END + """, + (uuid.UUID(att_id), pipeline_version), + ).fetchall() + return [ + { + "name": row[0], + "content_type": row[1], + "size_bytes": row[2], + "width": row[3], + "height": row[4], + "sha256": row[5], + } + for row in rows + ] + + +def _attachment_public(row, variants: list) -> dict: + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "owner_id": str(row[2]), + "filename": row[3], + "content_type": row[4], + "size_bytes": row[5], + "sha256": row[6], + "status": row[7], + "variants": variants if row[7] == "ready" else [], + "error_code": row[8], + "created_at": format_ts(row[9]), + "updated_at": format_ts(row[10]), + } + + +def _insert_outbox(conn, job_id, payload: dict, available_at: datetime) -> None: + conn.execute( + """ + INSERT INTO outbox (id, job_id, payload, created_at, available_at) + VALUES (%s, %s, %s::jsonb, %s, %s) + ON CONFLICT (job_id) WHERE published_at IS NULL DO NOTHING + """, + ( + uuid.uuid4(), + job_id if isinstance(job_id, uuid.UUID) else uuid.UUID(str(job_id)), + json_dumps(payload), + available_at, + available_at, + ), + ) + + +def json_dumps(payload: dict) -> str: + import json + return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + + +def claim_job(job_id: str, lease_seconds: int) -> dict: + """Захватить работу. Готовый или окончательно failed job подтверждается без эффекта.""" + token = uuid.uuid4() + with connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', + lease_token = %s, + lease_until = now() + make_interval(secs => %s), + updated_at = now() + WHERE id = %s + AND status IN ('queued', 'processing') + AND ( + status = 'queued' + OR lease_until IS NULL + OR lease_until < now() + ) + RETURNING attempt, attachment_id::text, pipeline_version + """, + (token, int(lease_seconds), uuid.UUID(job_id)), + ).fetchone() + if row is not None: + conn.execute( + """ + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing') + """, + (uuid.UUID(row[1]), row[2]), + ) + return { + "action": "work", + "lease_token": str(token), + "attempt": row[0], + "attachment_id": row[1], + "pipeline_version": row[2], + } + state = conn.execute( + "SELECT status FROM attachment_jobs WHERE id = %s", + (uuid.UUID(job_id),), + ).fetchone() + if state is None or state[0] in ("ready", "failed"): + return {"action": "ack"} + return {"action": "requeue"} + + +def finish_job(job_id: str, lease_token: str, variants: list, pipeline_version: int) -> str: + """Атомарно записать варианты и ready. Чужой lease и уже готовый job не затираются.""" + with connection() as conn: + with conn.transaction(): + current = conn.execute( + """ + SELECT status, attachment_id::text, pipeline_version + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (uuid.UUID(job_id),), + ).fetchone() + if current is None: + return "missing" + if current[0] == "ready": + return "ready" + if current[2] != pipeline_version: + return "stale" + owned = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', + lease_token = NULL, + lease_until = NULL, + error_code = NULL, + next_attempt_at = NULL, + updated_at = now() + WHERE id = %s AND lease_token = %s AND status = 'processing' + RETURNING attachment_id::text + """, + (uuid.UUID(job_id), uuid.UUID(lease_token)), + ).fetchone() + if owned is None: + return "lost" + attachment_id = owned[0] + for item in variants: + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) + ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE + SET content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + WHERE attachment_variants.sha256 IS DISTINCT FROM EXCLUDED.sha256 + """, + ( + uuid.UUID(attachment_id), + pipeline_version, + item["name"], + item["content_type"], + item["size_bytes"], + item["width"], + item["height"], + item["sha256"], + item["object_key"], + ), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status <> 'failed' + """, + (uuid.UUID(attachment_id), pipeline_version), + ) + return "ready" + + +def fail_job(job_id: str, lease_token: str, error_code: str, attempt: int) -> str: + with connection() as conn: + with conn.transaction(): + owned = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', + error_code = %s, + lease_token = NULL, + lease_until = NULL, + next_attempt_at = NULL, + updated_at = now() + WHERE id = %s AND lease_token = %s AND status = 'processing' + RETURNING attachment_id::text, pipeline_version + """, + (error_code, uuid.UUID(job_id), uuid.UUID(lease_token)), + ).fetchone() + if owned is None: + state = conn.execute( + "SELECT status FROM attachment_jobs WHERE id = %s", + (uuid.UUID(job_id),), + ).fetchone() + if state is not None and state[0] == "ready": + return "ready" + return "lost" + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (error_code, uuid.UUID(owned[0])), + ) + conn.execute( + """ + INSERT INTO job_failures ( + id, job_id, attachment_id, attempt, error_code, permanent, created_at + ) + VALUES (%s, %s, %s, %s, %s, TRUE, now()) + """, + (uuid.uuid4(), uuid.UUID(job_id), uuid.UUID(owned[0]), attempt, error_code), + ) + return "failed" + + +def retry_job(job_id: str, lease_token: str, error_code: str, payload: dict, available_at: datetime, max_attempts: int) -> str: + with connection() as conn: + with conn.transaction(): + current = conn.execute( + """ + SELECT attempt, attachment_id::text, status + FROM attachment_jobs + WHERE id = %s AND lease_token = %s AND status = 'processing' + FOR UPDATE + """, + (uuid.UUID(job_id), uuid.UUID(lease_token)), + ).fetchone() + if current is None: + state = conn.execute( + "SELECT status FROM attachment_jobs WHERE id = %s", + (uuid.UUID(job_id),), + ).fetchone() + if state is not None and state[0] == "ready": + return "ready" + return "lost" + attempt = current[0] + attachment_id = current[1] + conn.execute( + """ + INSERT INTO job_failures ( + id, job_id, attachment_id, attempt, error_code, permanent, created_at + ) + VALUES (%s, %s, %s, %s, %s, FALSE, now()) + """, + (uuid.uuid4(), uuid.UUID(job_id), uuid.UUID(attachment_id), attempt, error_code), + ) + if attempt >= max_attempts: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = 'retry_exhausted', + lease_token = NULL, lease_until = NULL, updated_at = now() + WHERE id = %s + """, + (uuid.UUID(job_id),), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (uuid.UUID(attachment_id),), + ) + return "failed" + next_attempt = attempt + 1 + payload = dict(payload) + payload["attempt"] = next_attempt + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + attempt = %s, + lease_token = NULL, + lease_until = NULL, + error_code = %s, + next_attempt_at = %s, + updated_at = now() + WHERE id = %s + """, + (next_attempt, error_code, available_at, uuid.UUID(job_id)), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (uuid.UUID(attachment_id),), + ) + _insert_outbox(conn, job_id, payload, available_at) + return "retry" + + +def recover_expired_leases(max_attempts: int) -> int: + """Истёкший processing возвращается в очередь или становится failed. Эффект не дублируется.""" + changed = 0 + with connection() as conn: + with conn.transaction(): + rows = conn.execute( + """ + SELECT id::text, attachment_id::text, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE status = 'processing' AND lease_until < now() + FOR UPDATE SKIP LOCKED + """ + ).fetchall() + for job_id, attachment_id, pipeline_version, attempt, created_at in rows: + conn.execute( + """ + INSERT INTO job_failures ( + id, job_id, attachment_id, attempt, error_code, permanent, created_at + ) + VALUES (%s, %s, %s, %s, 'lease_expired', FALSE, now()) + """, + (uuid.uuid4(), uuid.UUID(job_id), uuid.UUID(attachment_id), attempt), + ) + if attempt >= max_attempts: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = 'retry_exhausted', + lease_token = NULL, lease_until = NULL, updated_at = now() + WHERE id = %s + """, + (uuid.UUID(job_id),), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', updated_at = now() + WHERE id = %s AND status = 'processing' + """, + (uuid.UUID(attachment_id),), + ) + else: + next_attempt = attempt + 1 + payload = { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": pipeline_version, + "attempt": next_attempt, + "created_at": format_ts(created_at), + } + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, lease_token = NULL, + lease_until = NULL, next_attempt_at = now(), updated_at = now() + WHERE id = %s + """, + (next_attempt, uuid.UUID(job_id)), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = %s AND status = 'processing' + """, + (uuid.UUID(attachment_id),), + ) + _insert_outbox(conn, job_id, payload, datetime.now(timezone.utc)) + changed += 1 + return changed + + +def reconcile_queued(stale_seconds: int) -> int: + """Если wake-up потерялся, создать ещё один outbox. Повторная доставка остаётся идемпотентной.""" + inserted = 0 + with connection() as conn: + with conn.transaction(): + rows = conn.execute( + """ + SELECT j.id::text, j.attachment_id::text, j.pipeline_version, j.attempt, j.created_at + FROM attachment_jobs j + WHERE j.status = 'queued' + AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= now()) + AND NOT EXISTS ( + SELECT 1 FROM outbox o + WHERE o.job_id = j.id AND o.published_at IS NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM outbox o + WHERE o.job_id = j.id + AND o.published_at > now() - make_interval(secs => %s) + ) + FOR UPDATE OF j SKIP LOCKED + """, + (int(stale_seconds),), + ).fetchall() + now = datetime.now(timezone.utc) + for job_id, attachment_id, pipeline_version, attempt, created_at in rows: + payload = { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": format_ts(created_at), + } + _insert_outbox(conn, job_id, payload, now) + inserted += 1 + return inserted + + +def take_outbox(limit: int) -> list: + with connection() as conn: + with conn.transaction(): + rows = conn.execute( + """ + WITH picked AS ( + SELECT id + FROM outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at + FOR UPDATE SKIP LOCKED + LIMIT %s + ) + UPDATE outbox AS o + SET publish_attempts = o.publish_attempts + 1 + FROM picked + WHERE o.id = picked.id + RETURNING o.id::text, o.payload + """, + (int(limit),), + ).fetchall() + items = [] + for row_id, payload in rows: + if isinstance(payload, str): + import json + payload = json.loads(payload) + items.append((row_id, payload)) + return items + + +def mark_outbox_published(outbox_id: str) -> None: + with connection() as conn: + with conn.transaction(): + conn.execute( + "UPDATE outbox SET published_at = now() WHERE id = %s AND published_at IS NULL", + (uuid.UUID(outbox_id),), + ) + + +def clear_staging_key(attachment_id: str) -> None: + with connection() as conn: + with conn.transaction(): + conn.execute( + "UPDATE attachments SET staging_key = NULL WHERE id = %s", + (uuid.UUID(attachment_id),), + ) + + +def staging_ready_for_cleanup(limit: int) -> list: + with connection() as conn: + rows = conn.execute( + """ + SELECT id::text, staging_key + FROM attachments + WHERE status = 'ready' AND staging_key IS NOT NULL + LIMIT %s + """, + (int(limit),), + ).fetchall() + return [(row[0], row[1]) for row in rows] + + +def expired_failed_staging(retention_seconds: int, limit: int) -> list: + with connection() as conn: + rows = conn.execute( + """ + SELECT id::text, staging_key + FROM attachments + WHERE status = 'failed' + AND staging_key IS NOT NULL + AND updated_at < now() - make_interval(secs => %s) + LIMIT %s + """, + (int(retention_seconds), int(limit)), + ).fetchall() + return [(row[0], row[1]) for row in rows] + + +def replay_attachment(att_id: str, user_id: str) -> dict: + """Явный replay. Уже ready job не запускается повторно.""" + with connection() as conn: + with conn.transaction(): + row = _attachment_row(conn, att_id, user_id) + status = row[7] + pipeline_version = row[11] + if status == "ready": + conn.execute( + """ + INSERT INTO job_replays (id, job_id, attachment_id, outcome, created_at) + SELECT %s, j.id, a.id, 'ignored_ready', now() + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = %s + """, + (uuid.uuid4(), uuid.UUID(att_id)), + ) + variants = _variants(conn, att_id, pipeline_version) + return _attachment_public(row, variants) + if status in ("queued", "processing"): + raise ApiError(409, "conflict", "Задание уже выполняется") + job = conn.execute( + """ + SELECT id::text, attempt, created_at + FROM attachment_jobs + WHERE attachment_id = %s AND pipeline_version = %s + FOR UPDATE + """, + (uuid.UUID(att_id), pipeline_version), + ).fetchone() + if job is None: + raise ApiError(404, "not_found", "Задание не найдено") + if job[1] >= 1000: + raise ApiError(409, "conflict", "Слишком много повторов") + next_attempt = job[1] + 1 + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, error_code = NULL, + lease_token = NULL, lease_until = NULL, next_attempt_at = now(), + updated_at = now() + WHERE id = %s AND status = 'failed' + """, + (next_attempt, uuid.UUID(job[0])), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = %s AND status = 'failed' + """, + (uuid.UUID(att_id),), + ) + payload = { + "schema_version": 1, + "job_id": job[0], + "type": "attachment.process.v1", + "attachment_id": att_id, + "pipeline_version": pipeline_version, + "attempt": next_attempt, + "created_at": format_ts(job[2]), + } + _insert_outbox(conn, job[0], payload, datetime.now(timezone.utc)) + conn.execute( + """ + INSERT INTO job_replays (id, job_id, attachment_id, outcome, created_at) + VALUES (%s, %s, %s, 'queued', now()) + """, + (uuid.uuid4(), uuid.UUID(job[0]), uuid.UUID(att_id)), + ) + return get_attachment(att_id, user_id) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..c8aee871 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,8 @@ +"""Ошибки API. Текст не содержит SQL, паролей и токенов.""" + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status = status + self.code = code + self.message = message diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..2abbb177 --- /dev/null +++ b/src/images.py @@ -0,0 +1,74 @@ +"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей.""" +from __future__ import annotations + +from errors import ApiError + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC. +_JPEG_SOF = { + 0xC0, 0xC1, 0xC2, 0xC3, + 0xC5, 0xC6, 0xC7, + 0xC9, 0xCA, 0xCB, + 0xCD, 0xCE, 0xCF, +} + + +def sniff(data: bytes) -> str: + if data.startswith(PNG_SIG): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG") + + +def dimensions(data: bytes, content_type: str) -> tuple[int, int]: + if content_type == "image/png": + return _png_size(data) + if content_type == "image/jpeg": + return _jpeg_size(data) + raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG") + + +def _png_size(data: bytes) -> tuple[int, int]: + if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR": + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG") + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + if width < 1 or height < 1: + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG") + return width, height + + +def _jpeg_size(data: bytes) -> tuple[int, int]: + if len(data) < 4 or data[:2] != b"\xff\xd8": + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + index = 2 + size = len(data) + while index < size: + if data[index] != 0xFF: + break + while index < size and data[index] == 0xFF: + index += 1 + if index >= size: + break + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9) or marker == 0xDA: + break + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if index + 2 > size: + break + segment = int.from_bytes(data[index:index + 2], "big") + if segment < 2 or index + segment > size: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + if marker in _JPEG_SOF: + if segment < 7: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + if width < 1 or height < 1: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + return width, height + index += segment + raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..3a0f8bf9 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Однократные миграции. Реплики API этот файл не запускают.""" +import sys + +import db + + +def main() -> None: + db.migrate_with_retry() + sys.stderr.write("migrations applied\n") + + +if __name__ == "__main__": + main() diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 00000000..305eb35c --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,43 @@ +"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах.""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import secrets + +N = 2**14 +R = 8 +P = 1 +DKLEN = 32 + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN, + maxmem=64 * 1024 * 1024) + return "scrypt$%d$%d$%d$%s$%s" % ( + N, R, P, + base64.b64encode(salt).decode("ascii"), + base64.b64encode(digest).decode("ascii"), + ) + + +def verify_password(password: str, stored: str) -> bool: + try: + algo, n, r, p, salt_b64, digest_b64 = stored.split("$") + if algo != "scrypt": + return False + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(digest_b64) + actual = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p), + dklen=len(expected), maxmem=64 * 1024 * 1024) + except (ValueError, TypeError): + return False + return hmac.compare_digest(actual, expected) + + +# Одинаковая стоимость проверки, когда пользователя нет. +DUMMY_HASH = hash_password("dummy-password-not-used") diff --git a/src/relay.py b/src/relay.py new file mode 100644 index 00000000..866180a7 --- /dev/null +++ b/src/relay.py @@ -0,0 +1,24 @@ +"""Доставка outbox в брокер. Строка остаётся неопубликованной, пока нет confirm.""" +from __future__ import annotations + +import os +import sys + +import broker +import db + + +def publish_batch(limit: int = 20) -> int: + if os.environ.get("OUTBOX_PUBLISH_ENABLED", "1") == "0": + return 0 + rows = db.take_outbox(limit) + published = 0 + for outbox_id, payload in rows: + try: + broker.publish_job(payload) + except Exception: + sys.stderr.write("broker did not confirm job %s\n" % payload.get("job_id", "")) + return published + db.mark_outbox_published(outbox_id) + published += 1 + return published diff --git a/src/s3_init.py b/src/s3_init.py new file mode 100644 index 00000000..c9ec2bda --- /dev/null +++ b/src/s3_init.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Однократное создание приватного bucket. Повторный запуск ничего не публикует.""" +import sys +import time + +import storage + + +def main() -> None: + storage.init() + last = None + for _ in range(30): + try: + storage.ensure_bucket() + sys.stderr.write("bucket ready\n") + return + except Exception as exc: + last = exc + time.sleep(1) + sys.stderr.write("bucket bootstrap failed\n") + raise SystemExit(1) from last + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..c1cb3a89 --- /dev/null +++ b/src/server.py @@ -0,0 +1,730 @@ +#!/usr/bin/env python3 +"""Messenger lab5. Приём файла отделён от обработки: staging, outbox, брокер, worker.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import secrets +import signal +import socket +import socketserver +import sys +import threading +import time +import unicodedata +import uuid +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse + +import db +import images +import passwords +import sessions +import storage +from errors import ApiError + +APP_HOST = os.environ.get("APP_HOST", "0.0.0.0") +APP_PORT = int(os.environ.get("APP_PORT", "8080")) +APP_ENV = os.environ.get("APP_ENV", "lab") +INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z") +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z") +TIMESTAMP_RE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z") +MAX_BODY = 1024 * 1024 + +_draining = threading.Event() +_inflight = 0 +_inflight_lock = threading.Lock() + + +def drain_seconds() -> float: + raw = os.environ.get("DRAIN_SECONDS", "3") + try: + value = float(raw) + except ValueError: + return 3.0 + if value < 0: + return 0.0 + if value > 6: + return 6.0 + return value + + +def now_utc() -> datetime: + current = datetime.now(timezone.utc) + return current.replace(microsecond=(current.microsecond // 1000) * 1000) + + +def is_blank(value: str) -> bool: + return value.strip() == "" + + +def require_object(body): + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом") + + +def reject_extra(body: dict, allowed: set): + extra = set(body) - allowed + if extra: + raise ApiError(400, "invalid_request", + "Лишние поля: " + ", ".join(sorted(extra))) + + +def parse_uuid(value, field: str) -> str: + if not isinstance(value, str) or not UUID_RE.match(value): + raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}") + return value.lower() + + +def parse_display_name(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "display_name должен быть строкой") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "display_name: 1..100 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "display_name не может быть пустым") + return value + + +def parse_username(value) -> str: + if not isinstance(value, str) or not USERNAME_RE.match(value): + raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$") + return value + + +def parse_password(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "password должен быть строкой") + if not (12 <= len(value) <= 128): + raise ApiError(400, "invalid_request", "password: 12..128 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "password не может быть пустым") + return value + + +def parse_title(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "title должен быть строкой") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "title: 1..100 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "title не может быть пустым") + return value + + +def max_upload_bytes() -> int: + raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760") + try: + value = int(raw) + except ValueError: + return 10485760 + return value if value >= 1 else 10485760 + + +def max_image_pixels() -> int: + raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000") + try: + value = int(raw) + except ValueError: + return 40000000 + return value if value >= 1 else 40000000 + + +def parse_text(value, has_attachments: bool) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "text должен быть строкой") + if len(value) > 4000: + raise ApiError(400, "invalid_request", "text: не больше 4000 code points") + if is_blank(value) and not has_attachments: + raise ApiError(400, "invalid_request", "text не может быть пустым без вложений") + return value + + +def parse_attachment_ids(value) -> list: + if value is None: + return [] + if not isinstance(value, list): + raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом") + if len(value) > 10: + raise ApiError(400, "invalid_request", "attachment_ids: не больше 10") + seen = set() + result = [] + for item in value: + att_id = parse_uuid(item, "attachment_ids[]") + if att_id in seen: + raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты") + seen.add(att_id) + result.append(att_id) + return result + + +def parse_filename(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "filename: 1..255 code points") + if not (1 <= len(value) <= 255): + raise ApiError(400, "invalid_request", "filename: 1..255 code points") + for char in value: + if char in "/\\" or unicodedata.category(char) == "Cc": + raise ApiError(400, "invalid_request", "filename содержит запрещённый символ") + return value + + +def parse_member_ids(value) -> list: + if not isinstance(value, list): + raise ApiError(400, "invalid_request", "member_ids должен быть массивом") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "member_ids: 1..100") + seen = set() + result = [] + for item in value: + uid = parse_uuid(item, "member_ids[]") + if uid in seen: + raise ApiError(400, "invalid_request", "member_ids содержит дубликаты") + seen.add(uid) + result.append(uid) + return result + + +def parse_limit(raw, default=50) -> int: + if raw is None: + return default + if isinstance(raw, str) and raw.strip() == "": + raise ApiError(400, "invalid_request", "limit должен быть целым") + try: + value = int(raw) + except (TypeError, ValueError): + raise ApiError(400, "invalid_request", "limit должен быть целым") + if not (1 <= value <= 100): + raise ApiError(400, "invalid_request", "limit: 1..100") + return value + + +def parse_optional_cursor(raw): + if raw is None: + return None + if not isinstance(raw, str) or raw.strip() == "": + raise ApiError(400, "invalid_request", "Некорректный cursor") + return raw + + +def parse_cursor_timestamp(value: str) -> datetime: + if not isinstance(value, str) or not TIMESTAMP_RE.match(value): + raise ApiError(400, "invalid_request", "Некорректный cursor") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + return parsed + + +def parse_idempotency_key(headers) -> str: + raw = headers.get("Idempotency-Key") + if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw): + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return raw + + +def canonical_hash(text: str, attachment_ids: list) -> str: + # Порядок ключей JSON не важен. Отсутствующий attachment_ids равен []. + # Порядок элементов массива сохраняется: это часть тела запроса. + payload = {"attachment_ids": attachment_ids, "text": text} + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def challenge_for(path: str) -> str: + if path == "/api/v1/auth/sessions": + return 'Basic realm="messenger", charset="UTF-8"' + return 'Bearer realm="messenger"' + + +def bearer_token(headers) -> str: + auth = headers.get("Authorization") or "" + parts = auth.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip(): + raise ApiError(401, "unauthorized", "Требуется Bearer") + return parts[1].strip() + + +def current_user(headers) -> dict: + token = bearer_token(headers) + user = db.get_user(sessions.user_id(token)) + if user is None: + raise ApiError(401, "unauthorized", "Сессия недействительна") + return user + + +def basic_credentials(headers) -> tuple[str, str]: + auth = headers.get("Authorization") or "" + parts = auth.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip(): + raise ApiError(401, "unauthorized", "Требуется Basic") + try: + decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные") + username, password = decoded.split(":", 1) + return username, password + + +def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str: + raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str, chat_id: str): + pad = "=" * (-len(cursor) % 4) + try: + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except Exception as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + parts = raw.split("|") + if len(parts) != 3: + raise ApiError(400, "invalid_request", "Некорректный cursor") + c_chat, c_created, c_id = parts + if not UUID_RE.match(c_chat): + raise ApiError(400, "invalid_request", "Некорректный cursor") + if c_chat.lower() != chat_id: + raise ApiError(400, "invalid_request", "cursor от другого чата") + return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor") + + +def encode_chat_cursor(created_at: str, chat_id: str) -> str: + raw = f"chat|{created_at}|{chat_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_chat_cursor(cursor: str): + pad = "=" * (-len(cursor) % 4) + try: + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except Exception as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + parts = raw.split("|") + if len(parts) != 3 or parts[0] != "chat": + raise ApiError(400, "invalid_request", "Некорректный cursor") + return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor") + + +def create_user(body) -> dict: + require_object(body) + reject_extra(body, {"username", "display_name", "password"}) + username = parse_username(body.get("username")) + display_name = parse_display_name(body.get("display_name")) + password = parse_password(body.get("password")) + return db.create_user(username, display_name, passwords.hash_password(password), now_utc()) + + +def create_session(headers) -> dict: + username, password = basic_credentials(headers) + row = None + try: + row = db.user_for_login(username) + except ApiError: + raise + stored = row[3] if row is not None else passwords.DUMMY_HASH + password_ok = passwords.verify_password(password, stored) + if row is None or not password_ok: + raise ApiError(401, "unauthorized", "Неверные учётные данные") + token = secrets.token_urlsafe(32) + ttl = sessions.session_ttl() + expires = now_utc() + timedelta(seconds=ttl) + sessions.create(token, str(row[0]), ttl) + return { + "token": token, + "token_type": "Bearer", + "expires_at": db.format_ts(expires), + "user": db.user_json((row[0], row[1], row[2], row[4])), + } + + +def logout(headers) -> None: + token = bearer_token(headers) + sessions.user_id(token) + sessions.revoke(token) + + +def create_chat(body, actor: dict) -> dict: + require_object(body) + reject_extra(body, {"title", "member_ids"}) + title = parse_title(body.get("title")) + member_ids = parse_member_ids(body.get("member_ids")) + if actor["id"] not in member_ids: + raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids") + return db.create_chat(title, member_ids, now_utc()) + + +def list_chats(actor: dict, limit: int, cursor) -> dict: + decoded = decode_chat_cursor(cursor) if cursor else None + page = db.list_chats(actor["id"], limit, decoded) + next_cursor = None + if page["next_cursor_parts"] is not None: + created, chat_id = page["next_cursor_parts"] + next_cursor = encode_chat_cursor(db.format_ts(created), chat_id) + return {"items": page["items"], "next_cursor": next_cursor} + + +def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict: + cid = parse_uuid(chat_id, "chat_id") + require_object(body) + reject_extra(body, {"text", "attachment_ids"}) + attachment_ids = parse_attachment_ids(body.get("attachment_ids")) + text = parse_text(body.get("text"), bool(attachment_ids)) + path = "/api/v1/chats/%s/messages" % cid + return db.create_message_idempotent( + cid, + actor["id"], + text, + attachment_ids, + now_utc(), + "POST", + path, + idem_key, + canonical_hash(text, attachment_ids), + ) + + +def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict: + detected = images.sniff(data) + if detected != declared: + raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type") + width, height = images.dimensions(data, detected) + if width * height > max_image_pixels(): + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + att_id = str(uuid.uuid4()) + created = now_utc() + staging_key = "staging/" + att_id + object_key = "originals/" + att_id + payload = { + "schema_version": 1, + "job_id": str(uuid.uuid4()), + "type": "attachment.process.v1", + "attachment_id": att_id, + "pipeline_version": 1, + "attempt": 1, + "created_at": db.format_ts(created), + } + storage.put_bytes(staging_key, data, detected) + try: + return db.insert_queued_attachment( + att_id, chat_id, actor["id"], filename, detected, len(data), + hashlib.sha256(data).hexdigest(), object_key, staging_key, + width, height, created, payload, + ) + except Exception: + storage.delete_quiet(staging_key) + raise + + +def read_attachment(attachment_id: str, actor: dict) -> dict: + return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"]) + + +def read_attachment_bytes(attachment_id: str, actor: dict, variant: str): + if variant not in ("original", "thumbnail", "crop", "optimized"): + raise ApiError(400, "invalid_request", "Некорректный variant") + record = db.get_download(parse_uuid(attachment_id, "attachment_id"), actor["id"], variant) + data = storage.get_bytes(record["object_key"]) + if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]: + raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными") + return record["content_type"], data + + +def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict: + cid = parse_uuid(chat_id, "chat_id") + decoded = decode_cursor(cursor, cid) if cursor else None + page = db.list_messages(cid, actor["id"], limit, decoded) + next_cursor = None + if page["next_cursor_parts"] is not None: + created, message_id = page["next_cursor_parts"] + next_cursor = encode_cursor(cid, db.format_ts(created), message_id) + return {"items": page["items"], "next_cursor": next_cursor} + + +class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + daemon_threads = True + block_on_close = False + allow_reuse_address = True + request_queue_size = 256 + + +class Handler(BaseHTTPRequestHandler): + server_version = "mtusi-messenger/5.0" + protocol_version = "HTTP/1.0" + + def log_message(self, fmt, *args): + return + + def _request_id(self) -> str: + return self.headers.get("X-Request-Id") or str(uuid.uuid4()) + + def _send_json(self, status: int, payload, request_id: str, extra_headers=None): + body = b"" + if status != 204: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + if status != 204: + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("Connection", "close") + for key, value in (extra_headers or {}).items(): + self.send_header(key, value) + self.end_headers() + if body: + self.wfile.write(body) + + def _send_bytes(self, content_type: str, payload: bytes, request_id: str): + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("Connection", "close") + self.end_headers() + if payload: + self.wfile.write(payload) + + def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) + + def _read_body(self): + length = self.headers.get("Content-Length") + if not length: + return None + try: + size = int(length) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc + if size > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело слишком большое") + raw = self.rfile.read(size) + ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip() + if ctype and ctype != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + if not raw: + raise ApiError(400, "invalid_request", "Пустое тело") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ApiError(400, "invalid_request", "Malformed JSON") from exc + + def _read_upload(self) -> tuple[str, bytes]: + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Нужен Content-Length") + try: + size = int(length) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc + if size > max_upload_bytes(): + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + if size < 1: + raise ApiError(415, "unsupported_media_type", "Пустое тело") + declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() + if declared not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") + data = self.rfile.read(size) + if len(data) != size: + raise ApiError(400, "invalid_request", "Тело короче Content-Length") + return declared, data + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + global _inflight + with _inflight_lock: + _inflight += 1 + rid = self._request_id() + parsed = urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = parse_qs(parsed.query, keep_blank_values=True) + try: + self._route(method, path, query, rid) + except ApiError as exc: + headers = {} + if exc.status == 401: + headers["WWW-Authenticate"] = challenge_for(path) + self._send_error(exc.status, exc.code, exc.message, rid, headers) + except BrokenPipeError: + pass + except Exception: + sys.stderr.write("internal error on %s %s\n" % (method, path)) + self._send_error(500, "internal_error", "Внутренняя ошибка", rid) + finally: + with _inflight_lock: + _inflight -= 1 + + def _route(self, method: str, path: str, query: dict, rid: str): + if method == "GET" and path == "/health/live": + return self._send_json(200, {"status": "ok"}, rid) + if method == "GET" and path == "/health/ready": + if _draining.is_set(): + raise ApiError(503, "unavailable", "Реплика останавливается") + if db.ping() and sessions.ping() and storage.ping(): + return self._send_json(200, {"status": "ok"}, rid) + raise ApiError(503, "unavailable", "Зависимость недоступна") + if _draining.is_set(): + raise ApiError(503, "unavailable", "Реплика останавливается") + + if method == "POST" and path == "/api/v1/users": + return self._send_json(201, create_user(self._read_body()), rid) + if method == "POST" and path == "/api/v1/auth/sessions": + return self._send_json(201, create_session(self.headers), rid) + if method == "DELETE" and path == "/api/v1/auth/sessions/current": + logout(self.headers) + return self._send_json(204, None, rid) + if method == "GET" and path == "/api/v1/users/me": + return self._send_json(200, current_user(self.headers), rid) + + if method == "POST" and path == "/api/v1/chats": + actor = current_user(self.headers) + return self._send_json(201, create_chat(self._read_body(), actor), rid) + if method == "GET" and path == "/api/v1/chats": + actor = current_user(self.headers) + limit = parse_limit(query.get("limit", [None])[0]) + cursor = parse_optional_cursor(query.get("cursor", [None])[0]) + return self._send_json(200, list_chats(actor, limit, cursor), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)", path) + if match and method == "GET": + actor = current_user(self.headers) + return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path) + if match and method == "POST": + actor = current_user(self.headers) + chat_id = parse_uuid(match.group(1), "chat_id") + if not db.is_member(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Чат не найден") + names = query.get("filename") or [] + if len(names) != 1: + raise ApiError(400, "invalid_request", "Нужен один filename") + filename = parse_filename(names[0]) + declared, data = self._read_upload() + body = upload_attachment(chat_id, actor, filename, declared, data) + return self._send_json(202, body, rid) + + match = re.fullmatch(r"/api/v1/attachments/([^/]+)/replay", path) + if match and method == "POST": + actor = current_user(self.headers) + att_id = parse_uuid(match.group(1), "attachment_id") + body = db.replay_attachment(att_id, actor["id"]) + status = 200 if body["status"] == "ready" else 202 + return self._send_json(status, body, rid) + + match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path) + if match and method == "GET": + actor = current_user(self.headers) + variants = query.get("variant") or ["original"] + if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "": + raise ApiError(400, "invalid_request", "Некорректный variant") + content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0]) + return self._send_bytes(content_type, payload, rid) + + match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path) + if match and method == "GET": + actor = current_user(self.headers) + return self._send_json(200, read_attachment(match.group(1), actor), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path) + if match and method == "POST": + actor = current_user(self.headers) + idem_key = parse_idempotency_key(self.headers) + body = create_message(match.group(1), self._read_body(), actor, idem_key) + return self._send_json(201, body, rid) + if match and method == "GET": + actor = current_user(self.headers) + limit = parse_limit(query.get("limit", [None])[0]) + cursor = parse_optional_cursor(query.get("cursor", [None])[0]) + return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid) + + raise ApiError(404, "not_found", "Маршрут не найден") + + +def _inflight_now() -> int: + with _inflight_lock: + return _inflight + + +def _publish_outbox() -> None: + import relay + while not _draining.is_set(): + try: + relay.publish_batch() + except Exception: + sys.stderr.write("outbox publish failed\n") + _draining.wait(0.4) + + +def main(): + try: + db.init() + except Exception: + sys.stderr.write("database is not configured\n") + try: + sessions.init() + except Exception: + sys.stderr.write("session store is not configured\n") + try: + storage.init() + except Exception: + sys.stderr.write("object storage is not configured\n") + threading.Thread(target=_publish_outbox, name="outbox", daemon=True).start() + + server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler) + + def drain_and_stop(): + time.sleep(drain_seconds()) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and _inflight_now() > 0: + time.sleep(0.05) + server.shutdown() + + def shutdown(signum, frame): + if _draining.is_set(): + return + _draining.set() + sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID) + threading.Thread(target=drain_and_stop, daemon=True).start() + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + sys.stderr.write( + "listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID) + ) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + db.close() + sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID) + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..6e7bc918 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,85 @@ +"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии. + +Источник истины сессий — общий Valkey, не память процесса API. +AOF и volume сохраняют ключ при перезапуске хранилища. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import redis + +from errors import ApiError + +_client: redis.Redis | None = None + + +def session_ttl() -> int: + raw = os.environ.get("SESSION_TTL_SECONDS", "3600") + try: + value = int(raw) + except ValueError: + return 3600 + return value if value >= 1 else 1 + + +def redis_url() -> str: + path = os.environ.get("CACHE_URL_FILE", "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("REDIS_URL", "").strip() + + +def init() -> None: + global _client + url = redis_url() + if not url: + _client = None + return + _client = redis.Redis.from_url( + url, socket_connect_timeout=0.5, socket_timeout=0.5, + decode_responses=True) + + +def _key(token: str) -> str: + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "session:" + digest + + +def _require() -> redis.Redis: + if _client is None: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") + return _client + + +def ping() -> bool: + try: + return bool(_require().ping()) + except (ApiError, redis.RedisError): + return False + + +def create(token: str, user_id: str, ttl: int) -> None: + try: + _require().set(_key(token), user_id, ex=ttl) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc + + +def user_id(token: str) -> str: + try: + value = _require().get(_key(token)) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc + if not value: + raise ApiError(401, "unauthorized", "Сессия недействительна") + return value + + +def revoke(token: str) -> None: + try: + _require().delete(_key(token)) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 00000000..3412b4b4 --- /dev/null +++ b/src/settings.py @@ -0,0 +1,70 @@ +"""Численные пределы worker. Секреты здесь не читаются.""" +from __future__ import annotations + +import os +import random + + +def _int(name: str, default: int, low: int, high: int) -> int: + try: + value = int(os.environ.get(name, str(default))) + except ValueError: + return default + if value < low: + return low + if value > high: + return high + return value + + +def _float(name: str, default: float, low: float, high: float) -> float: + try: + value = float(os.environ.get(name, str(default))) + except ValueError: + return default + if value < low: + return low + if value > high: + return high + return value + + +def max_attempts() -> int: + return _int("JOB_MAX_ATTEMPTS", 5, 1, 20) + + +def job_timeout() -> int: + return _int("JOB_TIMEOUT_SECONDS", 60, 5, 600) + + +def worker_concurrency() -> int: + return _int("WORKER_CONCURRENCY", 2, 1, 8) + + +def max_image_pixels() -> int: + return _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000) + + +def max_upload_bytes() -> int: + return _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760) + + +def staging_retention() -> int: + return _int("STAGING_RETENTION_SECONDS", 86400, 60, 7 * 86400) + + +def jpeg_quality() -> int: + return _int("JPEG_QUALITY", 85, 40, 95) + + +def webp_quality() -> int: + return _int("WEBP_QUALITY", 80, 40, 95) + + +def backoff_delay(attempt: int) -> float: + """Экспоненциальная пауза с jitter. attempt — номер уже совершённой попытки.""" + base = _float("JOB_BACKOFF_BASE_SECONDS", 1.0, 0.1, 60.0) + cap = _float("JOB_BACKOFF_CAP_SECONDS", 30.0, 0.1, 300.0) + ratio = _float("JOB_BACKOFF_JITTER_RATIO", 0.2, 0.0, 1.0) + delay = min(cap, base * (2 ** max(0, attempt - 1))) + return delay + random.uniform(0, delay * ratio) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..a2a03dd2 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,118 @@ +"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент.""" +from __future__ import annotations + +import io +import os +import sys +from pathlib import Path + +from minio import Minio + +from errors import ApiError + +_client: Minio | None = None + + +def _read_setting(env_name: str, file_env: str) -> str: + path = os.environ.get(file_env, "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get(env_name, "").strip() + + +def endpoint() -> str: + return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip() + + +def bucket() -> str: + return os.environ.get("S3_BUCKET", "messenger-attachments").strip() + + +def _host_and_secure() -> tuple[str, bool]: + raw = endpoint() + secure = raw.startswith("https://") + host = raw.split("://", 1)[-1].rstrip("/") + return host, secure + + +def init() -> None: + global _client + host, secure = _host_and_secure() + access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if not host or not access or not secret or not bucket(): + raise RuntimeError("object storage is not configured") + _client = Minio( + host, + access_key=access, + secret_key=secret, + secure=secure, + region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + ) + + +def _api() -> Minio: + if _client is None: + raise ApiError(503, "unavailable", "Хранилище недоступно") + return _client + + +def ping() -> bool: + try: + return _api().bucket_exists(bucket()) + except Exception: + return False + + +def ensure_bucket() -> None: + api = _api() + name = bucket() + if not api.bucket_exists(name): + api.make_bucket(name) + + +def put_bytes(key: str, data: bytes, content_type: str) -> None: + try: + _api().put_object( + bucket(), + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + ) + except ApiError: + raise + except Exception as exc: + sys.stderr.write("s3 put failed\n") + raise ApiError(503, "unavailable", "Хранилище недоступно") from exc + + +def get_bytes(key: str) -> bytes: + response = None + try: + response = _api().get_object(bucket(), key) + return response.read() + except ApiError: + raise + except Exception as exc: + sys.stderr.write("s3 get failed\n") + raise ApiError(503, "unavailable", "Хранилище недоступно") from exc + finally: + if response is not None: + response.close() + response.release_conn() + + +def exists(key: str) -> bool: + try: + _api().stat_object(bucket(), key) + return True + except Exception: + return False + + +def delete_quiet(key: str) -> None: + try: + _api().remove_object(bucket(), key) + except Exception: + sys.stderr.write("s3 delete failed\n") diff --git a/src/variants.py b/src/variants.py new file mode 100644 index 00000000..528c2175 --- /dev/null +++ b/src/variants.py @@ -0,0 +1,124 @@ +"""Варианты изображения. Оригинал этот модуль не перекодирует. + +thumbnail: JPEG, вписать в 256×256 без увеличения, пропорции сохраняются, +размеры округляются вниз до минимума 1 px. EXIF orientation применяется до +масштаба. Альфа кладётся на белый. Качество JPEG_QUALITY (по умолчанию 85), +optimize, без EXIF/ICC. + +crop: JPEG тех же параметров, центральный квадрат, затем ровно 128×128. +Маленький исходник можно увеличить. + +optimized: WebP, вписать в 1280×1280 без увеличения. Качество WEBP_QUALITY +(по умолчанию 80), method=6. EXIF, ICC и XMP в файл не пишутся. Меньший размер +маленького файла не обещается. +""" +from __future__ import annotations + +import io + +from PIL import Image, ImageOps, UnidentifiedImageError + +import settings + +THUMBNAIL_LIMIT = 256 +CROP_SIZE = 128 +OPTIMIZED_LIMIT = 1280 + + +class PermanentJobError(Exception): + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +class TransientJobError(Exception): + def __init__(self, code: str = "unavailable"): + super().__init__(code) + self.code = code + + +def _open(data: bytes) -> Image.Image: + if len(data) > settings.max_upload_bytes(): + raise PermanentJobError("payload_too_large") + Image.MAX_IMAGE_PIXELS = settings.max_image_pixels() + try: + image = Image.open(io.BytesIO(data)) + width, height = image.size + if width < 1 or height < 1 or width * height > settings.max_image_pixels(): + raise PermanentJobError("payload_too_large") + image = ImageOps.exif_transpose(image) + image.load() + except PermanentJobError: + raise + except Image.DecompressionBombError as exc: + raise PermanentJobError("payload_too_large") from exc + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc: + raise PermanentJobError("invalid_image") from exc + return image + + +def _on_white(image: Image.Image) -> Image.Image: + if image.mode == "RGB": + return image + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + + +def _fit(image: Image.Image, limit: int) -> Image.Image: + width, height = image.size + if width <= limit and height <= limit: + return image + scale = min(limit / width, limit / height) + resized = (max(1, int(width * scale)), max(1, int(height * scale))) + return image.resize(resized, Image.Resampling.LANCZOS) + + +def _crop(image: Image.Image) -> Image.Image: + width, height = image.size + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + cropped = image.crop((left, top, left + side, top + side)) + if cropped.size != (CROP_SIZE, CROP_SIZE): + cropped = cropped.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg(image: Image.Image) -> bytes: + buffer = io.BytesIO() + _on_white(image).save( + buffer, + format="JPEG", + quality=settings.jpeg_quality(), + optimize=True, + progressive=False, + ) + return buffer.getvalue() + + +def _webp(image: Image.Image) -> bytes: + buffer = io.BytesIO() + _on_white(image).save( + buffer, + format="WEBP", + quality=settings.webp_quality(), + method=6, + ) + return buffer.getvalue() + + +def render(data: bytes) -> dict: + image = _open(data) + thumb = _fit(image, THUMBNAIL_LIMIT) + crop = _crop(image) + optimized = _fit(image, OPTIMIZED_LIMIT) + thumb_bytes = _jpeg(thumb) + crop_bytes = _jpeg(crop) + optimized_bytes = _webp(optimized) + return { + "thumbnail": (thumb_bytes, "image/jpeg", thumb.size), + "crop": (crop_bytes, "image/jpeg", crop.size), + "optimized": (optimized_bytes, "image/webp", optimized.size), + } diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..64ef76e8 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize.""" +from __future__ import annotations + +import hashlib +import json +import os +import sys +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone + +import broker +import db +import relay +import settings +import storage +import variants +from variants import PermanentJobError, TransientJobError + +_stop = threading.Event() + + +def _validate(payload) -> dict: + if not isinstance(payload, dict): + raise ValueError("payload") + allowed = { + "schema_version", "job_id", "type", "attachment_id", "pipeline_version", + "attempt", "created_at", "traceparent", "tracestate", + } + if set(payload) - allowed: + raise ValueError("extra") + if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1": + raise ValueError("type") + for field in ("job_id", "attachment_id"): + uuid.UUID(str(payload.get(field))) + if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1: + raise ValueError("attempt") + if not isinstance(payload.get("created_at"), str): + raise ValueError("created_at") + return payload + + +def _lab_fault(source: dict, attempt: int) -> None: + if os.environ.get("APP_ENV", "lab") != "lab": + return + name = source["filename"] + if name.startswith("__lab_fault_permanent"): + raise PermanentJobError("invalid_image") + if name.startswith("__lab_fault_transient") and attempt < 3: + raise TransientJobError("unavailable") + + +def _produce(source: dict, attempt: int) -> list: + _lab_fault(source, attempt) + if source["size_bytes"] > settings.max_upload_bytes(): + raise PermanentJobError("payload_too_large") + if source["width"] * source["height"] > settings.max_image_pixels(): + raise PermanentJobError("payload_too_large") + staging_key = source["staging_key"] + if not staging_key: + if storage.exists(source["object_key"]): + original = storage.get_bytes(source["object_key"]) + else: + raise TransientJobError("unavailable") + else: + original = storage.get_bytes(staging_key) + if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]: + raise TransientJobError("unavailable") + rendered = variants.render(original) + storage.put_bytes(source["object_key"], original, source["content_type"]) + stored = [] + for name, (data, content_type, size) in rendered.items(): + key = "variants/%s/%s" % (source["attachment_id"], name) + storage.put_bytes(key, data, content_type) + stored.append({ + "name": name, + "content_type": content_type, + "size_bytes": len(data), + "width": size[0], + "height": size[1], + "sha256": hashlib.sha256(data).hexdigest(), + "object_key": key, + }) + if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"): + sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"]) + os._exit(86) + return stored + + +def _retry(payload: dict, claim: dict, code: str) -> str: + delay = settings.backoff_delay(claim["attempt"]) + available = datetime.now(timezone.utc) + timedelta(seconds=delay) + sys.stderr.write( + "retry job=%s attempt=%s delay=%.3f code=%s\n" + % (payload["job_id"], claim["attempt"], delay, code) + ) + return db.retry_job( + payload["job_id"], + claim["lease_token"], + code, + payload, + available, + settings.max_attempts(), + ) + + +def _dlq(channel, payload: dict) -> None: + try: + broker.publish_on(channel, payload, broker.FAILED_KEY) + except Exception: + sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id")) + + +def _cleanup(source: dict) -> None: + key = source.get("staging_key") + if not key: + return + storage.delete_quiet(key) + db.clear_staging_key(source["attachment_id"]) + + +def _work(channel, payload: dict) -> None: + claim = db.claim_job(payload["job_id"], settings.job_timeout()) + if claim["action"] == "ack": + return + if claim["action"] != "work": + time.sleep(0.4) + raise TransientJobError("lease_held") + source = db.load_job_source(payload["attachment_id"]) + if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready": + return + if int(payload["pipeline_version"]) != int(source["pipeline_version"]): + return + try: + produced = _produce(source, claim["attempt"]) + outcome = db.finish_job( + payload["job_id"], claim["lease_token"], produced, source["pipeline_version"], + ) + except PermanentJobError as exc: + outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"]) + if outcome == "failed": + _dlq(channel, payload) + return + except TransientJobError as exc: + if exc.code == "lease_held": + raise + outcome = _retry(payload, claim, exc.code) + if outcome == "failed": + _dlq(channel, payload) + return + except Exception as exc: + sys.stderr.write("transient processing error %s\n" % type(exc).__name__) + outcome = _retry(payload, claim, "unavailable") + if outcome == "failed": + _dlq(channel, payload) + return + if outcome == "lost": + raise TransientJobError("lease_lost") + if outcome == "ready": + _cleanup(source) + + +def _on_message(channel, method, _properties, body: bytes) -> None: + try: + payload = _validate(json.loads(body.decode("utf-8"))) + except Exception: + sys.stderr.write("dropping invalid job message\n") + channel.basic_ack(method.delivery_tag) + return + try: + _work(channel, payload) + except TransientJobError: + channel.basic_nack(method.delivery_tag, requeue=True) + return + except Exception: + sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id")) + channel.basic_nack(method.delivery_tag, requeue=True) + return + channel.basic_ack(method.delivery_tag) + + +def _consume() -> None: + connection, channel = broker.connect() + channel.basic_qos(prefetch_count=1) + channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False) + channel.start_consuming() + + +def _consume_loop() -> None: + while not _stop.is_set(): + try: + _consume() + except Exception: + if _stop.is_set(): + return + sys.stderr.write("consumer reconnect\n") + time.sleep(1) + + +def _sweep() -> None: + while not _stop.is_set(): + try: + expired = db.recover_expired_leases(settings.max_attempts()) + restored = db.reconcile_queued(settings.job_timeout()) + if expired or restored: + sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored)) + for attachment_id, key in db.staging_ready_for_cleanup(40): + storage.delete_quiet(key) + db.clear_staging_key(attachment_id) + for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40): + storage.delete_quiet(key) + db.clear_staging_key(attachment_id) + relay.publish_batch() + except Exception: + sys.stderr.write("sweeper failed\n") + _stop.wait(2) + + +def _memory_limit() -> str: + for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"): + try: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + except OSError: + continue + return "unknown" + + +def main() -> None: + db.init() + storage.init() + sys.stderr.write( + "worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n" + % ( + settings.worker_concurrency(), + settings.max_attempts(), + settings.job_timeout(), + _memory_limit(), + ) + ) + threading.Thread(target=_sweep, name="sweep", daemon=True).start() + threads = [] + for index in range(settings.worker_concurrency()): + thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True) + thread.start() + threads.append(thread) + try: + while any(thread.is_alive() for thread in threads): + time.sleep(0.5) + except KeyboardInterrupt: + _stop.set() + + +if __name__ == "__main__": + main()