ЛР 5 - БПИ2303 - Любченко - на 4 #76
@@ -23,3 +23,8 @@ WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
PUBLIC_PORT=18082
|
||||
|
||||
# Maximum consecutive processing attempts before the job moves to failed.
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# MTUSI Messenger, lab5 — PostgreSQL + Redis/Valkey HTTP API.
|
||||
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
|
||||
# on another machine with the same platform yields the same layers.
|
||||
#
|
||||
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
|
||||
# Development-only tooling lives in requirements-dev.txt and never enters this image.
|
||||
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app/src
|
||||
|
||||
# Pinned runtime dependencies (versions fixed for reproducible builds).
|
||||
RUN pip install --no-cache-dir \
|
||||
psycopg2-binary==2.9.11 \
|
||||
redis==7.0.1 \
|
||||
bcrypt==4.0.1 \
|
||||
boto3==1.42.97 \
|
||||
Pillow==12.3.0 \
|
||||
pika==1.3.2
|
||||
|
||||
# Unprivileged runtime account; the image's default root user is never used at runtime.
|
||||
RUN groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
ARG APP_REVISION=local
|
||||
ENV APP_REVISION=$APP_REVISION
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=10001:10001 src/ /app/src/
|
||||
COPY --chown=10001:10001 migrations/ /app/migrations/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
# Configuration comes from the environment; secrets are read from files at runtime.
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_GRACE_SECONDS=8 \
|
||||
SESSION_TTL_SECONDS=3600 \
|
||||
SESSION_STORE=redis \
|
||||
DB_POOL_MIN=2 \
|
||||
DB_POOL_MAX=10 \
|
||||
DB_CONNECT_TIMEOUT=5 \
|
||||
DB_STATEMENT_TIMEOUT=10
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,6 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
PUBLIC_PORT ?= $(or $(shell sed -n 's/^PUBLIC_PORT=//p' .env 2>/dev/null),18082)
|
||||
BASE_URL ?= http://localhost:$(PUBLIC_PORT)
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
@@ -1,37 +1,117 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор — Любченко Никита, БПИ2303.
|
||||
- Целевая оценка — 4.
|
||||
- Использованы Python 3.12, Pillow, boto3, pika, PostgreSQL, Redis, Nginx, MinIO и RabbitMQ. ИИ использован при разработке и проверке реализации.
|
||||
- Результаты прогонов и их ограничения приведены ниже. Ссылку на PR добавляет автор.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Engine с Compose, Make и Python 3.9+ для публичных тестов. Первая сборка обращается к реестрам образов, GitHub, PyPI и прокси Go для зависимостей MinIO.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp -n .env.example .env
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait --wait-timeout 120
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Стенд открывается на http://localhost:18082. Порт задаёт PUBLIC_PORT. Секреты создаёт bootstrap. Команда `docker compose down` сохраняет тома. `docker compose down -v` удаляет учебные данные PostgreSQL, Redis, MinIO и RabbitMQ.
|
||||
|
||||
## Архитектура и согласованность
|
||||
|
||||
Две API-реплики за Nginx используют PostgreSQL для метаданных и MinIO для исходников. Redis сохраняет роль кэша сессий. До ответа 202 оригинал сохранён в staging, а метаданные, задача и outbox зафиксированы в одной транзакции PostgreSQL. Отдельный издатель использует устойчивую очередь RabbitMQ и publisher confirms. При повторной публикации сохраняются ID задачи и номер попытки. Повтор завершённой работы не меняет метаданные.
|
||||
|
||||
Worker забирает задачи с prefetch 1. После перехода в processing он получает оригинал по ключу из БД и создаёт три варианта. Файлы каждой попытки имеют отдельный префикс. Только текущая попытка с действующим lease фиксирует ready и варианты одной транзакцией. Consumer ACK следует после устойчивого решения по задаче. До ready скачивание возвращает 409. Участник чата может отправить сообщение с queued-вложением.
|
||||
|
||||
Worker ограничен 1 CPU и 768 MiB, таймер обработки составляет 20 с. Lease действует 60 с. Временные ошибки и истечение аренды обрабатываются через ограниченный backoff с jitter. Некорректный файл сразу переводит задачу в failed. Операторский replay сохраняет аудит. Cleaner удаляет staging после ready, а для failed сохраняет его на 24 часа. Старые результаты отдельных попыток удаляются после периода ожидания.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Результат | Артефакт |
|
||||
| --- | --- | --- |
|
||||
| Структура и OpenAPI | PASS | `evidence/check.txt`, `evidence/validate.txt` |
|
||||
| Перенос lab4 | Текстовый и файловый сценарий, общая сессия, приватный MinIO | `evidence/lab4-baseline.txt` |
|
||||
| Устойчивый ответ 202 и привязка до готовности | 202 queued, 409 при скачивании, одна транзакция задачи и outbox | `evidence/queued-upload.txt` |
|
||||
| Форматы, размеры и побайтный оригинал | Thumbnail 256×160, crop 128×128, WebP 320×200, повтор не меняет результат. Задача передана обработчику напрямую | `evidence/worker-variants.txt` |
|
||||
| Retry, failed, replay, очистка | Проверены на PostgreSQL и MinIO. Истечение lease смоделировано после записи файлов | `evidence/job-lifecycle.txt` |
|
||||
| Откат при отказе publish | Запись осталась неопубликованной на реальном PostgreSQL после имитации NACK | `evidence/outbox-nack.txt` |
|
||||
| Публичный smoke через RabbitMQ | PASS | `evidence/worker-smoke.txt` |
|
||||
| Разрыв между COMMIT и publish | Задача, принятая при остановленном издателе, опубликована после его запуска | `evidence/outbox-live.txt` |
|
||||
| Остановка worker и рестарт API/RabbitMQ | Та же задача дошла до ready, исходник совпал побайтно | `evidence/broker-resilience.txt` |
|
||||
| Остановка worker после записи в S3 до COMMIT/ACK | После истечения lease восстановление заняло 60,90 с; опубликованы три варианта одного результата | `evidence/broker-resilience.txt` |
|
||||
| Публичный persistence и failover | PASS, данные и действующая сессия сохранены | `evidence/persistence.txt`, `evidence/failover.txt` |
|
||||
| Приватность бакета | Чтение с учётной записью API прошло, анонимный GET получил 403 | `evidence/s3-private-final.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Сценарии защиты
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
```sh
|
||||
docker compose stop worker
|
||||
sh student-tests/outbox.sh
|
||||
python3 student-tests/broker_resilience.py --base-url http://localhost:18082
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 scripts/manage.py restore
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Сценарий outbox останавливает издателя и подтверждает публикацию после его запуска. Его нужно запускать с остановленным worker. Сценарий broker_resilience останавливает worker, проверяет переживание рестарта API и RabbitMQ, затем убивает отдельный тестовый worker после записи в S3 и до COMMIT/ACK. Оба сценария рассчитаны на учебный стенд и создают синтетические данные. Проверка отдельной попытки использует fixture PNG 320×200.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Прогон выполнен в Docker Engine 29.5.3, Compose 5.1.4, aarch64. Повторные проверки с актуальным кодом сохраняются в файлах `*-final.txt`. После повторной загрузки образа RabbitMQ 4.1.4 запущен. Публичный smoke и подтверждённая публикация прошли на реальном брокере.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## История реализации
|
||||
|
||||
Записи ниже отражают состояние проверок на соответствующем этапе. Итоговые результаты приведены в таблице выше.
|
||||
|
||||
## Этап 1. Перенос lab4
|
||||
|
||||
Проект `mtusi-lab5` слушает `http://localhost:18082`. PostgreSQL, Redis и MinIO получили отдельные тома. Локальные секреты сгенерированы для lab5.
|
||||
|
||||
```sh
|
||||
cp -n .env.example .env
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build
|
||||
make check
|
||||
```
|
||||
|
||||
Для проверки переноса использовались существующие образы lab4 с локальными тегами lab5. Публичный smoke lab4 прошёл на новом стенде. Отдельно проверены сессия на обеих репликах и отсутствие анонимного доступа к объекту в MinIO. Результаты сохранены в `evidence/lab4-baseline.txt`. На этом этапе upload ещё отвечает 201 по контракту lab4. Smoke lab5 рассчитан на переход к 202 и обработку в worker.
|
||||
|
||||
## Этап 2. Устойчивый приём вложения
|
||||
|
||||
Миграция `0004_attachment_jobs.sql` добавляет состояния обработки и таблицы задач, вариантов и outbox. После успешного PUT под `attachments/staging/` API в одной транзакции сохраняет задачу, событие outbox и состояние `queued`. До готовности вложение можно прикрепить к сообщению, а скачивание возвращает 409.
|
||||
|
||||
Проверка на Compose с PostgreSQL и MinIO: `python3 student-tests/queued_upload.py --base-url http://localhost:18082`. SQL дополнительно подтвердил одну задачу и одну неопубликованную запись outbox. Результаты в `evidence/queued-upload.txt`. Из-за TLS-таймаута реестра для прогона поверх локального API-образа lab4 скопированы актуальные исходники и миграции lab5.
|
||||
|
||||
## Этап 3. RabbitMQ и публикация outbox
|
||||
|
||||
В Compose добавлены RabbitMQ 4.1.4 с постоянным томом и отдельный outbox-publisher. Издатель берёт готовую строку outbox через FOR UPDATE SKIP LOCKED, публикует постоянное сообщение в durable-очередь и только после publisher confirm записывает published_at. При сбое между confirm и коммитом возможна повторная доставка с тем же job_id.
|
||||
|
||||
После доступной загрузки образа запустить `docker compose build api-a`, затем `docker compose up -d --no-build` и `sh student-tests/outbox.sh`. Сценарий останавливает издателя, создаёт вложение, проверяет неопубликованную запись и ждёт подтверждения после запуска издателя. Worker появится на следующем этапе.
|
||||
|
||||
Конфигурация Compose и make check прошли. На настоящем PostgreSQL проверен откат после имитации broker NACK, см. evidence/outbox-nack.txt. Прогон с реальным RabbitMQ пока открыт: загрузка образа не удалась из-за ошибок доступа к реестрам.
|
||||
|
||||
## Этап 4. Worker и производные изображения
|
||||
|
||||
Worker обрабатывает по одной доставке (prefetch 1) с лимитом контейнера 768 MiB и таймером обработки 20 с. Он скачивает исходник из staging с проверкой размера и SHA-256, повторно валидирует изображение и сохраняет побайтный оригинал. Производные файлы записываются в ключи, зависящие от ID вложения и версии обработки. JPEG использует качество 85, WebP — 82. EXIF-ориентация применяется перед преобразованием, метаданные исходника не копируются, прозрачность сводится на белый фон.
|
||||
|
||||
На этом этапе блокировка строки PostgreSQL защищает обработку от дублирования. Статус ready и три строки вариантов фиксируются одной транзакцией после записи всех файлов. ACK после COMMIT. Отдельно наблюдаемое processing, lease, управляемые повторы и очистка staging относятся к этапу 5.
|
||||
|
||||
`python3 student-tests/worker_variants.py --base-url http://localhost:18082` проверяет обработчик на реальных PostgreSQL и MinIO через прямую передачу job, затем сверяет скачанные байты и метаданные через Nginx. Результат в `evidence/worker-variants.txt`.
|
||||
|
||||
## Этап 5. Lease, retry и очистка
|
||||
|
||||
Worker транзакционно захватывает queued-задачу на 60 секунд и переводит её в processing. Одна доставка обрабатывается за раз. Таймер worker — 20 секунд. Публикация результата проверяет номер попытки и действительность lease. Результаты попыток имеют разные S3-префиксы, поэтому запоздавшая попытка не перезаписывает актуальный вариант.
|
||||
|
||||
Временная ошибка и истёкший lease планируют новую попытку через outbox с задержкой 2^n секунд (исходная задержка ограничена 60 с) и jitter ±25%. По умолчанию JOB_MAX_ATTEMPTS=5. Некорректный оригинал завершает задачу с invalid_image. После исчерпания попыток возвращается retry_exhausted. Реестр failed хранится в PostgreSQL. Replay увеличивает поколение попытки, сбрасывает счётчик retry и пишет аудит. Replay доступен для failed до удаления staging.
|
||||
|
||||
```sh
|
||||
docker compose exec -T api-a python -m messenger.failed_jobs list
|
||||
docker compose exec -T api-a python -m messenger.failed_jobs replay JOB_UUID --operator local
|
||||
```
|
||||
|
||||
Cleaner удаляет staging после ready или спустя 24 часа после failed. Файлы старых и окончательно неуспешных попыток удаляются после 15-минутного периода ожидания. Операции очистки можно повторять после сбоя.
|
||||
|
||||
Проверка `python3 student-tests/job_lifecycle.py --base-url http://localhost:18082` выполнена с реальными PostgreSQL и MinIO. Истечение lease моделировалось обновлением дедлайна после записи файлов. Результаты в `evidence/job-lifecycle.txt`. Брокерный сценарий физической остановки worker до ACK остаётся для итогового прогона.
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api: &api
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
|
||||
environment: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: '8080'
|
||||
APP_ENV: lab5
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: redis
|
||||
SHUTDOWN_GRACE_SECONDS: '8'
|
||||
DB_POOL_MIN: '2'
|
||||
DB_POOL_MAX: '16'
|
||||
DB_CONNECT_TIMEOUT: '1'
|
||||
DB_STATEMENT_TIMEOUT: '4'
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
cpus: 0.50
|
||||
mem_limit: 256m
|
||||
stop_grace_period: 9s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 2
|
||||
start_period: 3s
|
||||
# Deliberately no auto-restart: failover hook must leave its target stopped.
|
||||
restart: 'no'
|
||||
|
||||
services:
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
hostname: rabbitmq
|
||||
entrypoint: [/bin/sh, -ec]
|
||||
command:
|
||||
- |
|
||||
export RABBITMQ_DEFAULT_USER=lab5
|
||||
RABBITMQ_DEFAULT_PASS=$$(cat /run/secrets/rabbitmq_password)
|
||||
export RABBITMQ_DEFAULT_PASS
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
secrets: [rabbitmq_password]
|
||||
volumes: [rabbitmq_data:/var/lib/rabbitmq]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, rabbitmq-diagnostics, -q, check_port_connectivity]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
|
||||
outbox-publisher:
|
||||
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.publish_jobs]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
DB_POOL_MIN: '1'
|
||||
DB_POOL_MAX: '2'
|
||||
secrets: [database_url, rabbitmq_password]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
s3:
|
||||
image: mtusi-lab5-s3:2025-09-07
|
||||
build: &s3-build
|
||||
context: .
|
||||
dockerfile: infra/Dockerfile.s3
|
||||
entrypoint: [/bin/sh, -ec]
|
||||
command:
|
||||
- |
|
||||
MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user)
|
||||
MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password)
|
||||
export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
|
||||
exec minio server /data
|
||||
environment:
|
||||
MINIO_BROWSER: "off"
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
networks: [backend]
|
||||
|
||||
s3-bootstrap:
|
||||
image: mtusi-lab5-s3:2025-09-07
|
||||
build: *s3-build
|
||||
entrypoint: [/bin/sh, /bootstrap.sh]
|
||||
command: []
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes: [./infra/s3-bootstrap.sh:/bootstrap.sh:ro]
|
||||
depends_on: [s3]
|
||||
networks: [backend]
|
||||
|
||||
postgres:
|
||||
image: postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [pg_data:/var/lib/postgresql/data]
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 96mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [backend]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url, cursor_secret]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
cpus: 0.50
|
||||
mem_limit: 256m
|
||||
restart: 'no'
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
attachment-cleaner:
|
||||
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.clean_attachments]
|
||||
environment: *api-env
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks: [backend]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
worker:
|
||||
image: mtusi-lab5-api:${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.worker]
|
||||
environment:
|
||||
<<: *api-env
|
||||
RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password
|
||||
DB_POOL_MIN: '1'
|
||||
DB_POOL_MAX: '2'
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, rabbitmq_password]
|
||||
depends_on:
|
||||
outbox-publisher:
|
||||
condition: service_started
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 768m
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
api-a:
|
||||
<<: *api
|
||||
api-b:
|
||||
<<: *api
|
||||
|
||||
nginx:
|
||||
image: nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e
|
||||
entrypoint: [/bin/sh, -ec]
|
||||
command:
|
||||
- 'cp /etc/nginx/lab/upstreams.conf /etc/nginx/upstreams.conf; /bin/sh /etc/nginx/lab/healthwatch.sh & exec nginx -g "daemon off;"'
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra:/etc/nginx/lab:ro
|
||||
ports: ['127.0.0.1:${PUBLIC_PORT:-18082}:8080']
|
||||
networks: [frontend, backend]
|
||||
depends_on:
|
||||
api-a:
|
||||
condition: service_healthy
|
||||
api-b:
|
||||
condition: service_healthy
|
||||
cpus: 0.50
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'kill -0 $$(cat /tmp/healthwatch.pid) && wget -T 1 -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
rabbitmq_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
cursor_secret:
|
||||
file: ./secrets/cursor_secret
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password
|
||||
@@ -3,3 +3,5 @@
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
|
||||
Файлы `*-final.txt` содержат повторный прогон на итоговом коде. `queued-upload-final.txt` проверяет HTTP-приём, `worker-variants-final.txt` — результаты обработки с прямой передачей задания, `job-lifecycle-final.txt` — lease, retry, replay и очистку на PostgreSQL и MinIO. Прямой сценарий не измеряет доставку RabbitMQ.
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
name durable messages_ready messages_unacknowledged
|
||||
attachment.process.v1 true 0 0
|
||||
@@ -0,0 +1,69 @@
|
||||
Container mtusi-lab5-worker-1 Stopping
|
||||
Container mtusi-lab5-worker-1 Stopped
|
||||
Container mtusi-lab5-api-a-1 Restarting
|
||||
Container mtusi-lab5-api-b-1 Restarting
|
||||
Container mtusi-lab5-rabbitmq-1 Restarting
|
||||
Container mtusi-lab5-api-a-1 Started
|
||||
Container mtusi-lab5-api-b-1 Started
|
||||
Container mtusi-lab5-rabbitmq-1 Started
|
||||
Container mtusi-lab5-api-b-1 Running
|
||||
Container mtusi-lab5-api-a-1 Running
|
||||
Container mtusi-lab5-api-a-1 Waiting
|
||||
Container mtusi-lab5-api-b-1 Waiting
|
||||
Container mtusi-lab5-api-a-1 Healthy
|
||||
Container mtusi-lab5-api-b-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Started
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi-lab5-worker-1 Starting
|
||||
Container mtusi-lab5-worker-1 Started
|
||||
PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready
|
||||
Container mtusi-lab5-worker-1 Stopping
|
||||
Container mtusi-lab5-worker-1 Stopped
|
||||
Container lab5-fault-e0d0159d35 Creating
|
||||
Container lab5-fault-e0d0159d35 Created
|
||||
558b991a978ef6f8660d068af2a737cea3e0fe7366185b65069dad119981ec7c
|
||||
INFO:pika.adapters.utils.connection_workflow:Pika version 1.3.2 connecting to ('172.23.0.9', 5672)
|
||||
INFO:pika.adapters.utils.io_services_utils:Socket connected: <socket.socket fd=7, family=2, type=1, proto=6, laddr=('172.23.0.11', 41090), raddr=('172.23.0.9', 5672)>
|
||||
INFO:pika.adapters.utils.connection_workflow:Streaming transport linked up: (<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60>, _StreamingProtocolShim: <SelectConnection PROTOCOL transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>).
|
||||
INFO:pika.adapters.utils.connection_workflow:AMQPConnector - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
INFO:pika.adapters.utils.connection_workflow:AMQPConnectionWorkflow - reporting success: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
INFO:pika.adapters.blocking_connection:Connection workflow succeeded: <SelectConnection OPEN transport=<pika.adapters.utils.io_services_utils._AsyncPlaintextTransport object at 0xffffb545ac60> params=<ConnectionParameters host=rabbitmq port=5672 virtual_host=/ ssl=False>>
|
||||
INFO:pika.adapters.blocking_connection:Created channel=1
|
||||
lab5-fault-e0d0159d35
|
||||
Container mtusi-lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Started
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi-lab5-worker-1 Starting
|
||||
Container mtusi-lab5-worker-1 Started
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Started
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi-lab5-s3-bootstrap-1 Exited
|
||||
PASS: worker killed after S3 write before COMMIT/ACK; recovered in 60.90s
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18082"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.682}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,12 @@
|
||||
api-a running healthy
|
||||
api-b running healthy
|
||||
attachment-cleaner running
|
||||
migrate exited
|
||||
nginx running healthy
|
||||
outbox-publisher running
|
||||
postgres running healthy
|
||||
rabbitmq running healthy
|
||||
redis running healthy
|
||||
s3 running
|
||||
s3-bootstrap exited
|
||||
worker running
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned
|
||||
PASS: ready original survives cleanup and replay requests
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned
|
||||
PASS: ready original survives cleanup and replay requests
|
||||
@@ -0,0 +1,6 @@
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Container mtusi-lab5-s3-1 Running
|
||||
Container mtusi-lab5-s3-bootstrap-run-63e5e364daea Creating
|
||||
Container mtusi-lab5-s3-bootstrap-run-63e5e364daea Created
|
||||
PASS: authenticated round trip; anonymous GET returned 403
|
||||
PASS: shared session on two lab5 API replicas: 74e294abbbf5 d55768f2410a
|
||||
@@ -0,0 +1,16 @@
|
||||
Container mtusi-lab5-outbox-publisher-1 Stopping
|
||||
Container mtusi-lab5-outbox-publisher-1 Stopped
|
||||
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
|
||||
Attachment ID: 168907b8-2152-42a4-ad05-d9e40aa211d8
|
||||
PASS: upload committed while publisher was stopped
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-rabbitmq-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-rabbitmq-1 Healthy
|
||||
Container mtusi-lab5-outbox-publisher-1 Starting
|
||||
Container mtusi-lab5-outbox-publisher-1 Started
|
||||
PASS: same attachment job confirmed after publisher startup
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: PostgreSQL outbox remains unpublished after simulated broker NACK
|
||||
The test used the real lab5 PostgreSQL and publisher code with an injected NACK in place of the broker channel.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18082"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
|
||||
Attachment ID: df03f2ef-0193-4b3d-8591-ef0f432a948e
|
||||
@@ -0,0 +1,3 @@
|
||||
PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked
|
||||
Attachment ID: 3f72ec80-cf76-4e14-a373-3946735437d7
|
||||
PASS: exactly one queued attachment, job and unpublished outbox row
|
||||
@@ -0,0 +1,2 @@
|
||||
ba452b9cf1c8e766d58801c2767f5bd719a36b93b39db8c58875cf09ac4b996b
|
||||
3d7a006ce123e652c1754efc4d7fcd5dbac77605d47a313ed6c5501fb3880d8c
|
||||
@@ -0,0 +1,4 @@
|
||||
Container mtusi-lab5-s3-1 Running
|
||||
Container mtusi-lab5-s3-bootstrap-run-6191998ad078 Creating
|
||||
Container mtusi-lab5-s3-bootstrap-run-6191998ad078 Created
|
||||
PASS: authenticated round trip; anonymous GET returned 403
|
||||
@@ -0,0 +1,4 @@
|
||||
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/bin/python scripts/validate_openapi.py
|
||||
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:18082"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200
|
||||
PASS: processing the same committed job twice keeps one set of variants
|
||||
@@ -0,0 +1,3 @@
|
||||
PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200
|
||||
PASS: processing the same committed job twice keeps one set of variants
|
||||
PASS: EXIF rotation applied, EXIF stripped, transparency on white, no upscaling
|
||||
@@ -0,0 +1,13 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM public.ecr.aws/docker/library/golang:1.24.2-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS build
|
||||
ENV CGO_ENABLED=0 GOPROXY="https://proxy.golang.org|https://goproxy.io|direct"
|
||||
WORKDIR /src/minio
|
||||
ADD https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-09-07T16-13-09Z /tmp/minio.tar.gz
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/minio.tar.gz --strip-components=1 && go build -o /out/minio .
|
||||
WORKDIR /src/mc
|
||||
ADD https://codeload.github.com/minio/mc/tar.gz/refs/tags/RELEASE.2025-08-13T08-35-41Z /tmp/mc.tar.gz
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/mc.tar.gz --strip-components=1 && go build -o /out/mc .
|
||||
|
||||
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
|
||||
COPY --from=build /out/minio /out/mc /usr/local/bin/
|
||||
ENTRYPOINT ["minio"]
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# Active readiness bridge for open-source Nginx. No Docker socket or credentials.
|
||||
set -eu
|
||||
probe() {
|
||||
if wget -T 1 -q -O /dev/null "http://$1:8080/health/ready" 2>/dev/null; then
|
||||
echo up > "/tmp/$1.health"
|
||||
else
|
||||
echo down > "/tmp/$1.health"
|
||||
fi
|
||||
}
|
||||
control() { cat "/tmp/control-$1" 2>/dev/null || echo serve:boot; }
|
||||
echo $$ > /tmp/healthwatch.pid
|
||||
while :; do
|
||||
sleep 1
|
||||
probe api-a &
|
||||
first=$!
|
||||
probe api-b &
|
||||
second=$!
|
||||
wait "$first"
|
||||
wait "$second"
|
||||
# Read desired state AFTER probes. A generation is acknowledged only after
|
||||
# the matching upstream configuration has been applied, never from a stale probe.
|
||||
a_control=$(control api-a)
|
||||
b_control=$(control api-b)
|
||||
a_state=$(cat /tmp/api-a.health)
|
||||
b_state=$(cat /tmp/api-b.health)
|
||||
case "$a_control" in drain:*) a_state=down;; esac
|
||||
case "$b_control" in drain:*) b_state=down;; esac
|
||||
: > /tmp/upstreams.next
|
||||
for service in api-a api-b; do
|
||||
if [ "$service" = api-a ]; then state=$a_state; else state=$b_state; fi
|
||||
suffix=''
|
||||
[ "$state" = up ] || suffix=' down'
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=2s%s;\n' "$service" "$suffix" >> /tmp/upstreams.next
|
||||
done
|
||||
if ! cmp -s /tmp/upstreams.next /etc/nginx/upstreams.conf; then
|
||||
cp /etc/nginx/upstreams.conf /tmp/upstreams.previous
|
||||
# Rename on the same filesystem: readers never see missing/partial config.
|
||||
cp /tmp/upstreams.next /etc/nginx/upstreams.next
|
||||
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
|
||||
if nginx -t && nginx -s reload; then
|
||||
echo 'readiness upstream set updated'
|
||||
else
|
||||
cp /tmp/upstreams.previous /etc/nginx/upstreams.next
|
||||
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
printf '%s %s\n' "$a_control" "$a_state" > /tmp/applied-api-a.next
|
||||
mv -f /tmp/applied-api-a.next /tmp/applied-api-a
|
||||
printf '%s %s\n' "$b_control" "$b_state" > /tmp/applied-api-b.next
|
||||
mv -f /tmp/applied-api-b.next /tmp/applied-api-b
|
||||
done
|
||||
@@ -0,0 +1,49 @@
|
||||
worker_processes 1;
|
||||
worker_shutdown_timeout 6s;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
log_format lab '$time_iso8601 $request_method $uri $status upstream=$upstream_addr '
|
||||
'instance=$upstream_http_x_instance_id time=$request_time';
|
||||
access_log /dev/stdout lab;
|
||||
error_log /dev/stderr warn;
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
include /etc/nginx/upstreams.conf;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 1m;
|
||||
client_body_timeout 5s;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection close;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Request-Id $http_x_request_id;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
# Keep a proxy ceiling above the API limit so the API can return its JSON 413.
|
||||
client_max_body_size 11m;
|
||||
proxy_read_timeout 45s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream off;
|
||||
proxy_pass http://api;
|
||||
}
|
||||
location ~ ^/api/v1/attachments/[^/]+/content$ {
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 10s;
|
||||
# A shared S3 outage returns 503 without marking the API itself down.
|
||||
proxy_next_upstream error timeout http_502 http_504;
|
||||
proxy_pass http://api;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
umask 077
|
||||
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
case "$bucket" in
|
||||
''|*[!a-z0-9.-]*) echo 'Invalid S3 bucket name' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
config=$(mktemp -d)
|
||||
trap 'rm -rf "$config"' EXIT
|
||||
export MC_CONFIG_DIR="$config"
|
||||
root_user=$(cat /run/secrets/s3_root_user)
|
||||
root_password=$(cat /run/secrets/s3_root_password)
|
||||
api_user=$(cat /run/secrets/s3_access_key_id)
|
||||
api_password=$(cat /run/secrets/s3_secret_access_key)
|
||||
endpoint=${S3_ENDPOINT:-http://s3:9000}
|
||||
|
||||
if [ "${1:-}" = verify ]; then
|
||||
export MC_HOST_api="http://$api_user:$api_password@${endpoint#http://}"
|
||||
key="attachments/probe-${config##*/}"
|
||||
object="api/$bucket/$key"
|
||||
trap 'mc rm "$object" >/dev/null 2>&1 || true; rm -rf "$config"' EXIT
|
||||
printf '%s' 'lab5-private-bucket-probe' | mc pipe "$object" >/dev/null
|
||||
test "$(mc cat "$object")" = 'lab5-private-bucket-probe'
|
||||
python3 - "$endpoint/$bucket/$key" <<'PYTEST'
|
||||
import sys
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
|
||||
try:
|
||||
with urlopen(sys.argv[1], timeout=5):
|
||||
raise SystemExit("FAIL: anonymous GET succeeded")
|
||||
except HTTPError as exc:
|
||||
if exc.code != 403:
|
||||
raise
|
||||
print("PASS: authenticated round trip; anonymous GET returned 403")
|
||||
PYTEST
|
||||
mc rm "$object" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
until mc alias set storage "$endpoint" "$root_user" "$root_password" >/dev/null 2>&1 &&
|
||||
mc admin info storage >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
echo 'S3 bootstrap could not connect or authenticate' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
mc mb --ignore-existing "storage/$bucket"
|
||||
mc anonymous set none "storage/$bucket"
|
||||
cat > "$config/api-policy.json" <<POLICY
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::$bucket/attachments/*"]
|
||||
}, {
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::$bucket"]
|
||||
}]
|
||||
}
|
||||
POLICY
|
||||
mc admin policy create storage lab5-attachments "$config/api-policy.json"
|
||||
mc admin user add storage "$api_user" "$api_password" >/dev/null
|
||||
mc admin policy attach storage lab5-attachments --user "$api_user"
|
||||
echo 'PASS: private S3 bucket and scoped API policy configured'
|
||||
@@ -0,0 +1,2 @@
|
||||
server api-a:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
server api-b:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
@@ -0,0 +1,51 @@
|
||||
-- Migration 0001: initial schema for lab2.
|
||||
-- Idempotent: safe to re-run on an existing database.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
password_hash text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
|
||||
-- full table scan on every page read.
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
|
||||
ON messages (chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
|
||||
ON sessions (user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
|
||||
ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Additive migration: compatible with the lab2 schema and existing messages.
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
key text NOT NULL,
|
||||
request_hash text NOT NULL,
|
||||
response_status integer NOT NULL,
|
||||
response_body jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
-- Results are retained indefinitely in this lab (therefore at least 24 hours).
|
||||
-- There is intentionally no independent cache expiry that could allow duplicates.
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Additive migration. Existing text messages have no attachment links.
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_id_chat_id_key UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
|
||||
object_key text NOT NULL UNIQUE CHECK (object_key LIKE 'attachments/%'),
|
||||
-- pending is internal. The upload API exposes ready after committing metadata.
|
||||
status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (id, chat_id),
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_pending_updated_idx
|
||||
ON attachments (updated_at, id) WHERE status = 'pending';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, position),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,33 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CHECK (status IN ('pending','queued','processing','ready','failed'));
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_outbox (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
|
||||
attempt integer NOT NULL CHECK (attempt > 0),
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
PRIMARY KEY (job_id, attempt)
|
||||
);
|
||||
CREATE INDEX attachment_outbox_due_idx ON attachment_outbox (available_at) WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/webp')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN lease_until timestamptz;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN attempts_used integer NOT NULL DEFAULT 0;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN staging_cleaned_at timestamptz;
|
||||
ALTER TABLE attachment_outbox ADD COLUMN objects_cleaned_at timestamptz;
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE lease_until IS NOT NULL;
|
||||
CREATE TABLE attachment_replays (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
|
||||
attempt integer NOT NULL,
|
||||
operator text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (job_id, attempt)
|
||||
);
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local credentials once, never print them or overwrite existing files."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = folder / 'postgres_password'
|
||||
|
||||
def save(name, value):
|
||||
path = folder / name
|
||||
if path.exists():
|
||||
return
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as handle:
|
||||
handle.write(value + '\n')
|
||||
|
||||
save('postgres_password', secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
|
||||
save('cache_url', 'redis://redis:6379/0')
|
||||
save('cursor_secret', secrets.token_hex(32))
|
||||
save('s3_root_user', 'lab5-admin-' + secrets.token_hex(8))
|
||||
save('s3_root_password', secrets.token_hex(32))
|
||||
save('s3_access_key_id', 'lab5-api-' + secrets.token_hex(8))
|
||||
save('s3_secret_access_key', secrets.token_hex(24))
|
||||
save('rabbitmq_password', secrets.token_hex(24))
|
||||
print('PASS: local secret files ready (existing values preserved)')
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit, bounded operator actions for this lab's Compose project only."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ('api-a', 'api-b')
|
||||
|
||||
|
||||
def command(args, capture=False):
|
||||
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
|
||||
|
||||
|
||||
def inspect(container):
|
||||
return json.loads(command(['docker', 'inspect', container], True))[0]
|
||||
|
||||
|
||||
def containers(service):
|
||||
return compose('ps', '-a', '-q', service, capture=True).splitlines()
|
||||
|
||||
|
||||
def start_services(*services):
|
||||
for service in services:
|
||||
for container in containers(service):
|
||||
command(['docker', 'start', container])
|
||||
|
||||
|
||||
def wait_healthy(service, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
ids = containers(service)
|
||||
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
|
||||
return
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('%s did not become healthy' % service)
|
||||
|
||||
|
||||
def target_service(target):
|
||||
if not target:
|
||||
raise RuntimeError('TARGET_INSTANCE_ID is required')
|
||||
for service in SERVICES:
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
if data['Config']['Hostname'] == target:
|
||||
return service, container
|
||||
raise RuntimeError('Unknown API instance alias')
|
||||
|
||||
|
||||
def set_backend(service, enabled):
|
||||
# Fixed allowlist; shell receives no untrusted instance or credential input.
|
||||
if service not in SERVICES:
|
||||
raise ValueError('Unknown service')
|
||||
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
|
||||
path = '/tmp/control-' + service
|
||||
compose('exec', '-T', 'nginx', 'sh', '-ec',
|
||||
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
|
||||
deadline = time.monotonic() + 10
|
||||
expected = token + (' up' if enabled else ' down')
|
||||
while time.monotonic() < deadline:
|
||||
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
|
||||
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
|
||||
if ack == expected:
|
||||
return
|
||||
time.sleep(.2)
|
||||
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
|
||||
|
||||
|
||||
def stop_one():
|
||||
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
|
||||
running = [c for name in SERVICES for c in containers(name)
|
||||
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
|
||||
and inspect(c)['State']['Running']]
|
||||
if len(running) < 2:
|
||||
raise RuntimeError('Restore the other replica before this experiment')
|
||||
print('Stopping API replica:', service, container[:12], flush=True)
|
||||
command(['docker', 'stop', '--time', '8', container])
|
||||
if inspect(container)['State']['Running']:
|
||||
raise RuntimeError('Target is still running')
|
||||
|
||||
|
||||
def restart():
|
||||
# Never start the one-shot migration service as part of persistence testing.
|
||||
compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker')
|
||||
for service in (*SERVICES, 'nginx'):
|
||||
wait_healthy(service)
|
||||
|
||||
|
||||
def rolling_update():
|
||||
# Run additive migrations once before replacing either API version.
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
for service in SERVICES:
|
||||
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
|
||||
wait_healthy(other)
|
||||
set_backend(other, True)
|
||||
before = containers(service)
|
||||
set_backend(service, False)
|
||||
# The old nginx worker can retain the old upstream table until it exits.
|
||||
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
|
||||
time.sleep(6.5)
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
|
||||
wait_healthy(service)
|
||||
set_backend(service, True)
|
||||
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
|
||||
after = containers(service)
|
||||
if before == after:
|
||||
raise RuntimeError('Replica was not replaced')
|
||||
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
|
||||
|
||||
|
||||
def restore():
|
||||
start_services(*SERVICES)
|
||||
for service in SERVICES:
|
||||
wait_healthy(service)
|
||||
for service in SERVICES:
|
||||
set_backend(service, True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
|
||||
args = parser.parse_args()
|
||||
if args.action == 'stop-one':
|
||||
stop_one()
|
||||
elif args.action == 'restart':
|
||||
restart()
|
||||
elif args.action == 'rolling-update':
|
||||
rolling_update()
|
||||
elif args.action == 'restore':
|
||||
restore()
|
||||
else:
|
||||
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
print(json.dumps({'service': service, 'id': data['Id'][:12],
|
||||
'instance': data['Config']['Hostname'],
|
||||
'running': data['State']['Running'],
|
||||
'started_at': data['State']['StartedAt'],
|
||||
'health': data['State'].get('Health', {}).get('Status'),
|
||||
'image_id': data['Image'],
|
||||
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_limit_bytes': data['HostConfig']['Memory'],
|
||||
'published_ports': data['HostConfig'].get('PortBindings')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py restart
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py rolling-update
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py stop-one
|
||||
@@ -0,0 +1,3 @@
|
||||
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
|
||||
|
||||
__version__ = '1.0.0'
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Entry point: python -m messenger."""
|
||||
import sys
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from . import validation
|
||||
from .attachments import AttachmentStorage, filename_from, upload_file
|
||||
from .db import Database
|
||||
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
|
||||
from .store import Store, utc_now, rfc3339
|
||||
|
||||
SCOPE_CHATS = 'chats'
|
||||
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
|
||||
|
||||
|
||||
class Request:
|
||||
"""One parsed HTTP request; the API layer never touches sockets."""
|
||||
|
||||
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
|
||||
|
||||
def __init__(self, method, path, query, headers, body, request_id, client):
|
||||
self.method = method
|
||||
self.path = path
|
||||
self.query = query
|
||||
self.headers = headers
|
||||
self.body = body
|
||||
self.request_id = request_id
|
||||
self.client = client
|
||||
|
||||
|
||||
class Api:
|
||||
UPLOAD_PATH = re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/attachments$')
|
||||
ROUTES = (
|
||||
('POST', UPLOAD_PATH, 'upload_attachment'),
|
||||
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)$'), 'get_attachment'),
|
||||
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)/content$'), 'download_attachment'),
|
||||
('GET', re.compile(r'^/health/live$'), 'health_live'),
|
||||
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
|
||||
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
|
||||
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
|
||||
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
|
||||
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
|
||||
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
|
||||
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
|
||||
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
|
||||
)
|
||||
|
||||
def __init__(self, config, store: Store, database: Database):
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.database = database
|
||||
self.attachments = AttachmentStorage()
|
||||
|
||||
def dispatch(self, request):
|
||||
"""Return (status, payload). Only ApiError and payloads leave this method."""
|
||||
path_matched = False
|
||||
for method, pattern, handler in self.ROUTES:
|
||||
match = pattern.match(request.path)
|
||||
if match:
|
||||
path_matched = True
|
||||
if method == request.method:
|
||||
return getattr(self, handler)(request, match.groupdict())
|
||||
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
|
||||
|
||||
# ------------------------------------------------------------- health
|
||||
def health_live(self, request, path_params):
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
def health_ready(self, request, path_params):
|
||||
try:
|
||||
self.database.ping()
|
||||
except Exception:
|
||||
raise Unavailable('База данных недоступна')
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
# -------------------------------------------------------------- users
|
||||
def create_user(self, request, path_params):
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('username', 'display_name', 'password'),
|
||||
('username', 'display_name', 'password'))
|
||||
username = validation.require_username(payload)
|
||||
display_name = validation.require_text_field(
|
||||
payload, 'display_name', 1, self.config.max_display_name_code_points)
|
||||
password = validation.require_text_field(
|
||||
payload, 'password', 12, 128, strip_check=False)
|
||||
password_hash = security.hash_password(password)
|
||||
return 201, self.store.create_user(username, display_name, password_hash)
|
||||
|
||||
def current_user(self, request, path_params):
|
||||
return 200, self._authenticate(request)
|
||||
|
||||
# -------------------------------------------------------------- chats
|
||||
def create_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
|
||||
title = validation.require_text_field(payload, 'title', 1,
|
||||
self.config.max_title_code_points)
|
||||
members = validation.require_member_ids(payload, 100)
|
||||
if actor['id'] not in members:
|
||||
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
|
||||
if not self.store.users_exist(members):
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей')
|
||||
return 201, self.store.create_chat(title, members)
|
||||
|
||||
def list_chats(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
limit, after_key = self._pagination(request, SCOPE_CHATS)
|
||||
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
|
||||
for item in items:
|
||||
item['member_ids'] = self._chat_member_ids(item['id'])
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
|
||||
|
||||
def get_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
chat['member_ids'] = self._chat_member_ids(chat['id'])
|
||||
return 200, chat
|
||||
|
||||
# ----------------------------------------------------------- messages
|
||||
def post_message(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
key = validation.require_idempotency_key(request.headers.get('Idempotency-Key'))
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('text',), ('text', 'attachment_ids'))
|
||||
attachment_ids = validation.require_attachment_ids(payload)
|
||||
text = validation.require_text_field(payload, 'text', 0 if attachment_ids else 1,
|
||||
self.config.max_text_code_points,
|
||||
strip_check=not attachment_ids)
|
||||
return 201, self.store.add_message(chat['id'], actor['id'], text, key, attachment_ids)
|
||||
|
||||
def list_messages(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
|
||||
limit, after_key = self._pagination(request, scope)
|
||||
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
|
||||
|
||||
def upload_attachment(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
filename = filename_from(request.query)
|
||||
mime = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower()
|
||||
with upload_file(request.body, mime) as (file, size, digest):
|
||||
aid = str(uuid.uuid4())
|
||||
key = 'attachments/staging/' + aid
|
||||
self.store.reserve_attachment(aid, chat['id'], actor['id'], filename,
|
||||
mime, size, digest, key)
|
||||
row = self.store.queue_attachment(
|
||||
aid, lambda object_key: self.attachments.put(object_key, file, size, mime))
|
||||
return 202, self.store.public_attachment(row)
|
||||
|
||||
def _visible_attachment(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
aid = validation.require_uuid_string(path_params['id'], 'attachment_id')
|
||||
row = self.store.visible_attachment(aid, actor['id'])
|
||||
if row is None:
|
||||
raise NotFound('Вложение не найдено')
|
||||
return row
|
||||
|
||||
def get_attachment(self, request, path_params):
|
||||
row = self._visible_attachment(request, path_params)
|
||||
variants = self.store.attachment_variants(row['id']) if row['status'] == 'ready' else ()
|
||||
return 200, self.store.public_attachment(row, variants)
|
||||
|
||||
def download_attachment(self, request, path_params):
|
||||
row = self._visible_attachment(request, path_params)
|
||||
if row['status'] != 'ready':
|
||||
raise Conflict('Вложение не готово')
|
||||
variant = request.query.get('variant', ['original'])
|
||||
if variant != ['original']:
|
||||
row = next((item for item in self.store.attachment_variants(row['id'])
|
||||
if [item['name']] == variant), None)
|
||||
if row is None:
|
||||
raise NotFound('Вариант вложения не найден')
|
||||
return 200, self.attachments.download(row)
|
||||
|
||||
# ----------------------------------------------------------- sessions
|
||||
def create_session(self, request, path_params):
|
||||
"""Login: only Authorization: Basic username:password, no JSON body."""
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('basic '):
|
||||
raise Unauthorized('Требуется авторизация Basic')
|
||||
try:
|
||||
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
|
||||
except Exception:
|
||||
raise Unauthorized('Некорректные учётные данные') from None
|
||||
if ':' not in raw:
|
||||
raise Unauthorized('Некорректные учётные данные')
|
||||
username, password = raw.split(':', 1)
|
||||
row = self.store.user_by_username(username)
|
||||
if row is None or not security.verify_password(password, row['password_hash']):
|
||||
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
|
||||
token = security.generate_token()
|
||||
token_hash = security.hash_token(token)
|
||||
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
|
||||
self.store.create_session(token_hash, row['id'], expires_at)
|
||||
user = self.store._public_user({
|
||||
'id': row['id'],
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': row['created_at'],
|
||||
})
|
||||
return 201, {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': rfc3339(expires_at),
|
||||
'user': user,
|
||||
}
|
||||
|
||||
def revoke_session(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
token = self._bearer_token(request)
|
||||
self.store.delete_session(security.hash_token(token))
|
||||
return 204, None
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
def _authenticate(self, request):
|
||||
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
|
||||
|
||||
X-User-Id is ignored: it does not change the session owner and alone
|
||||
grants no access. Basic on business endpoints is 401.
|
||||
"""
|
||||
token = self._bearer_token(request)
|
||||
if token is None:
|
||||
raise Unauthorized('Требуется идентификация пользователя',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user_id = self.store.get_session(security.hash_token(token))
|
||||
if user_id is None:
|
||||
raise Unauthorized('Сессия недействительна или истекла',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user = self.store.user(user_id)
|
||||
if user is None:
|
||||
raise Unauthorized('Пользователь не найден',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
return user
|
||||
|
||||
def _bearer_token(self, request):
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('bearer '):
|
||||
return None
|
||||
return authorization[len('bearer '):].strip()
|
||||
|
||||
def _chat_member_ids(self, chat_id):
|
||||
with self.database.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
return [str(row['user_id']) for row in cur.fetchall()]
|
||||
|
||||
def _member_chat(self, actor, raw_chat_id):
|
||||
"""Foreign and missing chats are indistinguishable: both are 404."""
|
||||
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
|
||||
chat = self.store.chat(chat_id)
|
||||
if chat is None or not self.store.is_member(chat, actor['id']):
|
||||
raise NotFound('Чат не найден')
|
||||
return chat
|
||||
|
||||
def _json_object(self, request):
|
||||
content_type = request.headers.get('Content-Type') or ''
|
||||
if content_type.split(';')[0].strip().lower() != 'application/json':
|
||||
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
|
||||
if not request.body:
|
||||
raise InvalidRequest('Требуется JSON-тело запроса')
|
||||
try:
|
||||
payload = json.loads(request.body.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
raise InvalidRequest('Тело запроса не является корректным JSON') from None
|
||||
return validation.require_object(payload)
|
||||
|
||||
def _pagination(self, request, scope):
|
||||
raw_limit = (request.query.get('limit') or [None])[0]
|
||||
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
|
||||
self.config.max_page_limit)
|
||||
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
|
||||
if raw_cursor is None:
|
||||
return limit, None
|
||||
try:
|
||||
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
|
||||
except cursor_codec.InvalidCursor:
|
||||
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
|
||||
return limit, (stamp, identifier)
|
||||
|
||||
def _next_cursor(self, scope, items, has_more):
|
||||
if not has_more or not items:
|
||||
return None
|
||||
last = items[-1]
|
||||
return cursor_codec.encode(scope, last['created_at'], last['id'],
|
||||
self.config.cursor_secret)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Bounded image validation and private S3 storage."""
|
||||
from contextlib import contextmanager
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
from functools import cached_property
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image
|
||||
|
||||
from .errors import InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
||||
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_IMAGE_PIXELS = 40_000_000
|
||||
CHUNK_SIZE = 64 * 1024
|
||||
FORMATS = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||
# Pillow can allocate a full decoded image. Allow only one decode per API process.
|
||||
_decode_slot = threading.Lock()
|
||||
_upload_slots = threading.BoundedSemaphore(10)
|
||||
|
||||
|
||||
def filename_from(query):
|
||||
names = query.get('filename', [])
|
||||
if len(names) != 1:
|
||||
raise InvalidRequest('Укажите одно имя файла')
|
||||
name = names[0]
|
||||
if (not 1 <= len(name) <= 255 or not name.strip()
|
||||
or any(c in '/\\' or unicodedata.category(c) in {'Cc', 'Cs'} for c in name)):
|
||||
raise InvalidRequest('Некорректное имя файла')
|
||||
return name
|
||||
|
||||
|
||||
def validate_image(file, declared_type):
|
||||
try:
|
||||
with _decode_slot:
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=list(FORMATS)) as image:
|
||||
if image.width * image.height > MAX_IMAGE_PIXELS:
|
||||
raise PayloadTooLarge('Изображение превышает 40 млн пикселей')
|
||||
if FORMATS[image.format] != declared_type:
|
||||
raise UnsupportedMediaType('Тип изображения не совпадает с Content-Type')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=list(FORMATS)) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise PayloadTooLarge('Изображение превышает 40 млн пикселей') from None
|
||||
except (OSError, SyntaxError, ValueError, OverflowError, EOFError, struct.error):
|
||||
raise UnsupportedMediaType('Повреждённое или неподдерживаемое изображение') from None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def upload_file(read_chunks, content_type):
|
||||
if content_type not in FORMATS.values():
|
||||
raise UnsupportedMediaType('Ожидается PNG или JPEG')
|
||||
if not _upload_slots.acquire(blocking=False):
|
||||
raise Unavailable('Слишком много одновременных загрузок')
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
for chunk in read_chunks():
|
||||
size += len(chunk)
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLarge()
|
||||
digest.update(chunk)
|
||||
file.write(chunk)
|
||||
if size == 0:
|
||||
raise UnsupportedMediaType('Пустое изображение')
|
||||
validate_image(file, content_type)
|
||||
file.seek(0)
|
||||
yield file, size, digest.hexdigest()
|
||||
finally:
|
||||
_upload_slots.release()
|
||||
|
||||
|
||||
class BinaryResponse:
|
||||
def __init__(self, stream, content_type, size):
|
||||
self.stream = stream
|
||||
self.content_type = content_type
|
||||
self.size = size
|
||||
|
||||
def chunks(self):
|
||||
try:
|
||||
yield from self.stream.iter_chunks(CHUNK_SIZE)
|
||||
except (BotoCoreError, OSError) as exc:
|
||||
raise OSError('S3 response interrupted') from exc
|
||||
|
||||
def close(self):
|
||||
self.stream.close()
|
||||
|
||||
|
||||
class AttachmentStorage:
|
||||
@cached_property
|
||||
def client(self):
|
||||
def secret(name):
|
||||
with open(os.environ[name + '_FILE'], encoding='utf-8') as file:
|
||||
return file.read().strip()
|
||||
return boto3.session.Session().client(
|
||||
's3', endpoint_url=os.environ['S3_ENDPOINT'],
|
||||
region_name=os.environ.get('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
|
||||
request_checksum_calculation="when_required",
|
||||
s3={"addressing_style": "path"}))
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return os.environ['S3_BUCKET']
|
||||
|
||||
def put(self, key, file, size, content_type):
|
||||
try:
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=file,
|
||||
ContentLength=size, ContentType=content_type)
|
||||
except (BotoCoreError, ClientError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
|
||||
def delete(self, key):
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
|
||||
def download(self, row):
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
|
||||
except ClientError as exc:
|
||||
if exc.response.get('Error', {}).get('Code') in {'NoSuchKey', '404'}:
|
||||
raise NotFound('Оригинал вложения не найден') from None
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
except (BotoCoreError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
if obj['ContentLength'] != row['size_bytes']:
|
||||
obj['Body'].close()
|
||||
raise Unavailable('Размер оригинала не совпадает с метаданными')
|
||||
return BinaryResponse(obj['Body'], row['content_type'], row['size_bytes'])
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Redis session cache with TTL and graceful degradation.
|
||||
|
||||
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
|
||||
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
|
||||
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
|
||||
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
|
||||
anonymous request as trusted
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import redis
|
||||
|
||||
from .config import Config
|
||||
|
||||
logger = logging.getLogger('messenger.cache')
|
||||
|
||||
|
||||
class SessionCache:
|
||||
"""Look-aside cache for session tokens in Redis/Valkey."""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
def open(self):
|
||||
if not self._config.cache_url:
|
||||
self._available = False
|
||||
return
|
||||
try:
|
||||
self._client = redis.Redis.from_url(
|
||||
self._config.cache_url,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
socket_keepalive=True,
|
||||
health_check_interval=30,
|
||||
decode_responses=True,
|
||||
)
|
||||
self._client.ping()
|
||||
self._available = True
|
||||
logger.info('cache_connected', extra={'event': 'cache_connected'})
|
||||
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
|
||||
self._available = False
|
||||
self._client = None
|
||||
logger.warning('cache_unavailable', extra={
|
||||
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
|
||||
|
||||
def close(self):
|
||||
if self._client is not None:
|
||||
try:
|
||||
self._client.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
return self._available and self._client is not None
|
||||
|
||||
def get(self, token_hash):
|
||||
"""Return cached user_id or None (miss or unavailable)."""
|
||||
if not self.available:
|
||||
return None
|
||||
try:
|
||||
value = self._client.get('session:%s' % token_hash)
|
||||
return value
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_read_failed', extra={
|
||||
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
|
||||
self._available = False
|
||||
return None
|
||||
|
||||
def set(self, token_hash, user_id, ttl_seconds):
|
||||
"""Cache a session with TTL. Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_write_failed', extra={
|
||||
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
|
||||
self._available = False
|
||||
|
||||
def delete(self, token_hash):
|
||||
"""Remove a session from the cache (logout). Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.delete('session:%s' % token_hash)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_delete_failed', extra={
|
||||
'event': 'cache_delete_failed'})
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Remove expired upload reservations and their S3 objects."""
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
from .attachments import AttachmentStorage
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .store import Store
|
||||
from .job_lifecycle import clean_staging, clean_abandoned_outputs
|
||||
|
||||
GRACE_SECONDS = 15 * 60
|
||||
INTERVAL_SECONDS = 60
|
||||
BATCH_SIZE = 25
|
||||
|
||||
|
||||
def clean_batch(store, storage):
|
||||
count = 0
|
||||
while count < BATCH_SIZE and store.clean_pending_attachment(GRACE_SECONDS, storage.delete):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--once', action='store_true', help='Process one bounded batch and exit')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
config = Config.from_env()
|
||||
database = Database(config)
|
||||
store = Store(config, database, None)
|
||||
storage = AttachmentStorage()
|
||||
try:
|
||||
database.open()
|
||||
while True:
|
||||
try:
|
||||
deleted = clean_batch(store, storage)
|
||||
for cleanup in (clean_staging, clean_abandoned_outputs):
|
||||
for _ in range(BATCH_SIZE):
|
||||
if not cleanup(database, storage.delete):
|
||||
break
|
||||
deleted += 1
|
||||
if deleted or args.once:
|
||||
logging.info('Expired attachment reservations removed: %s', deleted)
|
||||
except Unavailable:
|
||||
logging.warning('Attachment cleanup deferred because a dependency is unavailable')
|
||||
if args.once:
|
||||
return 1
|
||||
if args.once:
|
||||
return 0
|
||||
time.sleep(INTERVAL_SECONDS)
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Runtime configuration. Every value is read from the environment."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
|
||||
# Contract limits: they cannot be raised by configuration.
|
||||
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
|
||||
CONTRACT_MAX_PAGE_LIMIT = 100
|
||||
CONTRACT_SESSION_TTL_MIN = 1
|
||||
CONTRACT_SESSION_TTL_MAX = 86400
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when an environment value cannot be used."""
|
||||
|
||||
|
||||
def _str(environ, name, default):
|
||||
value = environ.get(name)
|
||||
if value is None or value == '':
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _int(environ, name, default, minimum, maximum):
|
||||
raw = environ.get(name)
|
||||
if raw is None or raw == '':
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
|
||||
if not minimum <= value <= maximum:
|
||||
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
|
||||
return value
|
||||
|
||||
|
||||
def _log_level(environ, name, default):
|
||||
value = _str(environ, name, default).upper()
|
||||
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
|
||||
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
|
||||
return value
|
||||
|
||||
|
||||
def _secret_from_file(environ, name, default=None):
|
||||
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
|
||||
|
||||
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
|
||||
файл (например, Docker secret), а приложение читает его содержимое.
|
||||
Если файл не задан или не существует, используется значение переменной
|
||||
окружения без суффикса _FILE (удобно для локального запуска без секретов).
|
||||
"""
|
||||
path_name = name + '_FILE'
|
||||
path = environ.get(path_name)
|
||||
if path:
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as handle:
|
||||
return handle.read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
return environ.get(name) or default
|
||||
|
||||
|
||||
class Config:
|
||||
"""Effective configuration of one process instance."""
|
||||
|
||||
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
|
||||
max_text_code_points, max_display_name_code_points, max_title_code_points,
|
||||
default_page_limit, max_page_limit, max_body_bytes,
|
||||
database_url, cache_url, session_ttl_seconds, session_store,
|
||||
pool_min, pool_max, connect_timeout, statement_timeout):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.env = env
|
||||
self.log_level = log_level
|
||||
self.cursor_secret = cursor_secret
|
||||
self.shutdown_grace_seconds = shutdown_grace_seconds
|
||||
self.max_text_code_points = max_text_code_points
|
||||
self.max_display_name_code_points = max_display_name_code_points
|
||||
self.max_title_code_points = max_title_code_points
|
||||
self.default_page_limit = default_page_limit
|
||||
self.max_page_limit = max_page_limit
|
||||
self.max_body_bytes = max_body_bytes
|
||||
self.database_url = database_url
|
||||
self.cache_url = cache_url
|
||||
self.session_ttl_seconds = session_ttl_seconds
|
||||
self.session_store = session_store
|
||||
self.pool_min = pool_min
|
||||
self.pool_max = pool_max
|
||||
self.connect_timeout = connect_timeout
|
||||
self.statement_timeout = statement_timeout
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environ=None):
|
||||
environ = os.environ if environ is None else environ
|
||||
secret = _secret_from_file(environ, 'CURSOR_SECRET', '')
|
||||
if not secret:
|
||||
secret = secrets.token_hex(32)
|
||||
session_store = _str(environ, 'SESSION_STORE', 'postgres')
|
||||
if session_store not in ('postgres', 'redis'):
|
||||
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
|
||||
return cls(
|
||||
host=_str(environ, 'APP_HOST', '0.0.0.0'),
|
||||
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
|
||||
env=_str(environ, 'APP_ENV', 'lab'),
|
||||
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
|
||||
cursor_secret=secret.encode('utf-8'),
|
||||
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
|
||||
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
|
||||
1, CONTRACT_MAX_TEXT_CODE_POINTS),
|
||||
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
|
||||
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
|
||||
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
|
||||
CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
|
||||
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
|
||||
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
|
||||
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
|
||||
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
|
||||
session_store=session_store,
|
||||
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
|
||||
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
|
||||
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
|
||||
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
|
||||
)
|
||||
|
||||
def as_log_fields(self):
|
||||
"""Configuration summary safe for logs (no secrets)."""
|
||||
return {
|
||||
'app_env': self.env,
|
||||
'host': self.host,
|
||||
'port': self.port,
|
||||
'log_level': self.log_level,
|
||||
'shutdown_grace_seconds': self.shutdown_grace_seconds,
|
||||
'default_page_limit': self.default_page_limit,
|
||||
'max_page_limit': self.max_page_limit,
|
||||
'max_text_code_points': self.max_text_code_points,
|
||||
'max_body_bytes': self.max_body_bytes,
|
||||
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
|
||||
'database_url_configured': bool(self.database_url),
|
||||
'cache_url_configured': bool(self.cache_url),
|
||||
'session_ttl_seconds': self.session_ttl_seconds,
|
||||
'session_store': self.session_store,
|
||||
'db_pool_min': self.pool_min,
|
||||
'db_pool_max': self.pool_max,
|
||||
'db_connect_timeout': self.connect_timeout,
|
||||
'db_statement_timeout': self.statement_timeout,
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import uuid
|
||||
|
||||
VERSION = 1
|
||||
MAC_BYTES = 16
|
||||
|
||||
|
||||
class InvalidCursor(ValueError):
|
||||
"""Cursor is malformed, forged, expired or belongs to another resource."""
|
||||
|
||||
|
||||
def _key(secret):
|
||||
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
|
||||
|
||||
|
||||
def encode(scope, created_at, object_id, secret):
|
||||
"""Serialize a keyset position bound to one resource scope."""
|
||||
payload = json.dumps(
|
||||
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
|
||||
separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
|
||||
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode(token, scope, secret):
|
||||
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
|
||||
if not isinstance(token, str) or not token:
|
||||
raise InvalidCursor('empty cursor')
|
||||
padding = '=' * (-len(token) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(token + padding)
|
||||
except (binascii.Error, ValueError):
|
||||
raise InvalidCursor('cursor is not base64url') from None
|
||||
if len(raw) <= MAC_BYTES:
|
||||
raise InvalidCursor('cursor is too short')
|
||||
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
|
||||
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
|
||||
raise InvalidCursor('cursor signature mismatch')
|
||||
try:
|
||||
data = json.loads(payload.decode('utf-8'))
|
||||
except (ValueError, UnicodeError):
|
||||
raise InvalidCursor('cursor payload is not JSON') from None
|
||||
if not isinstance(data, dict) or data.get('v') != VERSION:
|
||||
raise InvalidCursor('unsupported cursor version')
|
||||
if data.get('scope') != scope:
|
||||
raise InvalidCursor('cursor belongs to another resource')
|
||||
created_at = data.get('created_at')
|
||||
object_id = data.get('id')
|
||||
if not isinstance(created_at, str) or not isinstance(object_id, str):
|
||||
raise InvalidCursor('cursor payload types')
|
||||
try:
|
||||
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
|
||||
identifier = uuid.UUID(object_id)
|
||||
except ValueError:
|
||||
raise InvalidCursor('cursor payload values') from None
|
||||
if stamp.tzinfo is None:
|
||||
raise InvalidCursor('cursor timestamp has no timezone')
|
||||
return stamp.astimezone(dt.timezone.utc), identifier
|
||||
@@ -0,0 +1,97 @@
|
||||
"""PostgreSQL connection pool and schema migrations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg2 import pool, OperationalError, InterfaceError
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
from .config import Config
|
||||
from .errors import Unavailable
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MIGRATIONS_DIR = _ROOT / 'migrations'
|
||||
|
||||
|
||||
class Database:
|
||||
"""Thread-safe connection pool with prepared statement timeouts.
|
||||
|
||||
All queries are parameterised (%s). Connections are returned to the pool
|
||||
after every operation; the pool itself is closed on shutdown.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._pool = None
|
||||
|
||||
def open(self):
|
||||
if self._pool is not None:
|
||||
return
|
||||
try:
|
||||
self._pool = pool.ThreadedConnectionPool(
|
||||
minconn=self._config.pool_min,
|
||||
maxconn=self._config.pool_max,
|
||||
dsn=self._config.database_url,
|
||||
connect_timeout=self._config.connect_timeout,
|
||||
cursor_factory=RealDictCursor,
|
||||
)
|
||||
except Exception as exc: # psycopg2.Error or configuration error
|
||||
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
|
||||
|
||||
def close(self):
|
||||
if self._pool is not None:
|
||||
self._pool.closeall()
|
||||
self._pool = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
"""Yield a connection; rollback on exception, return to pool on exit."""
|
||||
if self._pool is None:
|
||||
raise Unavailable('База данных не инициализирована')
|
||||
try:
|
||||
conn = self._pool.getconn()
|
||||
except (pool.PoolError, OperationalError, InterfaceError):
|
||||
raise Unavailable('Пул соединений PostgreSQL недоступен') from None
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SET statement_timeout = %s',
|
||||
(self._config.statement_timeout * 1000,))
|
||||
yield conn
|
||||
except Exception as exc:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(exc, (OperationalError, InterfaceError)):
|
||||
raise Unavailable('PostgreSQL недоступен') from None
|
||||
raise
|
||||
finally:
|
||||
self._pool.putconn(conn, close=bool(conn.closed))
|
||||
|
||||
@contextmanager
|
||||
def cursor(self):
|
||||
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
|
||||
with self.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
yield cur
|
||||
conn.commit()
|
||||
|
||||
def ping(self):
|
||||
with self.cursor() as cur:
|
||||
cur.execute('SELECT 1')
|
||||
cur.fetchone()
|
||||
|
||||
def run_migrations(self):
|
||||
"""Apply versioned migrations once, under a transaction advisory lock."""
|
||||
with self.cursor() as cur:
|
||||
cur.execute('SELECT pg_advisory_xact_lock(3072026)')
|
||||
cur.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(_MIGRATIONS_DIR.glob('*.sql')):
|
||||
cur.execute('SELECT 1 FROM schema_migrations WHERE name = %s', (path.name,))
|
||||
if cur.fetchone() is not None:
|
||||
continue
|
||||
cur.execute(path.read_text(encoding='utf-8'))
|
||||
cur.execute('INSERT INTO schema_migrations(name) VALUES (%s)', (path.name,))
|
||||
@@ -0,0 +1,73 @@
|
||||
"""API error taxonomy; every message is safe to return to a client."""
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""Base class for errors that map onto the public error payload."""
|
||||
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.default_message
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
status = 400
|
||||
code = 'invalid_request'
|
||||
default_message = 'Некорректный запрос'
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
status = 401
|
||||
code = 'unauthorized'
|
||||
default_message = 'Требуется идентификация пользователя'
|
||||
www_authenticate = None
|
||||
|
||||
def __init__(self, message=None, www_authenticate=None):
|
||||
self.message = message or self.default_message
|
||||
self.www_authenticate = www_authenticate
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
status = 404
|
||||
code = 'not_found'
|
||||
default_message = 'Объект не найден'
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
status = 409
|
||||
code = 'conflict'
|
||||
default_message = 'Конфликт состояния'
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
status = 413
|
||||
code = 'payload_too_large'
|
||||
default_message = 'Тело запроса слишком велико'
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
status = 415
|
||||
code = 'unsupported_media_type'
|
||||
default_message = 'Поддерживается только application/json'
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
status = 429
|
||||
code = 'rate_limited'
|
||||
default_message = 'Слишком много запросов'
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
status = 503
|
||||
code = 'unavailable'
|
||||
default_message = 'Сервис временно недоступен'
|
||||
|
||||
|
||||
class InternalError(ApiError):
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
@@ -0,0 +1,39 @@
|
||||
"""List failed jobs and explicitly replay a failed job with retained staging."""
|
||||
import argparse
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import ApiError
|
||||
from .job_lifecycle import replay
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
actions = parser.add_subparsers(dest='action', required=True)
|
||||
actions.add_parser('list')
|
||||
command = actions.add_parser('replay')
|
||||
command.add_argument('job_id', type=UUID)
|
||||
command.add_argument('--operator', required=True)
|
||||
args = parser.parse_args()
|
||||
database = Database(Config.from_env())
|
||||
database.open()
|
||||
try:
|
||||
if args.action == 'replay':
|
||||
print(json.dumps({'job_id': str(args.job_id), 'attempt': replay(database, str(args.job_id), args.operator)}))
|
||||
else:
|
||||
with database.cursor() as cur:
|
||||
cur.execute("SELECT j.id AS job_id,a.id AS attachment_id,a.error_code,j.attempt, "
|
||||
"j.staging_cleaned_at IS NULL AS staging_retained "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE a.status='failed' ORDER BY a.updated_at DESC LIMIT 100")
|
||||
print(json.dumps(cur.fetchall(), default=str))
|
||||
except ApiError as exc:
|
||||
parser.exit(1, str(exc) + '\n')
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Transactional claims, bounded retries and explicit failed-job recovery."""
|
||||
import os
|
||||
import random
|
||||
|
||||
from .errors import Conflict, NotFound
|
||||
|
||||
LEASE_SECONDS = 60
|
||||
STAGING_RETENTION_SECONDS = 86400
|
||||
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
||||
if not 1 <= MAX_ATTEMPTS <= 20:
|
||||
raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20')
|
||||
|
||||
|
||||
def _locked_job(cur, job_id):
|
||||
cur.execute(
|
||||
'SELECT j.*, a.status, a.updated_at, a.error_code, '
|
||||
'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j '
|
||||
'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a',
|
||||
(job_id,))
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def claim(db, delivery):
|
||||
with db.cursor() as cur:
|
||||
row = _locked_job(cur, delivery['job_id'])
|
||||
if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt']
|
||||
or str(row['attachment_id']) != delivery['attachment_id']
|
||||
or row['pipeline_version'] != delivery['pipeline_version']):
|
||||
return None
|
||||
cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), '
|
||||
'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id']))
|
||||
cur.execute("UPDATE attachments SET status='processing', updated_at=now() "
|
||||
"WHERE id=%s RETURNING *", (row['attachment_id'],))
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def _requeue(cur, row, delay, reset=False):
|
||||
attempt = row['attempt'] + 1
|
||||
cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s',
|
||||
(attempt, 0 if reset else row['attempts_used'], row['id']))
|
||||
cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s",
|
||||
(row['attachment_id'],))
|
||||
cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) '
|
||||
'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay))
|
||||
return attempt
|
||||
|
||||
|
||||
def _fail_or_retry(cur, row, error_code):
|
||||
if error_code is None and row['attempts_used'] < MAX_ATTEMPTS:
|
||||
delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25)
|
||||
_requeue(cur, row, delay)
|
||||
return
|
||||
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
|
||||
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s",
|
||||
(error_code or 'retry_exhausted', row['attachment_id']))
|
||||
|
||||
|
||||
def complete(db, delivery, result=None, error_code=None):
|
||||
with db.cursor() as cur:
|
||||
row = _locked_job(cur, delivery['job_id'])
|
||||
if (not row or row['status'] != 'processing'
|
||||
or row['attempt'] != delivery['attempt'] or not row['lease_valid']):
|
||||
return False
|
||||
if result is None:
|
||||
_fail_or_retry(cur, row, error_code)
|
||||
return True
|
||||
key, variants = result
|
||||
cur.executemany(
|
||||
'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,'
|
||||
'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
|
||||
[(row['attachment_id'], row['pipeline_version'], *item) for item in variants])
|
||||
cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s",
|
||||
(key, row['attachment_id']))
|
||||
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
|
||||
return True
|
||||
|
||||
|
||||
def recover_expired(db):
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE a.status='processing' AND j.lease_until <= now() "
|
||||
"ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED")
|
||||
for row in cur.fetchall():
|
||||
_fail_or_retry(cur, row, None)
|
||||
|
||||
|
||||
def replay(db, job_id, operator):
|
||||
with db.cursor() as cur:
|
||||
row = _locked_job(cur, job_id)
|
||||
if row is None:
|
||||
raise NotFound('Задача не найдена')
|
||||
if row['status'] != 'failed' or row['staging_cleaned_at'] is not None:
|
||||
raise Conflict('Повтор доступен только для failed с сохранённым исходником')
|
||||
attempt = _requeue(cur, row, 0, reset=True)
|
||||
cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)',
|
||||
(job_id, attempt, operator))
|
||||
return attempt
|
||||
|
||||
|
||||
def clean_staging(db, remove):
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT j.id,j.attachment_id FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR "
|
||||
"(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) "
|
||||
"LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
remove('attachments/staging/' + str(row['attachment_id']))
|
||||
cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],))
|
||||
return True
|
||||
|
||||
|
||||
def clean_abandoned_outputs(db, remove):
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt "
|
||||
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' "
|
||||
"AND (o.attempt < j.attempt OR (a.status='failed' AND "
|
||||
"a.updated_at < now()-interval '15 minutes')) "
|
||||
"ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED")
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row)
|
||||
for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'):
|
||||
remove(prefix + name)
|
||||
cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s',
|
||||
(row['id'], row['attempt']))
|
||||
return True
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Structured stdout logging without payload contents."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
|
||||
'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line; message text of API payloads is never logged."""
|
||||
|
||||
def format(self, record):
|
||||
payload = {
|
||||
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
|
||||
'.%03dZ' % int(record.msecs),
|
||||
'level': record.levelname.lower(),
|
||||
'logger': record.name,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
for field in _FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None:
|
||||
payload[field] = value
|
||||
if record.exc_info and record.levelno >= logging.ERROR:
|
||||
payload['exception'] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def configure(level_name):
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
for existing in list(root.handlers):
|
||||
root.removeHandler(existing)
|
||||
root.addHandler(handler)
|
||||
root.setLevel(getattr(logging, level_name))
|
||||
return logging.getLogger('messenger')
|
||||
|
||||
|
||||
def access_logger():
|
||||
return logging.getLogger('messenger.access')
|
||||
@@ -0,0 +1,17 @@
|
||||
"""One-shot schema migration entry point; API processes never migrate."""
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
|
||||
|
||||
def main():
|
||||
database = Database(Config.from_env())
|
||||
try:
|
||||
database.open()
|
||||
database.run_migrations()
|
||||
print('PASS: migrations applied')
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Publish committed attachment jobs using RabbitMQ publisher confirms."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
import pika
|
||||
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .store import rfc3339
|
||||
from .job_lifecycle import recover_expired
|
||||
|
||||
QUEUE = 'attachment.process.v1'
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def connect():
|
||||
password = Path(os.environ['RABBITMQ_PASSWORD_FILE']).read_text().strip()
|
||||
return pika.BlockingConnection(pika.ConnectionParameters(
|
||||
host='rabbitmq', credentials=pika.PlainCredentials('lab5', password),
|
||||
heartbeat=30, socket_timeout=3, stack_timeout=5, blocked_connection_timeout=5))
|
||||
|
||||
|
||||
def publish_one(database, channel):
|
||||
with database.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT j.id, j.attachment_id, j.pipeline_version, j.created_at, o.attempt '
|
||||
'FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id '
|
||||
'WHERE o.published_at IS NULL AND o.available_at <= now() '
|
||||
'ORDER BY o.available_at, o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED')
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return False
|
||||
job = {
|
||||
'schema_version': 1, 'job_id': str(row['id']), 'type': QUEUE,
|
||||
'attachment_id': str(row['attachment_id']),
|
||||
'pipeline_version': row['pipeline_version'], 'attempt': row['attempt'],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
channel.basic_publish(
|
||||
exchange='', routing_key=QUEUE, body=json.dumps(job).encode(),
|
||||
properties=pika.BasicProperties(content_type='application/json',
|
||||
delivery_mode=2, message_id=job['job_id']),
|
||||
mandatory=True)
|
||||
cur.execute('UPDATE attachment_outbox SET published_at=now() WHERE job_id=%s AND attempt=%s',
|
||||
(row['id'], row['attempt']))
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
database = Database(Config.from_env())
|
||||
database.open()
|
||||
try:
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.confirm_delivery()
|
||||
next_recovery = 0
|
||||
while True:
|
||||
if time.monotonic() >= next_recovery:
|
||||
recover_expired(database)
|
||||
next_recovery = time.monotonic() + 1
|
||||
if not publish_one(database, channel):
|
||||
connection.sleep(.25)
|
||||
else:
|
||||
connection.process_data_events()
|
||||
except (pika.exceptions.AMQPError, OSError, Unavailable) as exc:
|
||||
log.warning('Publisher will retry: %s', type(exc).__name__)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection and connection.is_open:
|
||||
try:
|
||||
connection.close()
|
||||
except pika.exceptions.AMQPError:
|
||||
pass
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Password hashing and session tokens.
|
||||
|
||||
Passwords are hashed with a library password KDF: scrypt from the standard
|
||||
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
|
||||
passwords longer than bcrypt's 72-byte limit are still supported.
|
||||
Session tokens are opaque random values (>= 256 bits of entropy); only their
|
||||
SHA-256 is stored in the database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
import bcrypt
|
||||
|
||||
try: # scrypt is part of hashlib on OpenSSL-enabled builds
|
||||
hashlib.scrypt(b'x', salt=b's', n=16, r=8, p=1, dklen=16)
|
||||
_HAS_SCRYPT = True
|
||||
except Exception:
|
||||
_HAS_SCRYPT = False
|
||||
|
||||
if not _HAS_SCRYPT:
|
||||
import bcrypt # type: ignore
|
||||
|
||||
SCRYPT_N = 16384
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
SCRYPT_DKLEN = 32
|
||||
SALT_BYTES = 16
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a UTF-8 password with a library password KDF.
|
||||
|
||||
Returns a single storable string:
|
||||
scrypt:$n:$r:$p:$salt_b64:$hash_b64
|
||||
or
|
||||
bcrypt:$salt_b64:$hash_b64
|
||||
"""
|
||||
raw = password.encode('utf-8')
|
||||
if _HAS_SCRYPT:
|
||||
salt = secrets.token_bytes(SALT_BYTES)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=SCRYPT_N,
|
||||
r=SCRYPT_R,
|
||||
p=SCRYPT_P,
|
||||
dklen=SCRYPT_DKLEN,
|
||||
)
|
||||
return 'scrypt:%d:%d:%d:%s:%s' % (
|
||||
SCRYPT_N, SCRYPT_R, SCRYPT_P,
|
||||
base64.b64encode(salt).decode('ascii'),
|
||||
base64.b64encode(dk).decode('ascii'),
|
||||
)
|
||||
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
|
||||
# (up to 128 code points per contract) are fully covered.
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
|
||||
return 'bcrypt:%s' % hashed.decode('ascii')
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Verify a password against a KDF hash string. Constant-time compare."""
|
||||
try:
|
||||
raw = password.encode('utf-8')
|
||||
if password_hash.startswith('scrypt:'):
|
||||
parts = password_hash.split(':')
|
||||
if len(parts) != 6 or parts[0] != 'scrypt':
|
||||
return False
|
||||
_, n, r, p, salt_b64, hash_b64 = parts
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(hash_b64)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(expected),
|
||||
)
|
||||
return hmac.compare_digest(dk, expected)
|
||||
if password_hash.startswith('bcrypt:'):
|
||||
stored = password_hash[len('bcrypt:'):].encode('ascii')
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
return bcrypt.checkpw(digest, stored)
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Opaque random bearer token, at least 256 bits of entropy."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
|
||||
return hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
@@ -0,0 +1,305 @@
|
||||
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import __version__, logging_utils
|
||||
from .api import Api, Request
|
||||
from .attachments import BinaryResponse, CHUNK_SIZE, MAX_UPLOAD_BYTES
|
||||
from .cache import SessionCache
|
||||
from .config import Config, ConfigError
|
||||
from .db import Database
|
||||
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
|
||||
from .store import Store
|
||||
|
||||
MAX_REQUEST_ID_LENGTH = 128
|
||||
MAX_LOGGED_LINE = 200
|
||||
|
||||
|
||||
def sanitize_request_id(raw):
|
||||
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
|
||||
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
|
||||
return raw
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def route_template(path):
|
||||
for _method, pattern, _handler in Api.ROUTES:
|
||||
if pattern.match(path):
|
||||
return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$')
|
||||
return 'unmatched'
|
||||
|
||||
|
||||
def encode_payload(payload):
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
|
||||
|
||||
class ApiServer(ThreadingHTTPServer):
|
||||
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address, api, config, logger, database):
|
||||
self.api = api
|
||||
self.config = config
|
||||
self.logger = logger
|
||||
self.database = database
|
||||
self.instance_id = os.environ.get("HOSTNAME") or uuid.uuid4().hex
|
||||
self.shutting_down = False
|
||||
self._active = 0
|
||||
self._idle = threading.Condition()
|
||||
super().__init__(address, ApiHandler)
|
||||
|
||||
def enter(self):
|
||||
with self._idle:
|
||||
self._active += 1
|
||||
|
||||
def leave(self):
|
||||
with self._idle:
|
||||
self._active -= 1
|
||||
if self._active == 0:
|
||||
self._idle.notify_all()
|
||||
|
||||
def wait_idle(self, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
with self._idle:
|
||||
while self._active > 0:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
self._idle.wait(remaining)
|
||||
return True
|
||||
|
||||
|
||||
class ApiHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
server_version = 'mtusi-messenger/' + __version__
|
||||
sys_version = ''
|
||||
timeout = 30
|
||||
|
||||
# --------------------------------------------------------- entry points
|
||||
def do_GET(self):
|
||||
self._handle()
|
||||
|
||||
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
"""Base-class chatter stays debug-only; access logs are structured."""
|
||||
self.server.logger.debug('http_note', extra={
|
||||
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
|
||||
|
||||
# ------------------------------------------------------------- pipeline
|
||||
def _handle(self):
|
||||
started = time.monotonic()
|
||||
server = self.server
|
||||
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
|
||||
head_only = self.command == 'HEAD'
|
||||
path = urlsplit(self.path).path
|
||||
status, error_code = 500, None
|
||||
self._last_exc = None
|
||||
server.enter()
|
||||
try:
|
||||
if server.shutting_down:
|
||||
raise Unavailable('Сервер завершает работу')
|
||||
if self.command == 'POST' and Api.UPLOAD_PATH.fullmatch(path):
|
||||
# Rejected uploads may leave unread bytes in the connection.
|
||||
self.close_connection = True
|
||||
body = self._upload_chunks
|
||||
else:
|
||||
body = self._read_body()
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
request = Request('GET' if head_only else self.command, path, query, self.headers,
|
||||
body, request_id, self.client_address[0])
|
||||
status, payload = server.api.dispatch(request)
|
||||
encoded = payload if isinstance(payload, BinaryResponse) else encode_payload(payload)
|
||||
except ApiError as exc:
|
||||
self._last_exc = exc
|
||||
status, error_code = exc.status, exc.code
|
||||
encoded = self._error_payload(exc, request_id)
|
||||
except Exception: # last resort: log privately, answer generically
|
||||
server.logger.exception('unhandled_error', extra={
|
||||
'event': 'unhandled_error', 'request_id': request_id,
|
||||
'method': self.command, 'route': route_template(path)})
|
||||
status, error_code = 500, 'internal_error'
|
||||
encoded = self._error_payload(ApiError(), request_id)
|
||||
finally:
|
||||
server.logger.info('http_request', extra={
|
||||
'event': 'http_request', 'request_id': request_id, 'method': self.command,
|
||||
'route': route_template(path), 'status': status,
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3),
|
||||
'error_code': error_code, 'client': self.client_address[0]})
|
||||
try:
|
||||
self._send(status, encoded, request_id, head_only)
|
||||
finally:
|
||||
if isinstance(encoded, BinaryResponse):
|
||||
encoded.close()
|
||||
server.leave()
|
||||
|
||||
def _read_body(self):
|
||||
if self.headers.get('Transfer-Encoding'):
|
||||
self.close_connection = True
|
||||
raise InvalidRequest('Transfer-Encoding не поддерживается')
|
||||
raw_length = self.headers.get('Content-Length')
|
||||
if raw_length is None:
|
||||
return b''
|
||||
try:
|
||||
length = int(raw_length.strip())
|
||||
except ValueError:
|
||||
raise InvalidRequest('Некорректный Content-Length') from None
|
||||
if length < 0:
|
||||
raise InvalidRequest('Некорректный Content-Length')
|
||||
if length > self.server.config.max_body_bytes:
|
||||
self.close_connection = True
|
||||
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
|
||||
if length == 0:
|
||||
return b''
|
||||
return self.rfile.read(length)
|
||||
|
||||
def _upload_chunks(self):
|
||||
if self.headers.get('Transfer-Encoding'):
|
||||
raise InvalidRequest('Требуется Content-Length')
|
||||
lengths = self.headers.get_all('Content-Length') or []
|
||||
try:
|
||||
length = int(lengths[0]) if len(lengths) == 1 else -1
|
||||
except ValueError:
|
||||
length = -1
|
||||
if length < 0:
|
||||
raise InvalidRequest('Некорректный Content-Length')
|
||||
if length > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLarge()
|
||||
deadline = time.monotonic() + 30
|
||||
while length:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise InvalidRequest('Истекло время загрузки')
|
||||
self.connection.settimeout(min(5, remaining))
|
||||
try:
|
||||
chunk = self.rfile.read1(min(CHUNK_SIZE, length))
|
||||
except OSError:
|
||||
raise InvalidRequest('Загрузка прервана') from None
|
||||
finally:
|
||||
self.connection.settimeout(self.timeout)
|
||||
if not chunk:
|
||||
raise InvalidRequest('Загрузка прервана')
|
||||
length -= len(chunk)
|
||||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
def _error_payload(exc, request_id):
|
||||
return encode_payload({'error': {
|
||||
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
|
||||
|
||||
def _send(self, status, encoded, request_id, head_only):
|
||||
try:
|
||||
self.send_response(status)
|
||||
binary = isinstance(encoded, BinaryResponse)
|
||||
self.send_header('Content-Type', encoded.content_type if binary else 'application/json; charset=utf-8')
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
if status == 204:
|
||||
self.send_header('Content-Length', '0')
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
self.end_headers()
|
||||
return
|
||||
self.send_header('Content-Length', str(encoded.size if binary else len(encoded)))
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
if status == 401 and self._last_exc is not None:
|
||||
www_auth = getattr(self._last_exc, 'www_authenticate', None)
|
||||
if www_auth:
|
||||
self.send_header('WWW-Authenticate', www_auth)
|
||||
if self.close_connection:
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
if not head_only and encoded:
|
||||
if binary:
|
||||
for chunk in encoded.chunks():
|
||||
self.wfile.write(chunk)
|
||||
else:
|
||||
self.wfile.write(encoded)
|
||||
except OSError:
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def build_server(config, logger, api, database):
|
||||
"""Create the listening server; raises OSError when the port is taken."""
|
||||
return ApiServer((config.host, config.port), api, config, logger, database)
|
||||
|
||||
|
||||
def run(config, logger, database, cache):
|
||||
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
|
||||
api = Api(config, Store(config, database, cache), database)
|
||||
try:
|
||||
server = build_server(config, logger, api, database)
|
||||
except OSError as exc:
|
||||
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
|
||||
return 1
|
||||
stop = threading.Event()
|
||||
|
||||
def handle_signal(signum, _frame):
|
||||
if stop.is_set():
|
||||
return
|
||||
stop.set()
|
||||
server.shutting_down = True
|
||||
logger.warning('shutdown_signal', extra={
|
||||
'event': 'shutdown_signal', 'signal': signum,
|
||||
'detail': 'stop accepting new requests'})
|
||||
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
|
||||
|
||||
for name in ('SIGTERM', 'SIGINT'):
|
||||
if hasattr(signal, name):
|
||||
signal.signal(getattr(signal, name), handle_signal)
|
||||
|
||||
logger.info('server_started', extra=dict(
|
||||
{'event': 'server_started', 'version': __version__,
|
||||
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
|
||||
**config.as_log_fields()))
|
||||
started = time.monotonic()
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
finally:
|
||||
server.server_close()
|
||||
drained = server.wait_idle(config.shutdown_grace_seconds)
|
||||
database.close()
|
||||
logger.info('server_stopped', extra={
|
||||
'event': 'server_stopped',
|
||||
'outcome': 'graceful' if drained else 'grace_period_exceeded',
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
config = Config.from_env()
|
||||
except ConfigError as exc:
|
||||
print('configuration error: %s' % exc, file=sys.stderr)
|
||||
return 2
|
||||
logger = logging_utils.configure(config.log_level)
|
||||
database = Database(config)
|
||||
cache = SessionCache(config)
|
||||
try:
|
||||
database.open()
|
||||
cache.open()
|
||||
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
|
||||
logger.error('database_start_failed', extra={
|
||||
'event': 'database_start_failed', 'detail': str(exc)})
|
||||
database.close()
|
||||
cache.close()
|
||||
return 1
|
||||
try:
|
||||
return run(config, logger, database, cache)
|
||||
finally:
|
||||
cache.close()
|
||||
@@ -0,0 +1,390 @@
|
||||
"""PostgreSQL-backed state store. All writes go through parameterised queries
|
||||
inside explicit transactions; reads use keyset pagination over indexed columns."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from psycopg2.extras import Json
|
||||
from psycopg2.errors import UniqueViolation, ForeignKeyViolation
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from .cache import SessionCache
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Conflict, InvalidRequest, NotFound
|
||||
|
||||
|
||||
def utc_now():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def rfc3339(stamp):
|
||||
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
|
||||
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL state: users, chats, membership, messages, sessions."""
|
||||
|
||||
def __init__(self, config: Config, database: Database, cache: SessionCache):
|
||||
self._config = config
|
||||
self._db = database
|
||||
self._cache = cache
|
||||
|
||||
# ---------------------------------------------------------------- reads
|
||||
def stats(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
|
||||
' (SELECT count(*) FROM chats) AS chats,'
|
||||
' (SELECT count(*) FROM messages) AS messages')
|
||||
row = cur.fetchone()
|
||||
return {
|
||||
'users': row['users'],
|
||||
'chats': row['chats'],
|
||||
'messages': row['messages'],
|
||||
}
|
||||
|
||||
def user(self, user_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
|
||||
(str(user_id),))
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row) if row else None
|
||||
|
||||
def user_by_username(self, username):
|
||||
"""Return user row including password_hash, or None."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, password_hash, display_name, created_at FROM users '
|
||||
'WHERE username = %s', (username,))
|
||||
return cur.fetchone()
|
||||
|
||||
def users_exist(self, identifiers):
|
||||
if not identifiers:
|
||||
return True
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
|
||||
([str(i) for i in identifiers],))
|
||||
row = cur.fetchone()
|
||||
return row['c'] == len(identifiers)
|
||||
|
||||
# --------------------------------------------------------------- writes
|
||||
def create_user(self, username, display_name, password_hash):
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, username, display_name, created_at',
|
||||
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
|
||||
except UniqueViolation:
|
||||
# Unique violation on username -> conflict, rolled back by cursor context.
|
||||
raise Conflict('username уже занят') from None
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row)
|
||||
|
||||
def create_chat(self, title, member_ids):
|
||||
"""Atomically create a chat and its membership. Rollback on any failure."""
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(identifier, title, stamp))
|
||||
for member_id in member_ids:
|
||||
cur.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
|
||||
(identifier, str(member_id)))
|
||||
except ForeignKeyViolation:
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
|
||||
return {
|
||||
'id': identifier,
|
||||
'title': title,
|
||||
'member_ids': list(member_ids),
|
||||
'created_at': rfc3339(stamp),
|
||||
}
|
||||
|
||||
def reserve_attachment(self, aid, chat_id, owner_id, filename, mime, size, digest, key):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, "
|
||||
"size_bytes, sha256, object_key, status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')",
|
||||
(aid, chat_id, owner_id, filename, mime, size, digest, key))
|
||||
|
||||
def queue_attachment(self, aid, put_object):
|
||||
"""Keep a row lock from before PUT until staging metadata, job and outbox commit."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute("SELECT object_key FROM attachments WHERE id=%s "
|
||||
"AND status='pending' FOR UPDATE", (aid,))
|
||||
pending = cur.fetchone()
|
||||
if pending is None:
|
||||
raise Conflict('Загрузка вложения не может быть завершена')
|
||||
put_object(pending['object_key'])
|
||||
job_id = str(uuid.uuid4())
|
||||
cur.execute('INSERT INTO attachment_jobs (id, attachment_id, pipeline_version) '
|
||||
'VALUES (%s,%s,1)', (job_id, aid))
|
||||
cur.execute('INSERT INTO attachment_outbox (job_id, attempt) VALUES (%s,1)', (job_id,))
|
||||
cur.execute("UPDATE attachments SET status='queued', updated_at=now() "
|
||||
"WHERE id=%s RETURNING *", (aid,))
|
||||
return cur.fetchone()
|
||||
|
||||
def clean_pending_attachment(self, grace_seconds, delete_object):
|
||||
"""Remove one expired reservation and its object, or leave it for retry."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status='pending' "
|
||||
"AND updated_at < now() - make_interval(secs => %s) "
|
||||
"ORDER BY updated_at, id LIMIT 1 FOR UPDATE SKIP LOCKED",
|
||||
(grace_seconds,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
delete_object(row['object_key'])
|
||||
cur.execute('DELETE FROM attachments WHERE id=%s', (row['id'],))
|
||||
return True
|
||||
|
||||
def visible_attachment(self, aid, user_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s AND a.status<>'pending'", (aid, user_id))
|
||||
return cur.fetchone()
|
||||
|
||||
def attachment_variants(self, aid):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT ON (variant) variant AS name, object_key, content_type, '
|
||||
'size_bytes, sha256, width, height FROM attachment_variants '
|
||||
'WHERE attachment_id=%s ORDER BY variant, pipeline_version DESC', (aid,))
|
||||
return cur.fetchall()
|
||||
|
||||
@staticmethod
|
||||
def public_attachment(row, variants=()):
|
||||
return {
|
||||
'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
|
||||
'filename': row['filename'], 'content_type': row['content_type'],
|
||||
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
|
||||
'error_code': row['error_code'],
|
||||
'variants': [{k: v for k, v in item.items() if k != 'object_key'} for item in variants],
|
||||
'created_at': rfc3339(row['created_at']), 'updated_at': rfc3339(row['updated_at']),
|
||||
}
|
||||
|
||||
def add_message(self, chat_id, sender_id, text, key, attachment_ids=()):
|
||||
"""One transaction owns both the message and its replayable result.
|
||||
|
||||
The advisory lock serializes only the same user/method/path/key across
|
||||
processes. Hash collisions serialize extra requests, never alias keys:
|
||||
the complete key is still checked against the primary key in the table.
|
||||
A crash before COMMIT rolls back the message, links and replay record. A lost reply after COMMIT
|
||||
is recovered by replay. Transient failures are never recorded.
|
||||
"""
|
||||
path = '/api/v1/chats/%s/messages' % chat_id
|
||||
body = {'text': text}
|
||||
# Keep the lab3 digest for text-only requests, including explicit [].
|
||||
if attachment_ids:
|
||||
body['attachment_ids'] = list(attachment_ids)
|
||||
body_hash = hashlib.sha256(json.dumps(body, ensure_ascii=False,
|
||||
sort_keys=True).encode('utf-8')).hexdigest()
|
||||
scope = json.dumps([str(sender_id), 'POST', path, key], separators=(',', ':'))
|
||||
lock_id = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('SELECT pg_advisory_xact_lock(%s)', (lock_id,))
|
||||
cur.execute('SELECT request_hash, response_body FROM message_idempotency '
|
||||
'WHERE user_id=%s AND method=%s AND path=%s AND key=%s',
|
||||
(str(sender_id), 'POST', path, key))
|
||||
replay = cur.fetchone()
|
||||
if replay is not None:
|
||||
if replay['request_hash'] != body_hash:
|
||||
raise Conflict('Idempotency-Key уже использован с другим телом запроса')
|
||||
return {'attachment_ids': [], **replay['response_body']}
|
||||
if attachment_ids:
|
||||
cur.execute(
|
||||
'SELECT id, status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) FOR SHARE',
|
||||
(str(chat_id), list(attachment_ids)))
|
||||
attachments = cur.fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
raise NotFound('Вложение не найдено')
|
||||
if any(row['status'] not in ('ready', 'queued', 'processing') for row in attachments):
|
||||
raise Conflict('Вложение не готово')
|
||||
cur.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, chat_id, sender_id, text, created_at',
|
||||
(identifier, str(chat_id), str(sender_id), text, stamp))
|
||||
result = self._public_message(cur.fetchone())
|
||||
if attachment_ids:
|
||||
cur.executemany(
|
||||
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
|
||||
'VALUES (%s,%s,%s,%s)',
|
||||
[(identifier, aid, str(chat_id), pos) for pos, aid in enumerate(attachment_ids)])
|
||||
result['attachment_ids'] = list(attachment_ids)
|
||||
cur.execute('INSERT INTO message_idempotency '
|
||||
'(user_id, method, path, key, request_hash, response_status, response_body) '
|
||||
'VALUES (%s,%s,%s,%s,%s,201,%s)',
|
||||
(str(sender_id), 'POST', path, key, body_hash, Json(result)))
|
||||
return result
|
||||
|
||||
# --------------------------------------------------------------- pages
|
||||
def page_messages(self, chat_id, after_key, limit):
|
||||
"""Return (public messages, has_more) for a keyset window."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s', (str(chat_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'AND (created_at, id) > (%s, %s) '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s',
|
||||
(str(chat_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
items = [self._public_message(row) for row in rows[:limit]]
|
||||
if items:
|
||||
by_id = {item['id']: item for item in items}
|
||||
cur.execute(
|
||||
'SELECT message_id, attachment_id FROM message_attachments '
|
||||
'WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position',
|
||||
(list(by_id),))
|
||||
for row in cur.fetchall():
|
||||
by_id[str(row['message_id'])]['attachment_ids'].append(str(row['attachment_id']))
|
||||
has_more = len(rows) > limit
|
||||
return items, has_more
|
||||
|
||||
def page_chats(self, user_id, after_key, limit):
|
||||
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s', (str(user_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'AND (c.created_at, c.id) > (%s, %s) '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s',
|
||||
(str(user_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
has_more = len(rows) > limit
|
||||
items = [self._public_chat(dict(row)) for row in rows[:limit]]
|
||||
return items, has_more
|
||||
|
||||
# ----------------------------------------------------------- projection
|
||||
def chat(self, chat_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
chat = self._public_chat(dict(row))
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
|
||||
return chat
|
||||
|
||||
@staticmethod
|
||||
def is_member(chat, user_id):
|
||||
return str(user_id) in chat['member_ids']
|
||||
|
||||
@staticmethod
|
||||
def _public_user(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_chat(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'title': row['title'],
|
||||
'member_ids': [], # filled by API when needed
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_message(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'sender_id': str(row['sender_id']),
|
||||
'text': row['text'],
|
||||
'attachment_ids': [],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- sessions
|
||||
def create_session(self, token_hash, user_id, expires_at):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
|
||||
'VALUES (%s, %s, %s, %s)',
|
||||
(token_hash, str(user_id), expires_at, utc_now()))
|
||||
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, str(user_id), ttl)
|
||||
|
||||
def get_session(self, token_hash):
|
||||
"""Return user_id for a valid non-expired session, or None.
|
||||
|
||||
The cache is consulted first, but every cache hit is re-validated
|
||||
against PostgreSQL so that a revoked or expired session is never
|
||||
trusted from a stale Redis key. On miss, read PostgreSQL and
|
||||
backfill the cache. Expired sessions found during lookup are deleted.
|
||||
"""
|
||||
cached = self._cache.get(token_hash)
|
||||
if cached is not None:
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
|
||||
(token_hash, utc_now()))
|
||||
if cur.fetchone() is None:
|
||||
self._cache.delete(token_hash)
|
||||
return None
|
||||
return cached
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
|
||||
(token_hash,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
if row['expires_at'] <= utc_now():
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
return None
|
||||
user_id = str(row['user_id'])
|
||||
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, user_id, ttl)
|
||||
return user_id
|
||||
|
||||
def delete_session(self, token_hash):
|
||||
self._cache.delete(token_hash)
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
|
||||
def cleanup_expired_sessions(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Strict request payload validation; any deviation is 400 invalid_request."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from .errors import InvalidRequest
|
||||
|
||||
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
|
||||
DIGITS_PATTERN = re.compile(r'[0-9]+')
|
||||
|
||||
|
||||
def safe_label(value):
|
||||
"""Render untrusted text for an error message without breaking its encoding.
|
||||
|
||||
Field names come straight from the client JSON, so they may contain the same
|
||||
lone surrogates that the payload itself is rejected for.
|
||||
"""
|
||||
text = value if isinstance(value, str) else repr(value)
|
||||
return text.encode('utf-8', 'replace').decode('utf-8')
|
||||
|
||||
|
||||
def require_encodable(value, name):
|
||||
"""Reject text that cannot survive a UTF-8 round trip.
|
||||
|
||||
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
|
||||
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
|
||||
otherwise the write is committed and the response encoding fails afterwards.
|
||||
"""
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise InvalidRequest(
|
||||
'Поле %s содержит недопустимые символы Unicode' % name) from None
|
||||
return value
|
||||
|
||||
|
||||
def require_object(payload):
|
||||
if not isinstance(payload, dict):
|
||||
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
|
||||
return payload
|
||||
|
||||
|
||||
def require_fields(payload, required, allowed):
|
||||
"""Reject missing required fields and any unknown field."""
|
||||
missing = [name for name in required if name not in payload]
|
||||
if missing:
|
||||
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
|
||||
extra = [name for name in payload if name not in allowed]
|
||||
if extra:
|
||||
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
|
||||
return payload
|
||||
|
||||
|
||||
def require_text_field(payload, name, minimum, maximum, strip_check=True):
|
||||
value = payload.get(name)
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой' % name)
|
||||
require_encodable(value, name)
|
||||
if strip_check and not value.strip():
|
||||
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
|
||||
if len(value) < minimum:
|
||||
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
|
||||
if len(value) > maximum:
|
||||
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
|
||||
return value
|
||||
|
||||
|
||||
def require_username(payload):
|
||||
username = payload.get('username')
|
||||
# fullmatch, not match: a trailing "$" would also accept a final newline.
|
||||
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
|
||||
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
|
||||
return require_encodable(username, 'username')
|
||||
|
||||
|
||||
def require_uuid_string(value, name):
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
|
||||
|
||||
|
||||
def require_attachment_ids(payload):
|
||||
value = payload.get('attachment_ids', [])
|
||||
if not isinstance(value, list) or len(value) > 10:
|
||||
raise InvalidRequest('attachment_ids должен содержать от 0 до 10 UUID')
|
||||
identifiers = [require_uuid_string(item, 'attachment_ids') for item in value]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise InvalidRequest('attachment_ids содержит повторяющиеся UUID')
|
||||
return identifiers
|
||||
|
||||
|
||||
def require_member_ids(payload, maximum, minimum=1):
|
||||
value = payload.get('member_ids')
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest('member_ids должен быть массивом UUID')
|
||||
if not minimum <= len(value) <= maximum:
|
||||
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
|
||||
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise InvalidRequest('member_ids содержит повторяющихся участников')
|
||||
return identifiers
|
||||
|
||||
|
||||
def parse_limit(raw, default, maximum):
|
||||
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
|
||||
|
||||
An empty value is a client error rather than an absent parameter, and the
|
||||
digit run is length-bounded before int() because Python refuses to convert
|
||||
strings longer than its integer-string limit, which would surface as a 500.
|
||||
"""
|
||||
if raw is None:
|
||||
return default
|
||||
if not DIGITS_PATTERN.fullmatch(raw):
|
||||
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
|
||||
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
|
||||
# significant digits against the maximum first rejects them without converting.
|
||||
significant = raw.lstrip('0') or '0'
|
||||
if len(significant) > len(str(maximum)):
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
value = int(significant)
|
||||
if not 1 <= value <= maximum:
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
return value
|
||||
|
||||
|
||||
def parse_cursor(raw):
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, str) or not raw:
|
||||
raise InvalidRequest('cursor должен быть непустой строкой')
|
||||
return raw
|
||||
|
||||
|
||||
def require_idempotency_key(value):
|
||||
"""Only the contract's bounded ASCII token is accepted."""
|
||||
if not isinstance(value, str) or re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', value) is None:
|
||||
raise InvalidRequest('Требуется корректный Idempotency-Key (1–128 ASCII символов)')
|
||||
return value
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Process attachment jobs and acknowledge only committed results."""
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import tempfile
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
import pika
|
||||
|
||||
from .attachments import AttachmentStorage, MAX_UPLOAD_BYTES, validate_image
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
||||
from .job_lifecycle import claim, complete
|
||||
from .publish_jobs import QUEUE, connect
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fitted(image, bound):
|
||||
width, height = image.size
|
||||
longest = max(width, height)
|
||||
if longest <= bound:
|
||||
return image.copy()
|
||||
return image.resize((max(1, width * bound // longest),
|
||||
max(1, height * bound // longest)), Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def save_variants(source, storage, prefix):
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as original:
|
||||
with ImageOps.exif_transpose(original) as corrected:
|
||||
with corrected.convert('RGBA') as rgba:
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
rgb.paste(rgba, mask=rgba.getchannel('A'))
|
||||
with rgb:
|
||||
specs = (
|
||||
('thumbnail', 'jpeg', 85, lambda: fitted(rgb, 256)),
|
||||
('crop', 'jpeg', 85, lambda: ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)),
|
||||
('optimized', 'webp', 82, lambda: fitted(rgb, 1280)),
|
||||
)
|
||||
results = []
|
||||
for name, fmt, quality, render in specs:
|
||||
with render() as image, BytesIO() as file:
|
||||
image.save(file, format=fmt.upper(), quality=quality)
|
||||
size = file.tell()
|
||||
digest = hashlib.sha256(file.getbuffer()).hexdigest()
|
||||
file.seek(0)
|
||||
key = f'{prefix}/{name}.{fmt}'
|
||||
mime = f'image/{fmt}'
|
||||
storage.put(key, file, size, mime)
|
||||
results.append((name, key, mime, size, digest, image.width, image.height))
|
||||
return results
|
||||
|
||||
|
||||
def materialize(attachment, storage, version, attempt):
|
||||
prefix = f"attachments/final/{attachment['id']}/v{version}/a{attempt}"
|
||||
key = prefix + '/original'
|
||||
stream = storage.download(attachment)
|
||||
try:
|
||||
with tempfile.TemporaryFile() as source:
|
||||
digest, size = hashlib.sha256(), 0
|
||||
for chunk in stream.chunks():
|
||||
size += len(chunk)
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLarge()
|
||||
digest.update(chunk)
|
||||
source.write(chunk)
|
||||
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
|
||||
raise UnsupportedMediaType('Контрольная сумма оригинала не совпадает')
|
||||
validate_image(source, attachment['content_type'])
|
||||
source.seek(0)
|
||||
variants = save_variants(source, storage, prefix)
|
||||
source.seek(0)
|
||||
storage.put(key, source, size, attachment['content_type'])
|
||||
finally:
|
||||
stream.close()
|
||||
return key, variants
|
||||
|
||||
|
||||
def process(database, storage, job):
|
||||
attachment = claim(database, job)
|
||||
if attachment is None:
|
||||
return
|
||||
try:
|
||||
result = materialize(attachment, storage, job['pipeline_version'], job['attempt'])
|
||||
except (PayloadTooLarge, UnsupportedMediaType, NotFound):
|
||||
complete(database, job, error_code='invalid_image')
|
||||
except (Unavailable, OSError):
|
||||
complete(database, job)
|
||||
else:
|
||||
complete(database, job, result=result)
|
||||
|
||||
|
||||
def parse_job(body):
|
||||
job = json.loads(body)
|
||||
if not isinstance(job, dict) or job.get('type') != QUEUE or job.get('schema_version') != 1:
|
||||
raise ValueError('Unknown job format')
|
||||
for field in ('job_id', 'attachment_id'):
|
||||
UUID(job[field])
|
||||
for field in ('pipeline_version', 'attempt'):
|
||||
if type(job[field]) is not int or job[field] < 1:
|
||||
raise ValueError('Invalid job number')
|
||||
return job
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
database = Database(Config.from_env())
|
||||
database.open()
|
||||
storage = AttachmentStorage()
|
||||
connection = None
|
||||
|
||||
def timeout(_signal, _frame):
|
||||
raise TimeoutError('Attachment processing exceeded 20 seconds')
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout)
|
||||
|
||||
def consume(channel, method, _properties, body):
|
||||
try:
|
||||
job = parse_job(body)
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
channel.basic_reject(method.delivery_tag, requeue=False)
|
||||
return
|
||||
signal.alarm(20)
|
||||
try:
|
||||
process(database, storage, job)
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
|
||||
channel.start_consuming()
|
||||
finally:
|
||||
if connection and connection.is_open:
|
||||
connection.close()
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Run cleanup scenarios against a disposable PostgreSQL database."""
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
from psycopg2.extensions import make_dsn
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.errors import Unavailable
|
||||
from messenger.store import Store
|
||||
|
||||
GRACE_SECONDS = 900
|
||||
|
||||
|
||||
def unavailable(_key):
|
||||
raise Unavailable('synthetic S3 failure')
|
||||
|
||||
|
||||
def expect_unavailable(operation):
|
||||
try:
|
||||
operation()
|
||||
except Unavailable:
|
||||
return
|
||||
raise AssertionError('Expected Unavailable')
|
||||
|
||||
|
||||
def exercise(database):
|
||||
store = Store(None, database, None)
|
||||
user, chat = str(uuid4()), str(uuid4())
|
||||
with database.cursor() as cur:
|
||||
cur.execute("INSERT INTO users VALUES (%s,'cleanup','unused','Cleanup')", (user,))
|
||||
cur.execute("INSERT INTO chats VALUES (%s,'Cleanup')", (chat,))
|
||||
|
||||
def reserve(expired=False):
|
||||
aid = str(uuid4())
|
||||
key = 'attachments/' + aid
|
||||
store.reserve_attachment(aid, chat, user, 'x.png', 'image/png', 1, 'a' * 64, key)
|
||||
if expired:
|
||||
with database.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (aid,))
|
||||
return aid, key
|
||||
|
||||
objects = set()
|
||||
fresh, fresh_key = reserve()
|
||||
objects.add(fresh_key)
|
||||
ready, ready_key = reserve(expired=True)
|
||||
store.finish_attachment(ready, objects.add)
|
||||
with database.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (ready,))
|
||||
|
||||
active, active_key = reserve(expired=True)
|
||||
def active_put(key):
|
||||
objects.add(key)
|
||||
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
store.finish_attachment(active, active_put)
|
||||
|
||||
failed, failed_key = reserve(expired=True)
|
||||
def put_then_fail(key):
|
||||
objects.add(key)
|
||||
# Inject a failure after the object exists and before the ready update.
|
||||
raise Unavailable('synthetic failure after PUT')
|
||||
expect_unavailable(lambda: store.finish_attachment(failed, put_then_fail))
|
||||
assert failed_key in objects
|
||||
expect_unavailable(lambda: store.clean_pending_attachment(GRACE_SECONDS, unavailable))
|
||||
with database.cursor() as cur:
|
||||
cur.execute('SELECT status FROM attachments WHERE id=%s', (failed,))
|
||||
assert cur.fetchone()['status'] == 'pending'
|
||||
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
assert objects == {fresh_key, ready_key, active_key}
|
||||
|
||||
reserve(expired=True) # Reservation whose PUT never created an object.
|
||||
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
with database.cursor() as cur:
|
||||
cur.execute('SELECT id::text AS id, status FROM attachments')
|
||||
assert {row['id']: row['status'] for row in cur.fetchall()} == {
|
||||
fresh: 'pending', ready: 'ready', active: 'ready',
|
||||
}
|
||||
print('PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved')
|
||||
print('PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely')
|
||||
|
||||
|
||||
def main():
|
||||
dsn = os.environ['LAB4_TEST_DATABASE_URL']
|
||||
schema = 'cleanup_' + uuid4().hex
|
||||
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
|
||||
config = Config.from_env({'DATABASE_URL': make_dsn(dsn, options='-csearch_path=' + schema)})
|
||||
database = Database(config)
|
||||
try:
|
||||
database.open()
|
||||
database.run_migrations()
|
||||
exercise(database)
|
||||
finally:
|
||||
database.close()
|
||||
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(schema)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Check upload errors through Nginx and exercise an actual S3 outage."""
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=45)
|
||||
alice, _, eve, chat = client.bootstrap()
|
||||
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments"
|
||||
good = path + '?filename=sample.png'
|
||||
item, _ = client.attachment(alice, chat)
|
||||
content = f"/api/v1/attachments/{item['id']}/content"
|
||||
|
||||
def upload(url, payload, expected, mime='image/png', actor=alice):
|
||||
return client.call('POST', url, expected, actor=actor, raw=payload,
|
||||
headers={'Content-Type': mime})
|
||||
|
||||
huge = BytesIO()
|
||||
Image.new('1', (8001, 5000)).save(huge, 'PNG')
|
||||
for payload, expected, mime in [
|
||||
(b'x' * (10 * 1024 * 1024 + 1), 413, 'image/png'),
|
||||
(huge.getvalue(), 413, 'image/png'),
|
||||
(data[:30], 415, 'image/png'),
|
||||
(data, 415, 'image/jpeg'),
|
||||
]:
|
||||
upload(good, payload, expected, mime)
|
||||
upload(path + '?filename=a%2Fb.png', data, 400)
|
||||
upload(good, data, 404, actor=eve)
|
||||
client.call('GET', content, 404, actor=eve)
|
||||
print('PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx')
|
||||
|
||||
def s3_command(action):
|
||||
subprocess.run(['docker', 'compose', action, 's3'], cwd=ROOT, check=True, timeout=30)
|
||||
|
||||
try:
|
||||
s3_command('stop')
|
||||
upload(good, data, 503)
|
||||
client.call('GET', content, 503, actor=alice)
|
||||
client.message(alice, chat, 'Текст при недоступном S3')
|
||||
finally:
|
||||
s3_command('start')
|
||||
|
||||
started = time.monotonic()
|
||||
for _ in range(40):
|
||||
try:
|
||||
downloaded = client.call('GET', content, actor=alice)
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(.25)
|
||||
else:
|
||||
raise Failure('S3 download did not recover after restart')
|
||||
check(downloaded['bytes'] == data, 'original changed after S3 restart')
|
||||
print(f'PASS: S3 outage returned 503, text remained available, original recovered in {time.monotonic()-started:.2f}s')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Stop the actual upload replica and check original bytes after failover and restart."""
|
||||
from hashlib import sha256
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=55)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
reply = client.call('POST', path, 201, actor=alice, raw=original,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
aid = reply['json']['id']
|
||||
upload_instance = reply['headers']['X-Instance-Id']
|
||||
digest = sha256(original).hexdigest()
|
||||
content = '/api/v1/attachments/' + aid + '/content'
|
||||
text = client.message(alice, chat, '', attachments=[aid])['json']
|
||||
|
||||
def verify():
|
||||
received = client.call('GET', content, actor=bob)
|
||||
check(received['bytes'] == original and sha256(received['bytes']).hexdigest() == digest,
|
||||
'original changed')
|
||||
history = client.call('GET', f"/api/v1/chats/{chat['id']}/messages",
|
||||
actor=bob, schema='MessagePage')['json']
|
||||
check(text in history['items'], 'message attachment link lost')
|
||||
return received['headers']['X-Instance-Id']
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
subprocess.run(['scripts/stop-one.sh'], cwd=ROOT, check=True, timeout=30,
|
||||
env={**os.environ, 'TARGET_INSTANCE_ID': upload_instance})
|
||||
instance = verify()
|
||||
check(instance != upload_instance, 'download was served by the stopped uploader')
|
||||
print(f'PASS: {upload_instance} -> {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
|
||||
finally:
|
||||
subprocess.run([sys.executable, 'scripts/manage.py', 'restore'], cwd=ROOT, check=True, timeout=100)
|
||||
|
||||
start = time.monotonic()
|
||||
subprocess.run(['scripts/restart.sh'], cwd=ROOT, check=True, timeout=100)
|
||||
instance = verify()
|
||||
print(f'PASS: after full restart via {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,141 @@
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
import logging
|
||||
import os
|
||||
import http.client
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
from botocore.response import StreamingBody
|
||||
from PIL import Image
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.environ['S3_BUCKET'] = 'test'
|
||||
sys.path.insert(0, str(root / 'src'))
|
||||
from messenger.api import Api
|
||||
from messenger.config import Config
|
||||
from messenger.security import hash_token
|
||||
from messenger.server import build_server
|
||||
from messenger.store import Store
|
||||
|
||||
chat, member, outsider = [str(uuid4()) for _ in range(3)]
|
||||
|
||||
class Metadata:
|
||||
rows = {}
|
||||
def get_session(self, token):
|
||||
return {hash_token('member'): member, hash_token('outsider'): outsider}.get(token)
|
||||
def user(self, uid):
|
||||
return {'id': uid}
|
||||
def chat(self, cid):
|
||||
return {'id': cid} if cid == chat else None
|
||||
def is_member(self, _chat, uid):
|
||||
return uid == member
|
||||
def reserve_attachment(self, aid, cid, uid, filename, mime, size, digest, key):
|
||||
now = datetime.now(timezone.utc)
|
||||
self.rows[aid] = dict(id=aid, chat_id=cid, owner_id=uid, filename=filename,
|
||||
content_type=mime, size_bytes=size, sha256=digest,
|
||||
object_key=key, status='pending', error_code=None,
|
||||
created_at=now, updated_at=now)
|
||||
def finish_attachment(self, aid, put_object):
|
||||
put_object(self.rows[aid]['object_key'])
|
||||
self.rows[aid]['status'] = 'ready'
|
||||
return self.rows[aid]
|
||||
def visible_attachment(self, aid, uid):
|
||||
row = self.rows.get(aid)
|
||||
if uid == member and row and row['status'] == 'ready':
|
||||
return row
|
||||
return None
|
||||
public_attachment = staticmethod(Store.public_attachment)
|
||||
|
||||
class S3:
|
||||
def __init__(self):
|
||||
self.objects = {}
|
||||
self.fail = False
|
||||
def put_object(self, **kw):
|
||||
if self.fail:
|
||||
from botocore.exceptions import EndpointConnectionError
|
||||
raise EndpointConnectionError(endpoint_url='http://s3.invalid')
|
||||
data = kw['Body'].read()
|
||||
assert len(data) == kw['ContentLength']
|
||||
self.objects[kw['Key']] = data
|
||||
def get_object(self, **kw):
|
||||
data = self.objects[kw['Key']]
|
||||
return {'ContentLength': len(data), 'Body': StreamingBody(BytesIO(data), len(data))}
|
||||
|
||||
config = Config.from_env({'APP_HOST':'127.0.0.1','APP_PORT':'8080'})
|
||||
config.port = 0
|
||||
api = Api(config, Metadata(), None)
|
||||
s3 = S3()
|
||||
api.attachments.client = s3
|
||||
server = build_server(config, logging.getLogger('lab4-check'), api, None)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = 'http://127.0.0.1:' + str(server.server_address[1])
|
||||
upload = f'/api/v1/chats/{chat}/attachments?filename=x.png'
|
||||
|
||||
def call(path, method='GET', data=None, mime='image/png', token='member'):
|
||||
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': mime}
|
||||
try:
|
||||
response = urlopen(Request(base + path, data=data, headers=headers, method=method), timeout=12)
|
||||
except HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
return response.code, response.headers, response.read()
|
||||
|
||||
try:
|
||||
for image_format, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]:
|
||||
buf = BytesIO()
|
||||
Image.new('RGB', (10, 10), 'green').save(buf, format=image_format)
|
||||
data = buf.getvalue()
|
||||
status, _, raw = call(upload, 'POST', data, mime)
|
||||
assert status == 201, (status, raw)
|
||||
item = json.loads(raw)
|
||||
assert item['status'] == 'ready' and item['sha256'] == sha256(data).hexdigest()
|
||||
assert item['size_bytes'] == len(data) and 'object_key' not in item
|
||||
path = '/api/v1/attachments/' + item['id']
|
||||
assert json.loads(call(path)[2]) == item
|
||||
status, headers, actual = call(path + '/content')
|
||||
assert (status, headers['Content-Type'], actual) == (200, mime, data)
|
||||
assert call(path, token='outsider')[0] == 404
|
||||
assert call(path + '/content', token='outsider')[0] == 404
|
||||
assert call(path + '/content?variant=thumbnail')[0] == 404
|
||||
png = (root / 'tests/fixtures/sample.png').read_bytes()
|
||||
assert call(upload, 'POST', png)[0] == 201
|
||||
assert call(upload, 'POST', png, 'image/jpeg')[0] == 415
|
||||
assert call(upload, 'POST', png[:30])[0] == 415
|
||||
assert call(upload.replace('x.png','a%2Fb.png'), 'POST', png)[0] == 400
|
||||
before = len(api.store.rows)
|
||||
for length, partial, expected in [(10 * 1024 * 1024 + 1, b'', 413), (64, b'abc', 400)]:
|
||||
conn = http.client.HTTPConnection(*server.server_address, timeout=5)
|
||||
try:
|
||||
conn.request('POST', upload, body=partial, headers={
|
||||
'Authorization': 'Bearer member', 'Content-Type': 'image/png',
|
||||
'Content-Length': str(length)})
|
||||
if partial:
|
||||
conn.sock.shutdown(socket.SHUT_WR)
|
||||
with conn.getresponse() as response:
|
||||
assert response.status == expected, response.status
|
||||
response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
assert len(api.store.rows) == before
|
||||
buf = BytesIO()
|
||||
Image.new('1', (8001, 5000)).save(buf, 'PNG')
|
||||
assert call(upload, 'POST', buf.getvalue())[0] == 413
|
||||
s3.fail = True
|
||||
assert call(upload, 'POST', png)[0] == 503
|
||||
assert list(api.store.rows.values())[-1]['status'] == 'pending'
|
||||
print('PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant')
|
||||
print('PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping')
|
||||
print('NOTE: metadata and S3 are test substitutes; real Compose integration not exercised')
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Exercise broker persistence and kill a worker after S3 writes, before SQL commit/ACK."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def cmd(*args, capture=False):
|
||||
return subprocess.run(args, cwd=ROOT, text=True, check=True, timeout=100,
|
||||
stdout=subprocess.PIPE if capture else None).stdout
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
return cmd('docker', 'compose', *args, capture=capture)
|
||||
|
||||
|
||||
def wait_for(probe, timeout, label):
|
||||
until = time.monotonic() + timeout
|
||||
while time.monotonic() < until:
|
||||
if probe():
|
||||
return
|
||||
time.sleep(.25)
|
||||
raise RuntimeError(label)
|
||||
|
||||
|
||||
def confirmed(aid):
|
||||
count = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-At', '-c', f"SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j "
|
||||
f"ON j.id=o.job_id WHERE j.attachment_id='{aid}' AND o.published_at IS NOT NULL",
|
||||
capture=True)
|
||||
return int(count.strip()) > 0
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
marker = 'AFTER_S3_BEFORE_COMMIT'
|
||||
fault_worker = 'lab5-fault-' + uuid4().hex[:10]
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
client.message(alice, chat, '', attachments=[attachment['id']])
|
||||
path = '/api/v1/attachments/' + attachment['id']
|
||||
client.call('GET', path + '/content', 409, actor=bob)
|
||||
wait_for(lambda: confirmed(attachment['id']), 30, 'no publisher confirm')
|
||||
compose('restart', 'rabbitmq', 'api-a', 'api-b')
|
||||
compose('up', '-d', '--no-build', '--no-deps', '--wait', 'api-a', 'api-b')
|
||||
queued = client.call('GET', path, actor=bob, schema='Attachment')['json']
|
||||
check(queued['status'] == 'queued', 'job lost during restart')
|
||||
compose('start', 'worker')
|
||||
client.wait_ready(bob, attachment)
|
||||
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original lost')
|
||||
print('PASS: worker stopped; after API and RabbitMQ restart the same attachment became ready', flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
path = '/api/v1/attachments/' + attachment['id']
|
||||
code = """
|
||||
import sys,time
|
||||
from messenger import worker
|
||||
commit = worker.complete
|
||||
def pause(db, job, result=None, error_code=None):
|
||||
if job['attachment_id'] == sys.argv[1] and result is not None:
|
||||
print('AFTER_S3_BEFORE_COMMIT', flush=True)
|
||||
time.sleep(120)
|
||||
return commit(db, job, result, error_code)
|
||||
worker.complete = pause
|
||||
worker.main()
|
||||
"""
|
||||
compose('run', '-d', '--no-deps', '--name', fault_worker, 'worker',
|
||||
'python', '-c', code, attachment['id'])
|
||||
wait_for(lambda: marker in cmd('docker', 'logs', fault_worker, capture=True),
|
||||
40, 'worker did not reach fault point')
|
||||
cmd('docker', 'kill', fault_worker)
|
||||
compose('start', 'worker')
|
||||
start = time.monotonic()
|
||||
def is_ready():
|
||||
state = client.call('GET', path, actor=bob, schema='Attachment')['json']
|
||||
check(state['status'] != 'failed', 'processing failed after crash')
|
||||
return state if state['status'] == 'ready' else None
|
||||
wait_for(is_ready, 90, 'expired processing lease did not recover')
|
||||
ready = is_ready()
|
||||
check(len(ready['variants']) == 3, 'expected exactly one set of variants')
|
||||
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
|
||||
print(f'PASS: worker killed after S3 write before COMMIT/ACK; recovered in {time.monotonic()-start:.2f}s')
|
||||
finally:
|
||||
subprocess.run(['docker', 'rm', '-f', fault_worker], cwd=ROOT, capture_output=True, timeout=15)
|
||||
compose('start', 'worker')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Exercise job fencing, failed replay and cleanup on the stage-5 stand."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
code = """
|
||||
import sys
|
||||
from messenger.attachments import AttachmentStorage
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.errors import Conflict
|
||||
import messenger.job_lifecycle as lifecycle
|
||||
from messenger.worker import materialize, process
|
||||
db = Database(Config.from_env())
|
||||
db.open()
|
||||
s3 = AttachmentStorage()
|
||||
aid = sys.argv[1]
|
||||
def delivery():
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT id AS job_id,attachment_id,pipeline_version,attempt FROM attachment_jobs WHERE attachment_id=%s', (aid,))
|
||||
return cur.fetchone()
|
||||
try:
|
||||
first = delivery()
|
||||
row = lifecycle.claim(db, first)
|
||||
assert row['status'] == 'processing' and lifecycle.claim(db, first) is None
|
||||
stale_result = materialize(row, s3, first['pipeline_version'], first['attempt'])
|
||||
with db.cursor() as cur:
|
||||
cur.execute("UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=%s", (first['job_id'],))
|
||||
lifecycle.recover_expired(db)
|
||||
second = delivery()
|
||||
assert second['attempt'] == first['attempt'] + 1
|
||||
assert not lifecycle.complete(db, first, result=stale_result)
|
||||
assert lifecycle.claim(db, second)
|
||||
lifecycle.MAX_ATTEMPTS = 2
|
||||
assert lifecycle.complete(db, second)
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT status,error_code FROM attachments WHERE id=%s', (aid,))
|
||||
assert cur.fetchone() == {'status':'failed','error_code':'retry_exhausted'}
|
||||
assert lifecycle.replay(db, first['job_id'], 'stage5-test') == 3
|
||||
third = delivery()
|
||||
process(db, s3, third)
|
||||
process(db, s3, third)
|
||||
try:
|
||||
lifecycle.replay(db, first['job_id'], 'stage5-test')
|
||||
except Conflict:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('ready job was replayed')
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT count(*) AS n FROM attachment_variants WHERE attachment_id=%s', (aid,))
|
||||
assert cur.fetchone()['n'] == 3
|
||||
cur.execute('SELECT count(*) AS n FROM attachment_replays WHERE job_id=%s', (first['job_id'],))
|
||||
assert cur.fetchone()['n'] == 1
|
||||
cur.execute("UPDATE attachment_outbox SET available_at=now()-interval '16 minutes' WHERE job_id=%s AND attempt=1",
|
||||
(first['job_id'],))
|
||||
# Check that the cleaner can remove an obsolete attempt even after replay succeeded.
|
||||
removed = []
|
||||
while lifecycle.clean_abandoned_outputs(db, lambda key: (s3.delete(key), removed.append(key))):
|
||||
pass
|
||||
assert stale_result[0] in removed, removed
|
||||
while lifecycle.clean_staging(db, s3.delete):
|
||||
pass
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT staging_cleaned_at,lease_until FROM attachment_jobs WHERE id=%s', (first['job_id'],))
|
||||
row = cur.fetchone()
|
||||
assert row['staging_cleaned_at'] is not None and row['lease_until'] is None
|
||||
print('PASS: expired claim fenced, retry exhausted, audited replay, stale objects and staging cleaned')
|
||||
finally:
|
||||
db.close()
|
||||
"""
|
||||
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
|
||||
code, attachment['id']], cwd=ROOT, check=True, timeout=45)
|
||||
path = f"/api/v1/attachments/{attachment['id']}/content"
|
||||
check(client.call('GET', path, actor=bob)['bytes'] == original, 'cleanup damaged original')
|
||||
print('PASS: ready original survives cleanup and replay requests')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Exercise attachment links and message replay over the running lab4 API."""
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Message attachment links and replay')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
first, _ = client.attachment(alice, chat)
|
||||
second, _ = client.attachment(bob, chat)
|
||||
ids = [second['id'], first['id']]
|
||||
key = uuid.uuid4().hex
|
||||
sent = client.message(bob, chat, '', key=key, attachments=ids)['json']
|
||||
replay = client.message(bob, chat, '', key=key, attachments=ids)['json']
|
||||
check(sent == replay and sent['attachment_ids'] == ids, 'ordered attachment replay mismatch')
|
||||
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
client.call('POST', path, 409, actor=bob, body={'text': '', 'attachment_ids': ids[::-1]},
|
||||
headers={'Idempotency-Key': key})
|
||||
for invalid in [[ids[0]] * 2, [str(uuid.uuid4()) for _ in range(11)], ['bad UUID'], None]:
|
||||
client.call('POST', path, 400, actor=bob, body={'text': '', 'attachment_ids': invalid},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
client.call('POST', path, 400, actor=bob, body={'text': ''},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
client.call('POST', path, 404, actor=bob,
|
||||
body={'text': '', 'attachment_ids': [str(uuid.uuid4())]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
|
||||
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]},
|
||||
schema='Chat')['json']
|
||||
foreign, _ = client.attachment(alice, other)
|
||||
client.call('POST', path, 404, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [foreign['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
|
||||
plain_key = uuid.uuid4().hex
|
||||
plain = client.message(alice, chat, 'Обычный текст', key=plain_key)['json']
|
||||
check(client.message(alice, chat, 'Обычный текст', key=plain_key, attachments=[])['json'] == plain,
|
||||
'omitted and empty attachment_ids must replay identically')
|
||||
page = client.call('GET', path, actor=alice, schema='MessagePage')['json']
|
||||
check([item for item in page['items'] if item['id'] == sent['id']] == [sent],
|
||||
'ordered attachment IDs missing or duplicated in history')
|
||||
print('PASS: ordered attachments, replay, invalid input, cross-chat rejection and history')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
# Run against a lab5 stage-3 stand without a worker.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
base_url=${BASE_URL:-http://localhost:18082}
|
||||
sql() {
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -At -v ON_ERROR_STOP=1 -c "$1"
|
||||
}
|
||||
docker compose stop outbox-publisher
|
||||
trap 'docker compose start outbox-publisher >/dev/null' EXIT
|
||||
upload=$(PYTHONDONTWRITEBYTECODE=1 python3 student-tests/queued_upload.py --base-url "$base_url")
|
||||
printf '%s\n' "$upload"
|
||||
aid=$(printf '%s\n' "$upload" | awk '/Attachment ID:/ {print $3}')
|
||||
query="SELECT count(*) FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='$aid' AND o.published_at IS"
|
||||
test "$(sql "$query NULL")" = 1
|
||||
echo 'PASS: upload committed while publisher was stopped'
|
||||
docker compose start outbox-publisher
|
||||
trap - EXIT
|
||||
for i in $(seq 1 40); do
|
||||
if test "$(sql "$query NOT NULL")" = 1; then
|
||||
echo 'PASS: same attachment job confirmed after publisher startup'
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo 'FAIL: outbox row not confirmed' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Verify durable handoff before a publisher or worker is enabled."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
path = '/api/v1/attachments/' + attachment['id']
|
||||
check(attachment['status'] == 'queued', 'expected queued with worker disabled')
|
||||
metadata = client.call('GET', path, actor=bob, schema='Attachment')['json']
|
||||
check(metadata == attachment, 'queued metadata differs')
|
||||
client.call('GET', path, 404, actor=eve)
|
||||
client.call('GET', path + '/content', 409, actor=bob)
|
||||
sent = client.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
check(sent['attachment_ids'] == [attachment['id']], 'queued attachment not linked')
|
||||
print('PASS: 202 queued, metadata ACL, 409 before ready, and queued attachment linked')
|
||||
print('Attachment ID:', attachment['id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Run via docker compose exec -T api-a python - with this file on stdin."""
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from PIL import Image
|
||||
|
||||
from messenger.attachments import AttachmentStorage
|
||||
from messenger.clean_attachments import GRACE_SECONDS, clean_batch
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.errors import Unavailable
|
||||
from messenger.store import Store
|
||||
|
||||
|
||||
def main():
|
||||
config = Config.from_env()
|
||||
db = Database(config)
|
||||
db.open()
|
||||
store = Store(config, db, None)
|
||||
s3 = AttachmentStorage()
|
||||
user, chat, aid = [str(uuid4()) for _ in range(3)]
|
||||
key = 'attachments/' + aid
|
||||
payload = BytesIO()
|
||||
Image.new('RGB', (2, 2), 'green').save(payload, format='PNG')
|
||||
data = payload.getvalue()
|
||||
|
||||
try:
|
||||
with db.cursor() as cur:
|
||||
cur.execute('INSERT INTO users (id,username,password_hash,display_name) VALUES (%s,%s,%s,%s)',
|
||||
(user, 'gc_' + user[:8], 'unused', 'Cleanup check'))
|
||||
cur.execute("INSERT INTO chats (id,title) VALUES (%s,'Cleanup check')", (chat,))
|
||||
store.reserve_attachment(aid, chat, user, 'probe.png', 'image/png',
|
||||
len(data), sha256(data).hexdigest(), key)
|
||||
|
||||
def put_then_fail(object_key):
|
||||
s3.put(object_key, BytesIO(data), len(data), 'image/png')
|
||||
raise Unavailable('synthetic failure after successful PUT')
|
||||
|
||||
try:
|
||||
store.finish_attachment(aid, put_then_fail)
|
||||
except Unavailable as exc:
|
||||
assert str(exc) == 'synthetic failure after successful PUT'
|
||||
else:
|
||||
raise AssertionError('fault did not run')
|
||||
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
|
||||
with obj['Body'] as body:
|
||||
assert body.read() == data
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT status FROM attachments WHERE id=%s', (aid,))
|
||||
assert cur.fetchone()['status'] == 'pending'
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-make_interval(secs => %s) WHERE id=%s",
|
||||
(GRACE_SECONDS + 1, aid))
|
||||
clean_batch(store, s3)
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT 1 FROM attachments WHERE id=%s', (aid,))
|
||||
assert cur.fetchone() is None
|
||||
try:
|
||||
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
assert exc.response['ResponseMetadata']['HTTPStatusCode'] == 404, exc
|
||||
else:
|
||||
obj['Body'].close()
|
||||
raise AssertionError('orphan object still exists')
|
||||
print('PASS: actual MinIO object existed after failed commit and was removed by cleanup')
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Measure sampled cgroup memory during ten simultaneous HTTP uploads."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
from threading import Barrier
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
COUNT = 10
|
||||
SAMPLER = """
|
||||
import select, sys, time
|
||||
from pathlib import Path
|
||||
usage = Path('/sys/fs/cgroup/memory.current')
|
||||
limit = Path('/sys/fs/cgroup/memory.max')
|
||||
cpu = Path('/sys/fs/cgroup/cpu.stat')
|
||||
print(limit.read_text().strip(), flush=True)
|
||||
while True:
|
||||
stats = dict(line.split() for line in cpu.read_text().splitlines())
|
||||
print(usage.read_text().strip(), stats['usage_usec'], flush=True)
|
||||
if select.select([sys.stdin], [], [], .05)[0]:
|
||||
break
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=60)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
width = 1700
|
||||
source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3))
|
||||
buffer = BytesIO()
|
||||
source.save(buffer, format='PNG', compress_level=0)
|
||||
payload = buffer.getvalue()
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit')
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png"
|
||||
barrier = Barrier(COUNT + 1)
|
||||
samplers = {}
|
||||
|
||||
def upload():
|
||||
local = Client(args.base_url, args.ca_file, timeout=60)
|
||||
barrier.wait(timeout=30)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
reply = local.call('POST', path, 201, actor=alice, raw=payload,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
check(reply['json']['sha256'] == digest, 'upload hash mismatch')
|
||||
return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'],
|
||||
'seconds': time.monotonic() - start}
|
||||
except Failure as exc:
|
||||
return {'error': str(exc), 'seconds': time.monotonic() - start}
|
||||
|
||||
try:
|
||||
for service in ('api-a', 'api-b'):
|
||||
proc = subprocess.Popen(
|
||||
['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
|
||||
samplers[service] = proc
|
||||
with ThreadPoolExecutor(max_workers=COUNT) as pool:
|
||||
futures = [pool.submit(upload) for _ in range(COUNT)]
|
||||
limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()}
|
||||
initial = {service: tuple(map(int, proc.stdout.readline().split()))
|
||||
for service, proc in samplers.items()}
|
||||
start = time.monotonic()
|
||||
barrier.wait(timeout=30)
|
||||
requests = [future.result() for future in futures]
|
||||
elapsed = time.monotonic() - start
|
||||
finally:
|
||||
for proc in samplers.values():
|
||||
if proc.poll() is None:
|
||||
proc.stdin.close()
|
||||
resources = {}
|
||||
for service, proc in samplers.items():
|
||||
samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout]
|
||||
check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed')
|
||||
resources[service] = {
|
||||
'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples),
|
||||
'limit_bytes': limits[service], 'samples': len(samples),
|
||||
'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6,
|
||||
}
|
||||
successes = [row for row in requests if 'id' in row]
|
||||
durations = sorted(row['seconds'] for row in successes)
|
||||
percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1]
|
||||
for p in (50, 95, 99)} if durations else {}
|
||||
print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest,
|
||||
'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed,
|
||||
'latency_seconds': percentiles, 'requests': requests,
|
||||
'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2))
|
||||
check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Verify worker effects through HTTP and MinIO with a direct job handoff."""
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
from image_probe import jpeg_size
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
code = """
|
||||
from messenger.attachments import AttachmentStorage
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.worker import process
|
||||
import sys
|
||||
db = Database(Config.from_env())
|
||||
db.open()
|
||||
try:
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT id AS job_id, attachment_id, pipeline_version, attempt '
|
||||
'FROM attachment_jobs WHERE attachment_id=%s', (sys.argv[1],))
|
||||
job = cur.fetchone()
|
||||
storage = AttachmentStorage()
|
||||
process(db, storage, job)
|
||||
process(db, storage, job)
|
||||
finally:
|
||||
db.close()
|
||||
"""
|
||||
subprocess.run(['docker', 'compose', 'exec', '-T', 'api-a', 'python', '-c',
|
||||
code, attachment['id']], cwd=ROOT, check=True, timeout=30)
|
||||
path = '/api/v1/attachments/' + attachment['id']
|
||||
ready = client.call('GET', path, actor=bob, schema='Attachment')['json']
|
||||
check(ready['status'] == 'ready' and len(ready['variants']) == 3, 'expected three variants')
|
||||
check(client.call('GET', path + '/content', actor=bob)['bytes'] == original, 'original changed')
|
||||
expected = {'thumbnail': (256, 160, 'image/jpeg'), 'crop': (128, 128, 'image/jpeg'),
|
||||
'optimized': (320, 200, 'image/webp')}
|
||||
for item in ready['variants']:
|
||||
width, height, mime = expected[item['name']]
|
||||
reply = client.call('GET', path + '/content?variant=' + item['name'], actor=bob)
|
||||
data = reply['bytes']
|
||||
check((item['width'], item['height'], item['content_type']) == (width, height, mime),
|
||||
'incorrect variant metadata')
|
||||
check(reply['headers'].get_content_type() == mime and len(data) == item['size_bytes']
|
||||
and hashlib.sha256(data).hexdigest() == item['sha256'], 'incorrect variant payload')
|
||||
if mime == 'image/jpeg':
|
||||
check(jpeg_size(data) == (width, height), 'invalid JPEG dimensions')
|
||||
else:
|
||||
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature')
|
||||
print('PASS: original unchanged, thumbnail 256x160, crop 128x128, WebP 320x200')
|
||||
print('PASS: processing the same committed job twice keeps one set of variants')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user