diff --git a/.dockerignore b/.dockerignore index bc16e556..f6879b61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store + +DEFENSE.md diff --git a/.env.example b/.env.example index 119384b5..ef5c3737 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,11 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Generate the local .env and isolated secret files with scripts/init-local.py. +COMPOSE_PROJECT_NAME=mtusi2026lab5 +PUBLIC_PORT=8080 +BROKER_CONSOLE_PORT=15685 +S3_CONSOLE_PORT=9005 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +PIPELINE_FAULTS=0 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..0130cce8 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,44 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + integration: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + PIPELINE_FAULTS: '1' + JOB_TIMEOUT_SECONDS: '20' + COMPOSE_PROJECT_NAME: mtusi2026lab5-ci + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Build and wait for readiness + run: | + python scripts/init-local.py + ./scripts/compose.sh up -d --build --wait --wait-timeout 150 + curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready + - name: Mandatory smoke and recovery + run: | + set -o pipefail + make test 2>&1 | tee evidence/ci-smoke.txt + make test-persistence ACTION_SCRIPT=scripts/restart.sh 2>&1 | tee evidence/ci-persistence.txt + make test-failover ACTION_SCRIPT=scripts/stop-one.sh 2>&1 | tee evidence/ci-failover.txt + ./scripts/compose.sh start api + curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready + - name: Track 2 experiments + run: | + set -o pipefail + python student-tests/pipeline.py --disposable 2>&1 | tee evidence/ci-pipeline.jsonl + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab5-evidence + path: | + evidence/ci-* + evidence/restart-*.txt + - name: Stop test project + if: always() + run: ./scripts/compose.sh down diff --git a/.gitignore b/.gitignore index b5a71896..363239b1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +/DEFENSE.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2c74f9a1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11.16-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV APP_HOST=0.0.0.0 +ENV APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . + +RUN python -m pip install --no-cache-dir -r requirements.txt + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser + +COPY --chown=appuser:appuser src/ ./src/ + +COPY --chown=appuser:appuser db/ ./db/ + +COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json + +COPY infra/app-start.sh /usr/local/bin/app-start +ENTRYPOINT ["/bin/sh", "/usr/local/bin/app-start"] + +EXPOSE 8080 + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()" + +CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2", "--limit-concurrency", "64"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..abb8d898 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,210 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа — Дугина Александра, БПИ2301. +- Целевая оценка — 5, трек 2 «Доказанная идемпотентность pipeline». Проверки выполнены на реальных контейнерах Docker Desktop. +- Использованные источники и помощь — я использовала контракты и тесты курса, собственную реализацию мессенджера из ЛР №4, FastAPI, psycopg, Pillow, boto3 и Pika. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Мне понадобятся Docker Engine или Docker Desktop с доступным daemon, Compose v2, +Python 3.11+ на хосте для проверок и локальные свободные порты 8080, 15685, 9005. + +```sh +python3 scripts/init-local.py +./scripts/compose.sh config --quiet +./scripts/compose.sh up -d --build --wait --wait-timeout 150 +curl -fsS http://localhost:8080/health/ready +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +./scripts/compose.sh start api +./scripts/compose.sh up -d --wait --wait-timeout 150 +``` + +`init-local.py` генерирует случайные пароли в игнорируемом каталоге `secrets/` +и создаёт локальный `.env`. При повторном запуске я сохраняю существующие значения. +Порты можно заменить в `.env`. При изменении `PUBLIC_PORT` я передаю +`BASE_URL=http://localhost:НОВЫЙ_ПОРТ` в Make и `--base-url` в тестовые скрипты. + +Swagger открывается на . Веб-консоли RabbitMQ и MinIO +доступны на и . Сетевые порты +привязаны к loopback. Остановить стенд с сохранением данных можно через +`./scripts/compose.sh down`. Для намеренного удаления **своих** учебных данных +существует `./scripts/compose.sh down --volumes`. Перед этим я проверяю имя +Compose-проекта и убеждаюсь, что его volumes больше не нужны. + +| Роль | Compose и версия | Готовность и данные | Доступ | +| --- | --- | --- | --- | +| Балансировщик | `lb`, HAProxy 3.0.9-alpine | HTTP-проверки двух реплик, локальных данных нет | 8080 через loopback | +| API и миграции | `api` ×2, `migrate`, Python 3.11.16-slim-bookworm | `/health/ready` проверяет БД и доступ к bucket, миграции одноразовые | Внутренняя сеть, secrets с DSN и доступом к S3 | +| База | `postgres`, PostgreSQL 16.8-bookworm | TCP `pg_isready`, `postgres_data` | Внутренняя сеть, файл пароля | +| Кэш профилей | `redis`, Redis 7.4.2-alpine | `redis-cli ping`, AOF в `redis_data` | Внутренняя сеть | +| Объекты | `s3`, MinIO 2025.7.23 с digest из Compose | MinIO live, `s3_data` | Приватный bucket, ограниченная policy приложения, локальная консоль 9005 | +| Брокер | `broker`, RabbitMQ 4.1.4-management | `rabbitmq-diagnostics check_running`, `broker_data` | Внутренний AMQP, консоль 15685, секрет аутентификации | +| Фоновые процессы | `publisher`, `worker` ×2, `recovery`, образ приложения | Проверка продвижения рабочего цикла, reconnect, restart policy | Нет публичных портов | + +Compose дожидается PostgreSQL и применяет SQL-миграции под advisory lock. Список +применённых файлов хранится в `schema_migrations`. MinIO bootstrap создаёт +приватный bucket, сервисную учётную запись и policy. Publisher и worker объявляют +одну и ту же durable quorum queue при подключении. `.env.example` показывает +переменные портов, TTL и pipeline, фактически используемые в Compose. Секреты +подключаются файлами, а entrypoint приложения копирует нужные файлы перед +запуском процессов от непривилегированного пользователя. + +Все три фоновые роли обновляют метку прогресса в собственном tmpfs. +Worker обновляет её после обработки доставки или успешного опроса очереди, +publisher — после итерации outbox через канал с publisher confirms, recovery — +после итерации проверки lease и плановой очистки. Метка появляется после +первой успешной итерации, сбрасывается, если ошибка зависимости прервала цикл, и при завершении +процесса. Она истекает через 60 секунд без прогресса. Compose опрашивает +проверку раз в 3 секунды и выставляет unhealthy после трёх неуспехов. +Миграции и S3 bootstrap остаются одноразовыми: Compose ждёт их успешного +завершения. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Моя цель — отвечать на upload после надёжного приёма файла и обрабатывать его +в отдельных процессах. PostgreSQL проверяет сессию при каждом +запросе. Redis ускоряет получение профиля. При конкурентных вставках keyset +пагинация видит записи после курсора с ключом `(created_at, id)`. Между +страницами нет общей snapshot-транзакции. + +1. API принимает PNG/JPEG потоком с лимитом 10 MiB и проверяет настоящий формат, + декодируемость и лимит 40 Мп в отдельном процессе. Временный файл нужен для + проверки и записи. Под advisory lock API записывает исходник в приватный S3 + prefix `staging/`. В одной PostgreSQL-транзакции сохраняются attachment, + job и outbox. После COMMIT API возвращает снимок `202 queued`. В состоянии + queued файл уже можно привязать к сообщению. Download вернёт 409. +2. Publisher читает только зафиксированные outbox-строки через + `FOR UPDATE SKIP LOCKED`, отправляет carrier по учебной JSON-схеме и удаляет + строку после confirm. RabbitMQ сохраняет сообщения в durable quorum queue. + При потере связи или аварии после confirm до удаления отправка может + повториться. В AMQP передаются идентификаторы и номера, байты остаются в S3. +3. Worker держит одну неподтверждённую доставку, перед работой атомарно + захватывает job, увеличивает fencing token и выставляет lease. По умолчанию + два worker-процесса, lease 60 секунд. Worker берёт серверный ключ из БД, + сверяет байты и SHA-256, применяет EXIF orientation, композитит альфа-канал + на белый. JPEG thumbnail вписывается в 256×256 без увеличения и с + округлением вниз. JPEG crop центрирован и имеет размер 128×128. + WebP вписывается в 1280×1280 без увеличения, method 4. Качество JPEG/WebP + для v1 равно 85/82, для v2 — 90/88. Производные сохраняются без исходных + EXIF/ICC. Исходник копируется побайтно. Linux-процесс декодирования worker + ограничен 512 MiB виртуальной памяти, 15 секундами CPU и 20 секундами по + часам родителя. Лимиты контейнера — 1 CPU и 768 MiB на worker. +4. Ключ результата включает attachment, pipeline version, вариант и SHA-256. + Повторная запись того же ключа содержит те же байты. В одной транзакции + worker публикует manifest и переключает активную версию, если lease ещё + действует, fencing token совпадает и версия остаётся желаемой. Затем + отправляет ack. Чтение через API использует только активный manifest. + При переходе с готовой v1 на v2 прежняя версия доступна до атомарной + публикации v2. +5. Временная ошибка или истёкший lease увеличивает номер попытки и возвращает + job в outbox с exponential backoff, jitter 0,75–1,25 от базовой задержки и + пределом базовой задержки 30 секунд. После пяти попыток по умолчанию либо + постоянной ошибки worker/recovery сохраняет failed и запись в PostgreSQL DLQ. + Я использую этот реестр как долговечную очередь ручного replay. Replay + разрешён только для failed job текущей версии и сохраняет UUID и историю + предыдущих попыток. +6. Recovery удаляет staging готовых вложений после commit. Непривязанные и + failed-исходники остаются по умолчанию сутки. Cleanup проверяет метаданные + и берёт тот же lock, что uploader. Во время reprocessing источником + служит опубликованный оригинал. Внутренний ключ бакета в API отсутствует. + +```mermaid +flowchart LR + C[Клиент] --> H[HAProxy] + H --> A[Две API-реплики] + A --> S[(MinIO)] + A --> P[(PostgreSQL)] + P --> O[Publisher] + O --> B[(RabbitMQ)] + B --> W[Worker ×2] + W --> S + W --> P + R[Recovery] --> P + R --> S +``` ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Docker VM использовала Linux aarch64, 8 CPU и 7,65 GiB RAM. +Хост Darwin arm64, 8 CPU, 16 GiB RAM, тестовый клиент Python 3.13.3. +Строки CPU и памяти в `container-resources.txt` — один снимок после запуска. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Проверка и наблюдение | Доказательство | +| --- | --- | --- | +| Контракты и примеры | `make check` и полная OpenAPI/JSON Schema валидация прошли | [offline](evidence/local-checks.txt) | +| Воспроизводимый запуск | `up -d --build --wait --wait-timeout 150` завершился с кодом 0. Две API-реплики, балансировщик, хранилища, брокер и все фоновые процессы healthy | [запуск](evidence/compose-up.txt), [после перезапуска](evidence/post-restart-health.txt) | +| Уровень 3, публичный API | `make test` прошёл, включая права, SHA-256 оригинала и размер thumbnail | [smoke](evidence/smoke.txt) | +| Уровень 3, сохранность | `make test-persistence` подтвердил сохранность данных и прежней сессии. Изменились timestamps запуска API, HAProxy, хранилищ, RabbitMQ и фоновых процессов | [тест](evidence/persistence.txt), [до](evidence/restart-before.txt), [после](evidence/restart-after.txt) | +| Уровень 3, failover | `make test-failover` прошёл. Поочерёдно остановлены `api-2` и `api-1`. Первая успешная запись через 2,280 и 1,925 с соответственно. В каждом прогоне подтверждено 120 записей | [тест](evidence/failover.txt), [состояние контейнеров](evidence/failover-containers.txt), [повтор](evidence/failover-repeat.txt) | +| Уровни 3–4, обработка изображений | Проверки с Pillow 11.2.1 внутри образа прошли, включая геометрию, детерминированность, EXIF, прозрачность и лимит пикселей | [render](evidence/container-render.txt) | +| Уровни 3–5, отказоустойчивость, retry, DLQ и идемпотентность | Сценарии и результаты приведены ниже | [JSONL](evidence/track2.jsonl) | + +Отдельный `make load` выполнил 7583 успешных POST сообщений за 30.037 с, concurrency 4. Ошибок 0, 252.458 успешных запросов/с. Задержки p50/p95/p99 успешных запросов составили 12.714/37.804/73.943 мс. Это один closed-loop замер HTTP-записей сообщений. [Метрики нагрузки](evidence/load.txt) · [снимок CPU и памяти](evidence/container-resources.txt) ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза.** В заданной модели отказов принятый upload дойдёт до готового +результата после восстановления компонентов. Повторные доставки не создадут +вторую публикацию того же manifest. Новая версия сохранится, если позднее +возобновить старый worker или доставить старую job. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Эксперимент использует отдельный Compose-проект, две реплики worker, +lease 20 секунд, PNG 320×200 и обычный HTTP API. Для каждого сценария +создаётся новый объект. Время в JSONL измеряется монотонными часами от начала +сценария до последнего assertion и включает команды Compose, рестарты и +ожидание lease. Отдельный сценарий отправляет шесть одновременных POST +сообщений с общим ключом через HAProxy. + +```sh +export COMPOSE_PROJECT_NAME=mtusi2026lab5-track2 +export PUBLIC_PORT=8085 BROKER_CONSOLE_PORT=15686 S3_CONSOLE_PORT=9006 +export PIPELINE_FAULTS=1 WORKER_CONCURRENCY=2 JOB_TIMEOUT_SECONDS=20 +./scripts/compose.sh up -d --build --wait --wait-timeout 150 +export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 +set -o pipefail +python3 student-tests/pipeline.py --disposable --base-url http://localhost:8085 | tee evidence/track2.jsonl +``` + +Исполнитель читает PostgreSQL и MinIO и проверяет скачанные байты по HTTP. +Для аварий он ждёт записи о достижении конкретной точки, сверяет журнал +контейнеров и проверяет число попыток, публикаций, ключей S3, состояние и хеши. +Сценарий брокера останавливает реальный контейнер. `crash` завершает процесс +через `os._exit(86)`. `pause` удерживает старого исполнителя до явного +освобождения в эксперименте. + +| Воздействие | Наблюдение | Время сценария | +| --- | --- | --- | +| Конкурентные записи и проверка входа | Шесть запросов с одним ключом через две API-реплики дали один ID сообщения. Отклонены чужая загрузка, неверный MIME, повреждённое и слишком большое тело | 0,041 / 0,379 с | +| Остановка RabbitMQ | Принятый upload остался в outbox. Publisher и оба worker стали unhealthy. После старта RabbitMQ они вернулись в healthy, а вложение стало ready с первой попытки | 19,129 с | +| Авария издателя до publish и после confirm | Задание дошло до ready без пересоздания. Повтор после confirm проигнорирован. В каждом сценарии зафиксирована одна публикация manifest | 12,328 / 12,365 с | +| Авария worker до записи S3 | В точке сбоя итоговых объектов не было. После истечения lease вторая попытка опубликовала manifest один раз | 28,879 с | +| Авария после S3 до PostgreSQL | В S3 уже лежали четыре объекта, а job была processing. После второй попытки сохранился один набор из четырёх ключей и одна публикация manifest | 28,390 с | +| Авария после COMMIT результата до ack | Дубль доставки проигнорирован, номер попытки остался 1, manifest опубликован один раз | 7,570 с | +| Дубль при активном lease и возобновление старого worker | При активном lease зафиксирован один захват. После истечения lease вторая попытка получила fencing token 2. Поздняя фиксация старой попытки отклонена | 33,826 с | +| v2 готова раньше приостановленной или запоздалой v1 | API продолжил выдавать v2. Устаревшая v1 не опубликовала manifest. Запоздалая v1 не получила lease и не создала объектов S3 | 14,563 / 16,117 с | +| Повторная обработка готовой v1 | Пока v2 строилась, скачивалась v1. После COMMIT все скачанные варианты соответствовали v2. Дубль v1 не изменил опубликованный manifest | 16,302 с | +| Временная и постоянная ошибки | Временная ошибка отложила повтор. Постоянная дала failed, безопасный error_code и запись DLQ. Явный replay завершил то же задание со второй попытки | 8,366 / 9,060 с | +| Рестарт API, брокера и хранилищ при остановленных worker | Подтверждённое задание осталось queued, скачивание вернуло 409. После запуска worker вложение стало ready с первой попытки, ссылка в сообщении сохранилась, staging очищен | 23,390 с | + +Все 15 сценариев завершились с PASS. После эксперимента Compose повторно подтвердил готовность всех долгоживущих сервисов. [Результаты JSONL](evidence/track2.jsonl) · [код завершения](evidence/track2-errors.txt) · [готовность после эксперимента](evidence/track2-final-health.txt) ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Я опираюсь на сохранность volumes PostgreSQL, RabbitMQ и MinIO на одном хосте. +Временная потеря связи поддерживается протоколом, а потеря диска или всего +хоста выходит за пределы эксперимента. Outbox и lease закрывают разные окна +отказа. Publisher confirm подтверждает приём брокером, а consumer ack снимает +ответственность за конкретную доставку после долговечного решения. Между ними +процесс может остановиться, и RabbitMQ может повторить доставку. Операция S3 +может выполниться несколько раз. Идемпотентность проявляется в единственной +публикации активного manifest и в стабильном содержимом адресуемых объектов. +Гарантии exactly-once доставки здесь нет. + +Старые версии результата и объекты незавершённой устаревшей версии могут +оставаться в закрытом bucket. Для длительной эксплуатации потребуется отдельная +политика их удаления и аудит потребления места. Lease фиксированный, без +продления, с запасом относительно бюджета обработки. RabbitMQ в стенде +одноузловой. Эксперимент измеряет восстановление одного стенда и использует +один профиль входного изображения. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e2588869 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,209 @@ +name: mtusi2026lab5 +services: + postgres: + image: postgres:16.8-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -h 127.0.0.1 -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 1.0 + mem_limit: 512m + restart: unless-stopped + redis: + image: redis:7.4.2-alpine + command: [redis-server, --appendonly, 'yes', --appendfsync, always] + volumes: [redis_data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 20 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + migrate: + build: . + image: mtusi2026lab5-api:local + command: [python, -m, src.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 256m + restart: 'no' + api: + build: . + image: mtusi2026lab5-api:local + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + deploy: + replicas: 2 + healthcheck: + test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()'] + interval: 2s + timeout: 3s + retries: 3 + start_period: 5s + tmpfs: + - /tmp:size=16m,mode=1777 + stop_grace_period: 6s + cpus: 0.75 + mem_limit: 768m + restart: unless-stopped + broker: + image: rabbitmq:4.1.4-management + entrypoint: [/bin/sh, /broker-start.sh] + environment: + RABBITMQ_DEFAULT_USER: messenger + secrets: [broker_password] + volumes: + - broker_data:/var/lib/rabbitmq + - ./infra/broker-start.sh:/broker-start.sh:ro + hostname: broker + ports: ['127.0.0.1:${BROKER_CONSOLE_PORT:-15685}:15672'] + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, check_running] + interval: 4s + timeout: 5s + retries: 20 + mem_limit: 768m + cpus: 1.0 + restart: unless-stopped + publisher: &pipeline + image: mtusi2026lab5-api:local + command: [python, -m, src.publisher] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + PIPELINE_FAULTS: ${PIPELINE_FAULTS:-0} + secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url] + depends_on: + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -m, src.pipeline_health] + interval: 3s + timeout: 2s + retries: 3 + start_period: 20s + tmpfs: + - /tmp:size=16m,mode=1777 + cpus: 0.25 + mem_limit: 256m + restart: unless-stopped + worker: + <<: *pipeline + command: [python, -m, src.worker] + deploy: + replicas: ${WORKER_CONCURRENCY:-2} + init: true + tmpfs: + - /tmp:size=64m,mode=1777 + cpus: 1.0 + mem_limit: 768m + stop_grace_period: 30s + recovery: + <<: *pipeline + command: [python, -m, src.recovery] + lb: + image: haproxy:3.0.9-alpine + ports: ['127.0.0.1:${PUBLIC_PORT:-8080}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: + api: + condition: service_healthy + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready'] + interval: 5s + timeout: 3s + retries: 3 + cpus: 0.5 + mem_limit: 128m + restart: unless-stopped + s3: + image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 + entrypoint: [/opt/bitnami/minio/bin/minio] + user: '0:0' + command: [server, /data, --console-address, ':9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + ports: ['127.0.0.1:${S3_CONSOLE_PORT:-9005}:9001'] + healthcheck: + test: [CMD, curl, -f, http://localhost:9000/minio/health/live] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + mem_limit: 768m + cpus: 1.0 + s3-bootstrap: + image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 + entrypoint: [/bin/sh, /bootstrap.sh] + volumes: + - ./infra/s3-bootstrap.sh:/bootstrap.sh:ro + - ./infra/s3-policy.json:/policy.json:ro + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + depends_on: + s3: + condition: service_healthy + restart: 'no' +volumes: + s3_data: + postgres_data: + redis_data: + broker_data: +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + broker_url: + file: ./secrets/broker_url + broker_password: + file: ./secrets/broker_password diff --git a/contracts/pipeline-operations.md b/contracts/pipeline-operations.md new file mode 100644 index 00000000..07d29eb8 --- /dev/null +++ b/contracts/pipeline-operations.md @@ -0,0 +1,28 @@ +# Операторские операции pipeline + +Эти команды выполняются внутри доверенного Compose-проекта. HTTP-контракт +мессенджера остаётся исходным. Идентификаторы берутся из PostgreSQL. + +```sh +./scripts/compose.sh exec -T recovery python -m src.operator reprocess ATTACHMENT_UUID 2 +./scripts/compose.sh exec -T recovery python -m src.operator replay JOB_UUID +./scripts/compose.sh exec -T recovery python -m src.operator redeliver JOB_UUID +``` + +- `reprocess` принимает версию выше желаемой. Реализованы 1 и 2. Старая готовая + версия доступна, пока новая строится. Для вложения без готовой версии + используются обычные состояния LAB5. +- `replay` принимает только failed job текущей желаемой версии с сохранённым + исходником. UUID job сохраняется, attempt растёт, бюджет попыток обновляется. + История отказа остаётся в `job_dead_letters`. +- `redeliver` служит для диагностики. Команда посылает текущий carrier через + RabbitMQ и ждёт publisher confirm. Она сама не выдаёт lease и не меняет статус. + +В `image_jobs` хранится состояние каждой версии. Журнал `job_events` сохраняет +захват, номер fencing token, подтверждение, retry, DLQ, публикацию результата и +отклонение устаревшего исполнителя. Выход CLI содержит поля carrier. + +Внедрение сбоев разрешает только переменная `PIPELINE_FAULTS=1`. Управление +производится непосредственно через PostgreSQL на выделенном тестовом стенде. +Разовые команды в `pipeline_faults` привязаны к UUID вложения и версии. Выключайте +внедрение сбоев в обычном запуске. diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 00000000..f47793f5 --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,51 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name VARCHAR(100) NOT NULL + CHECK (length(btrim(display_name)) > 0), + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL + CHECK (length(btrim(title)) > 0), + created_by UUID NOT NULL REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id), + joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL + CHECK (length(text) <= 4000 AND length(btrim(text)) > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id + ON chat_members (user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expires_at + ON sessions (expires_at); diff --git a/db/migrations/002_idempotency.sql b/db/migrations/002_idempotency.sql new file mode 100644 index 00000000..69ddcd33 --- /dev/null +++ b/db/migrations/002_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_requests ( + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL REFERENCES users(id), + key VARCHAR(128) NOT NULL, + text TEXT NOT NULL, + message_id UUID REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chat_id, sender_id, key) +); diff --git a/db/migrations/003_attachments.sql b/db/migrations/003_attachments.sql new file mode 100644 index 00000000..78952f5d --- /dev/null +++ b/db/migrations/003_attachments.sql @@ -0,0 +1,16 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + sha256 CHAR(64) NOT NULL, + status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0)); diff --git a/db/migrations/004_pipeline.sql b/db/migrations/004_pipeline.sql new file mode 100644 index 00000000..11b773d8 --- /dev/null +++ b/db/migrations/004_pipeline.sql @@ -0,0 +1,77 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +ALTER TABLE attachments ADD CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments + ADD COLUMN desired_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN active_version INTEGER, + ADD COLUMN staging_key TEXT UNIQUE, + ADD COLUMN error_code TEXT; + +CREATE TABLE image_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL CHECK (pipeline_version > 0), + state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')), + attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt > 0), + failures INTEGER NOT NULL DEFAULT 0, + fence BIGINT NOT NULL DEFAULT 0, + lease_until TIMESTAMPTZ, + error_code TEXT, + manifest JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((state = 'processing') = (lease_until IS NOT NULL)), + CHECK ((state = 'ready') = (manifest IS NOT NULL)) +); +CREATE INDEX image_jobs_expired ON image_jobs(lease_until) WHERE state = 'processing'; + +CREATE TABLE job_outbox ( + job_id UUID PRIMARY KEY REFERENCES image_jobs(id), + publish_after TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX job_outbox_due ON job_outbox(publish_after); + +CREATE TABLE job_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id UUID NOT NULL REFERENCES image_jobs(id), + attempt INTEGER NOT NULL, + event TEXT NOT NULL, + detail TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX job_events_lookup ON job_events(job_id, id); + +-- Durable operator-visible DLQ. Replays retain the historical failure rows. +CREATE TABLE job_dead_letters ( + job_id UUID NOT NULL REFERENCES image_jobs(id), + attempt INTEGER NOT NULL, + error_code TEXT NOT NULL, + failed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + replayed_at TIMESTAMPTZ, + PRIMARY KEY (job_id, attempt) +); + +-- Used only by an explicitly enabled synthetic fault-injection stand. +CREATE TABLE pipeline_faults ( + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL, + point TEXT NOT NULL, + action TEXT NOT NULL CHECK (action IN ('crash','pause','temporary','permanent')), + hit_at TIMESTAMPTZ, + released BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (attachment_id,pipeline_version,point) +); + +-- Upgrade persisted lab4 originals, which remain the source until v1 commits. +UPDATE attachments SET status='queued', updated_at=now(); +WITH inserted AS ( + INSERT INTO image_jobs(attachment_id,pipeline_version) + SELECT id,1 FROM attachments RETURNING id +) +INSERT INTO job_outbox(job_id) SELECT id FROM inserted; + +ALTER TABLE attachments + ADD CHECK (desired_version > 0 AND (active_version IS NULL OR active_version <= desired_version)), + ADD CHECK ((status = 'ready') = (active_version IS NOT NULL)), + ADD CHECK ((status = 'failed') = (error_code IS NOT NULL)), + ADD FOREIGN KEY (id, active_version) REFERENCES image_jobs(attachment_id, pipeline_version); diff --git a/evidence/compose-up.txt b/evidence/compose-up.txt new file mode 100644 index 00000000..dee0edb8 --- /dev/null +++ b/evidence/compose-up.txt @@ -0,0 +1,231 @@ +Compose now can delegate build to bake for better performances +Just set COMPOSE_BAKE=true +#0 building with "desktop-linux" instance using docker driver + +#1 [migrate internal] load build definition from Dockerfile +#1 transferring dockerfile: 1.04kB done +#1 DONE 0.0s + +#2 [migrate internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm +#2 DONE 0.0s + +#3 [migrate internal] load .dockerignore +#3 transferring context: 208B done +#3 DONE 0.0s + +#4 [migrate internal] load build context +#4 transferring context: 3.49kB done +#4 DONE 0.0s + +#5 [migrate 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done +#5 DONE 0.0s + +#6 [migrate 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt +#6 CACHED + +#7 [migrate 3/9] COPY requirements.txt . +#7 CACHED + +#8 [migrate 2/9] WORKDIR /app +#8 CACHED + +#9 [migrate 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser +#9 CACHED + +#10 [migrate 6/9] COPY --chown=appuser:appuser src/ ./src/ +#10 DONE 0.0s + +#11 [migrate 7/9] COPY --chown=appuser:appuser db/ ./db/ +#11 DONE 0.0s + +#12 [migrate 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json +#12 DONE 0.0s + +#13 [migrate 9/9] COPY infra/app-start.sh /usr/local/bin/app-start +#13 DONE 0.0s + +#14 [migrate] exporting to image +#14 exporting layers 0.1s done +#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44 +#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44 done +#14 exporting config sha256:6e878a3e1844f17aa93a4277e04be0bd98ca42f3ba53997976cb1499c4ab5a22 done +#14 exporting attestation manifest sha256:8998f445e472a60a93fd3480f4000dede2d0fa76d5fcb690911d90e747483592 0.0s done +#14 exporting manifest list sha256:404b7aaa2cb3fda8ba8e0ea873f7171be30797c57a39261d9b78ec48f3e477c5 done +#14 naming to docker.io/library/mtusi2026lab5-api:local done +#14 unpacking to docker.io/library/mtusi2026lab5-api:local 0.0s done +#14 DONE 0.2s + +#15 [migrate] resolving provenance for metadata file +#15 DONE 0.0s + +#16 [api internal] load build definition from Dockerfile +#16 transferring dockerfile: 1.04kB done +#16 DONE 0.0s + +#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm +#2 DONE 0.0s + +#17 [api internal] load .dockerignore +#17 transferring context: 208B done +#17 DONE 0.0s + +#18 [api internal] load build context +#18 transferring context: 2.45kB done +#18 DONE 0.0s + +#5 [api 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done +#5 DONE 0.0s + +#19 [api 3/9] COPY requirements.txt . +#19 CACHED + +#20 [api 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt +#20 CACHED + +#21 [api 7/9] COPY --chown=appuser:appuser db/ ./db/ +#21 CACHED + +#8 [api 2/9] WORKDIR /app +#8 CACHED + +#22 [api 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json +#22 CACHED + +#23 [api 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser +#23 CACHED + +#24 [api 6/9] COPY --chown=appuser:appuser src/ ./src/ +#24 CACHED + +#25 [api 9/9] COPY infra/app-start.sh /usr/local/bin/app-start +#25 CACHED + +#26 [api] exporting to image +#26 exporting layers done +#26 exporting manifest sha256:afc88e63bc12b5238aa444e8bdb808d7f36d417557b747036fa9fb8a16d604b3 done +#26 exporting config sha256:c50287970425ce9631e003b98819e176d36d3d36d2b33f9de55894b567a27cc3 done +#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179 +#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179 0.0s done +#26 exporting manifest list sha256:a8591065abe521d14b78bd1884238576e83d46e560f228be8faa90ec3e9a677f done +#26 naming to docker.io/library/mtusi2026lab5-api:local done +#26 unpacking to docker.io/library/mtusi2026lab5-api:local done +#26 DONE 0.1s + +#27 [api] resolving provenance for metadata file +#27 DONE 0.0s + api Built + migrate Built + Container mtusi2026lab5-s3-1 Running + Container mtusi2026lab5-postgres-1 Running + Container mtusi2026lab5-s3-bootstrap-1 Created + Container mtusi2026lab5-broker-1 Running + Container mtusi2026lab5-redis-1 Running + Container mtusi2026lab5-recovery-1 Stopping + Container mtusi2026lab5-api-2 Stopping + Container mtusi2026lab5-api-1 Stopping + Container mtusi2026lab5-worker-1 Stopping + Container mtusi2026lab5-worker-2 Stopping + Container mtusi2026lab5-publisher-1 Stopping + Container mtusi2026lab5-publisher-1 Stopped + Container mtusi2026lab5-worker-1 Stopped + Container mtusi2026lab5-worker-2 Stopped + Container mtusi2026lab5-api-2 Stopped + Container mtusi2026lab5-api-1 Stopped + Container mtusi2026lab5-recovery-1 Stopped + Container mtusi2026lab5-migrate-1 Recreate + Container mtusi2026lab5-migrate-1 Recreated + Container mtusi2026lab5-publisher-1 Recreate + Container mtusi2026lab5-worker-1 Recreate + Container mtusi2026lab5-worker-2 Recreate + Container mtusi2026lab5-recovery-1 Recreate + Container mtusi2026lab5-worker-1 Recreated + Container mtusi2026lab5-worker-2 Recreated + Container mtusi2026lab5-recovery-1 Recreated + Container mtusi2026lab5-publisher-1 Recreated + Container mtusi2026lab5-lb-1 Stopping + Container mtusi2026lab5-lb-1 Stopped + Container mtusi2026lab5-api-1 Recreate + Container mtusi2026lab5-api-1 Recreated + Container mtusi2026lab5-lb-1 Stopping + Container mtusi2026lab5-lb-1 Stopped + Container mtusi2026lab5-api-2 Recreate + Container mtusi2026lab5-api-2 Recreated + Container mtusi2026lab5-lb-1 Created + Container mtusi2026lab5-s3-1 Waiting + Container mtusi2026lab5-postgres-1 Waiting + Container mtusi2026lab5-s3-1 Healthy + Container mtusi2026lab5-postgres-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Starting + Container mtusi2026lab5-migrate-1 Starting + Container mtusi2026lab5-migrate-1 Started + Container mtusi2026lab5-s3-bootstrap-1 Started + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-redis-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-redis-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-recovery-1 Starting + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-api-2 Starting + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-worker-2 Starting + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-publisher-1 Starting + Container mtusi2026lab5-publisher-1 Started + Container mtusi2026lab5-api-2 Started + Container mtusi2026lab5-api-1 Starting + Container mtusi2026lab5-recovery-1 Started + Container mtusi2026lab5-worker-2 Started + Container mtusi2026lab5-worker-1 Starting + Container mtusi2026lab5-api-1 Started + Container mtusi2026lab5-api-2 Waiting + Container mtusi2026lab5-api-1 Waiting + Container mtusi2026lab5-worker-1 Started + Container mtusi2026lab5-api-2 Healthy + Container mtusi2026lab5-api-1 Healthy + Container mtusi2026lab5-lb-1 Starting + Container mtusi2026lab5-lb-1 Started + Container mtusi2026lab5-redis-1 Waiting + Container mtusi2026lab5-s3-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-worker-2 Waiting + Container mtusi2026lab5-worker-1 Waiting + Container mtusi2026lab5-api-2 Waiting + Container mtusi2026lab5-api-1 Waiting + Container mtusi2026lab5-publisher-1 Waiting + Container mtusi2026lab5-lb-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-postgres-1 Waiting + Container mtusi2026lab5-recovery-1 Waiting + Container mtusi2026lab5-postgres-1 Healthy + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-redis-1 Healthy + Container mtusi2026lab5-recovery-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-publisher-1 Healthy + Container mtusi2026lab5-s3-1 Healthy + Container mtusi2026lab5-api-2 Healthy + Container mtusi2026lab5-api-1 Healthy + Container mtusi2026lab5-worker-2 Healthy + Container mtusi2026lab5-worker-1 Healthy + Container mtusi2026lab5-lb-1 Healthy diff --git a/evidence/container-render.txt b/evidence/container-render.txt new file mode 100644 index 00000000..bd1522af --- /dev/null +++ b/evidence/container-render.txt @@ -0,0 +1,3 @@ +PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding +PASS: fixture geometry, original preserved, deterministic v1, different v2 +PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed diff --git a/evidence/container-resources.txt b/evidence/container-resources.txt new file mode 100644 index 00000000..aabaa1f3 --- /dev/null +++ b/evidence/container-resources.txt @@ -0,0 +1,11 @@ +mtusi2026lab5-api-1 12.98% 65.53MiB / 768MiB +mtusi2026lab5-api-2 14.53% 65.53MiB / 768MiB +mtusi2026lab5-broker-1 10.98% 92.7MiB / 768MiB +mtusi2026lab5-lb-1 0.37% 11.71MiB / 128MiB +mtusi2026lab5-postgres-1 2.31% 35.48MiB / 512MiB +mtusi2026lab5-publisher-1 0.22% 26MiB / 256MiB +mtusi2026lab5-recovery-1 0.09% 52.61MiB / 256MiB +mtusi2026lab5-redis-1 0.33% 9.324MiB / 128MiB +mtusi2026lab5-s3-1 3.14% 418.5MiB / 768MiB +mtusi2026lab5-worker-1 0.16% 53.38MiB / 768MiB +mtusi2026lab5-worker-2 0.18% 53.38MiB / 768MiB diff --git a/evidence/failover-containers.txt b/evidence/failover-containers.txt new file mode 100644 index 00000000..cbebee79 --- /dev/null +++ b/evidence/failover-containers.txt @@ -0,0 +1,4 @@ +/mtusi2026lab5-api-1 running healthy +/mtusi2026lab5-api-2 running healthy +/mtusi2026lab5-api-1 running healthy +/mtusi2026lab5-api-2 exited unhealthy diff --git a/evidence/failover-repeat.txt b/evidence/failover-repeat.txt new file mode 100644 index 00000000..4e16bae6 --- /dev/null +++ b/evidence/failover-repeat.txt @@ -0,0 +1,9 @@ +Repeat 1 +/mtusi2026lab5-api-1 running healthy +/mtusi2026lab5-api-2 running healthy +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 2, "first_recovery_seconds": 1.925} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +/mtusi2026lab5-api-1 exited unhealthy +/mtusi2026lab5-api-2 running healthy diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..6ddf8f6c --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 3, "first_recovery_seconds": 2.28} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..a8a2ef6d --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,18 @@ +python3 tests/load.py --base-url "http://localhost:8080" +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 30.037, + "concurrency": 4, + "successful_requests": 7583, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 252.458, + "successful_latency_ms": { + "p50": 12.714, + "p95": 37.804, + "p99": 73.943 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/local-checks.txt b/evidence/local-checks.txt new file mode 100644 index 00000000..040d9e7f --- /dev/null +++ b/evidence/local-checks.txt @@ -0,0 +1,15 @@ +Fri Nov 20 15:25:49 UTC 2026 +276dbda +Darwin arm64 +8 +17179869184 +Python 3.13.3 +Docker Compose version v2.33.1-desktop.1 +Engine=28.0.1 OS=linux arch=aarch64 CPUs=8 memory_bytes=8218316800 +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding +PASS: fixture geometry, original preserved, deterministic v1, different v2 +PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/post-restart-health.txt b/evidence/post-restart-health.txt new file mode 100644 index 00000000..39240ff1 --- /dev/null +++ b/evidence/post-restart-health.txt @@ -0,0 +1,75 @@ + Container mtusi2026lab5-s3-1 Running + Container mtusi2026lab5-broker-1 Running + Container mtusi2026lab5-postgres-1 Running + Container mtusi2026lab5-redis-1 Running + Container mtusi2026lab5-s3-bootstrap-1 Created + Container mtusi2026lab5-migrate-1 Created + Container mtusi2026lab5-publisher-1 Running + Container mtusi2026lab5-recovery-1 Running + Container mtusi2026lab5-worker-1 Running + Container mtusi2026lab5-api-1 Running + Container mtusi2026lab5-worker-2 Running + Container mtusi2026lab5-api-2 Running + Container mtusi2026lab5-lb-1 Running + Container mtusi2026lab5-postgres-1 Waiting + Container mtusi2026lab5-s3-1 Waiting + Container mtusi2026lab5-postgres-1 Healthy + Container mtusi2026lab5-migrate-1 Starting + Container mtusi2026lab5-s3-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Starting + Container mtusi2026lab5-s3-bootstrap-1 Started + Container mtusi2026lab5-migrate-1 Started + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-redis-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-redis-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-api-1 Waiting + Container mtusi2026lab5-api-2 Waiting + Container mtusi2026lab5-api-1 Healthy + Container mtusi2026lab5-api-2 Healthy + Container mtusi2026lab5-recovery-1 Waiting + Container mtusi2026lab5-migrate-1 Waiting + Container mtusi2026lab5-broker-1 Waiting + Container mtusi2026lab5-postgres-1 Waiting + Container mtusi2026lab5-s3-1 Waiting + Container mtusi2026lab5-publisher-1 Waiting + Container mtusi2026lab5-s3-bootstrap-1 Waiting + Container mtusi2026lab5-worker-1 Waiting + Container mtusi2026lab5-worker-2 Waiting + Container mtusi2026lab5-lb-1 Waiting + Container mtusi2026lab5-redis-1 Waiting + Container mtusi2026lab5-api-1 Waiting + Container mtusi2026lab5-api-2 Waiting + Container mtusi2026lab5-redis-1 Healthy + Container mtusi2026lab5-broker-1 Healthy + Container mtusi2026lab5-recovery-1 Healthy + Container mtusi2026lab5-migrate-1 Exited + Container mtusi2026lab5-publisher-1 Healthy + Container mtusi2026lab5-s3-1 Healthy + Container mtusi2026lab5-postgres-1 Healthy + Container mtusi2026lab5-s3-bootstrap-1 Exited + Container mtusi2026lab5-lb-1 Healthy + Container mtusi2026lab5-worker-1 Healthy + Container mtusi2026lab5-worker-2 Healthy + Container mtusi2026lab5-api-1 Healthy + Container mtusi2026lab5-api-2 Healthy diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 00000000..466189d5 --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,11 @@ +/mtusi2026lab5-api-1 2026-11-20T15:26:46.863943835Z running +/mtusi2026lab5-api-2 2026-11-20T15:26:46.921602543Z running +/mtusi2026lab5-broker-1 2026-11-20T15:26:47.614707835Z running +/mtusi2026lab5-lb-1 2026-11-20T15:26:46.466838501Z running +/mtusi2026lab5-postgres-1 2026-11-20T15:26:46.30309871Z running +/mtusi2026lab5-publisher-1 2026-11-20T15:26:46.328859918Z running +/mtusi2026lab5-recovery-1 2026-11-20T15:26:46.960652918Z running +/mtusi2026lab5-redis-1 2026-11-20T15:26:46.267219626Z running +/mtusi2026lab5-s3-1 2026-11-20T15:26:46.428910043Z running +/mtusi2026lab5-worker-1 2026-11-20T15:26:46.488719835Z running +/mtusi2026lab5-worker-2 2026-11-20T15:26:46.475586876Z running diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 00000000..2527c41f --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,11 @@ +/mtusi2026lab5-api-1 2026-11-20T15:26:01.213628633Z running +/mtusi2026lab5-api-2 2026-11-20T15:26:01.040028133Z running +/mtusi2026lab5-broker-1 2026-11-20T15:23:46.29031721Z running +/mtusi2026lab5-lb-1 2026-11-20T15:26:06.958527761Z running +/mtusi2026lab5-postgres-1 2026-11-20T15:23:44.823417251Z running +/mtusi2026lab5-publisher-1 2026-11-20T15:26:01.037922008Z running +/mtusi2026lab5-recovery-1 2026-11-20T15:26:01.0402713Z running +/mtusi2026lab5-redis-1 2026-11-20T15:23:44.789881876Z running +/mtusi2026lab5-s3-1 2026-11-20T15:23:45.012262667Z running +/mtusi2026lab5-worker-1 2026-11-20T15:26:01.26324605Z running +/mtusi2026lab5-worker-2 2026-11-20T15:26:01.0369378Z running diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..df05fe26 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/track2-errors.txt b/evidence/track2-errors.txt new file mode 100644 index 00000000..d432d119 --- /dev/null +++ b/evidence/track2-errors.txt @@ -0,0 +1 @@ +pipeline.py exit=0 diff --git a/evidence/track2-final-health.txt b/evidence/track2-final-health.txt new file mode 100644 index 00000000..ec66693a --- /dev/null +++ b/evidence/track2-final-health.txt @@ -0,0 +1,75 @@ + Container mtusi2026lab5-track2-postgres-1 Running + Container mtusi2026lab5-track2-s3-1 Running + Container mtusi2026lab5-track2-broker-1 Running + Container mtusi2026lab5-track2-redis-1 Running + Container mtusi2026lab5-track2-s3-bootstrap-1 Created + Container mtusi2026lab5-track2-migrate-1 Created + Container mtusi2026lab5-track2-recovery-1 Running + Container mtusi2026lab5-track2-publisher-1 Running + Container mtusi2026lab5-track2-worker-1 Running + Container mtusi2026lab5-track2-worker-2 Running + Container mtusi2026lab5-track2-api-1 Running + Container mtusi2026lab5-track2-api-2 Running + Container mtusi2026lab5-track2-lb-1 Running + Container mtusi2026lab5-track2-s3-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Healthy + Container mtusi2026lab5-track2-migrate-1 Starting + Container mtusi2026lab5-track2-s3-1 Healthy + Container mtusi2026lab5-track2-s3-bootstrap-1 Starting + Container mtusi2026lab5-track2-s3-bootstrap-1 Started + Container mtusi2026lab5-track2-migrate-1 Started + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-redis-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-redis-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-api-2 Waiting + Container mtusi2026lab5-track2-api-1 Waiting + Container mtusi2026lab5-track2-api-2 Healthy + Container mtusi2026lab5-track2-api-1 Healthy + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-api-2 Waiting + Container mtusi2026lab5-track2-api-1 Waiting + Container mtusi2026lab5-track2-recovery-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Waiting + Container mtusi2026lab5-track2-redis-1 Waiting + Container mtusi2026lab5-track2-worker-1 Waiting + Container mtusi2026lab5-track2-worker-2 Waiting + Container mtusi2026lab5-track2-lb-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-1 Waiting + Container mtusi2026lab5-track2-publisher-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Healthy + Container mtusi2026lab5-track2-publisher-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-redis-1 Healthy + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-lb-1 Healthy + Container mtusi2026lab5-track2-recovery-1 Healthy + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-s3-1 Healthy + Container mtusi2026lab5-track2-api-2 Healthy + Container mtusi2026lab5-track2-api-1 Healthy + Container mtusi2026lab5-track2-worker-1 Healthy + Container mtusi2026lab5-track2-worker-2 Healthy diff --git a/evidence/track2-up.txt b/evidence/track2-up.txt new file mode 100644 index 00000000..430f4320 --- /dev/null +++ b/evidence/track2-up.txt @@ -0,0 +1,112 @@ + Container mtusi2026lab5-track2-postgres-1 Running + Container mtusi2026lab5-track2-broker-1 Running + Container mtusi2026lab5-track2-s3-1 Running + Container mtusi2026lab5-track2-s3-bootstrap-1 Created + Container mtusi2026lab5-track2-redis-1 Running + Container mtusi2026lab5-track2-recovery-1 Stopping + Container mtusi2026lab5-track2-publisher-1 Stopping + Container mtusi2026lab5-track2-api-2 Stopping + Container mtusi2026lab5-track2-api-1 Stopping + Container mtusi2026lab5-track2-worker-1 Stopping + Container mtusi2026lab5-track2-worker-2 Stopping + Container mtusi2026lab5-track2-publisher-1 Stopped + Container mtusi2026lab5-track2-worker-2 Stopped + Container mtusi2026lab5-track2-worker-1 Stopped + Container mtusi2026lab5-track2-api-2 Stopped + Container mtusi2026lab5-track2-recovery-1 Stopped + Container mtusi2026lab5-track2-api-1 Stopped + Container mtusi2026lab5-track2-migrate-1 Recreate + Container mtusi2026lab5-track2-migrate-1 Recreated + Container mtusi2026lab5-track2-recovery-1 Recreate + Container mtusi2026lab5-track2-publisher-1 Recreate + Container mtusi2026lab5-track2-worker-1 Recreate + Container mtusi2026lab5-track2-worker-2 Recreate + Container mtusi2026lab5-track2-publisher-1 Recreated + Container mtusi2026lab5-track2-recovery-1 Recreated + Container mtusi2026lab5-track2-worker-2 Recreated + Container mtusi2026lab5-track2-worker-1 Recreated + Container mtusi2026lab5-track2-lb-1 Stopping + Container mtusi2026lab5-track2-lb-1 Stopped + Container mtusi2026lab5-track2-api-1 Recreate + Container mtusi2026lab5-track2-api-1 Recreated + Container mtusi2026lab5-track2-lb-1 Stopping + Container mtusi2026lab5-track2-lb-1 Stopped + Container mtusi2026lab5-track2-api-2 Recreate + Container mtusi2026lab5-track2-api-2 Recreated + Container mtusi2026lab5-track2-lb-1 Created + Container mtusi2026lab5-track2-s3-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Waiting + Container mtusi2026lab5-track2-postgres-1 Healthy + Container mtusi2026lab5-track2-migrate-1 Starting + Container mtusi2026lab5-track2-s3-1 Healthy + Container mtusi2026lab5-track2-s3-bootstrap-1 Starting + Container mtusi2026lab5-track2-migrate-1 Started + Container mtusi2026lab5-track2-s3-bootstrap-1 Started + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-redis-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-redis-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-api-2 Starting + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-publisher-1 Starting + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-recovery-1 Starting + Container mtusi2026lab5-track2-worker-1 Starting + Container mtusi2026lab5-track2-recovery-1 Started + Container mtusi2026lab5-track2-worker-1 Started + Container mtusi2026lab5-track2-worker-2 Starting + Container mtusi2026lab5-track2-publisher-1 Started + Container mtusi2026lab5-track2-api-2 Started + Container mtusi2026lab5-track2-api-1 Starting + Container mtusi2026lab5-track2-api-1 Started + Container mtusi2026lab5-track2-api-2 Waiting + Container mtusi2026lab5-track2-api-1 Waiting + Container mtusi2026lab5-track2-worker-2 Started + Container mtusi2026lab5-track2-api-2 Healthy + Container mtusi2026lab5-track2-api-1 Healthy + Container mtusi2026lab5-track2-lb-1 Starting + Container mtusi2026lab5-track2-lb-1 Started + Container mtusi2026lab5-track2-lb-1 Waiting + Container mtusi2026lab5-track2-api-2 Waiting + Container mtusi2026lab5-track2-api-1 Waiting + Container mtusi2026lab5-track2-recovery-1 Waiting + Container mtusi2026lab5-track2-redis-1 Waiting + Container mtusi2026lab5-track2-broker-1 Waiting + Container mtusi2026lab5-track2-publisher-1 Waiting + Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting + Container mtusi2026lab5-track2-s3-1 Waiting + Container mtusi2026lab5-track2-migrate-1 Waiting + Container mtusi2026lab5-track2-worker-1 Waiting + Container mtusi2026lab5-track2-worker-2 Waiting + Container mtusi2026lab5-track2-postgres-1 Waiting + Container mtusi2026lab5-track2-recovery-1 Healthy + Container mtusi2026lab5-track2-broker-1 Healthy + Container mtusi2026lab5-track2-publisher-1 Healthy + Container mtusi2026lab5-track2-postgres-1 Healthy + Container mtusi2026lab5-track2-s3-1 Healthy + Container mtusi2026lab5-track2-migrate-1 Exited + Container mtusi2026lab5-track2-redis-1 Healthy + Container mtusi2026lab5-track2-api-2 Healthy + Container mtusi2026lab5-track2-api-1 Healthy + Container mtusi2026lab5-track2-worker-1 Healthy + Container mtusi2026lab5-track2-worker-2 Healthy + Container mtusi2026lab5-track2-s3-bootstrap-1 Exited + Container mtusi2026lab5-track2-lb-1 Healthy diff --git a/evidence/track2.jsonl b/evidence/track2.jsonl new file mode 100644 index 00000000..d94e800c --- /dev/null +++ b/evidence/track2.jsonl @@ -0,0 +1,15 @@ +{"scenario": "concurrent_message", "result": "PASS", "seconds": 0.041, "attachment_id": null, "requests": 6} +{"scenario": "upload_validation", "result": "PASS", "seconds": 0.379, "attachment_id": null, "statuses": [404, 415, 415, 413]} +{"scenario": "broker_unavailable", "result": "PASS", "seconds": 19.129, "attachment_id": "5b35bebd-3c16-431c-9167-1fae38db338d", "attempt": 1, "pipeline_health_recovered": true} +{"scenario": "before_publish", "result": "PASS", "seconds": 12.328, "attachment_id": "ac1de244-d64b-43f2-98c9-d4662503eb35", "attempt": 1, "published": 1} +{"scenario": "after_confirm", "result": "PASS", "seconds": 12.365, "attachment_id": "75602f19-039e-4bbf-bfb9-25c677d39ba9", "attempt": 1, "published": 1} +{"scenario": "before_s3", "result": "PASS", "seconds": 28.879, "attachment_id": "e8fa8c69-04d7-4f51-b644-17b538a4f6c7", "attempt": 2, "published": 1} +{"scenario": "after_s3", "result": "PASS", "seconds": 28.39, "attachment_id": "7c7439af-6b20-4941-afbb-41c8de02bf3c", "attempt": 2, "published": 1} +{"scenario": "after_commit", "result": "PASS", "seconds": 7.57, "attachment_id": "753ad377-cac2-4630-8207-efeffce7522c", "attempt": 1, "published": 1} +{"scenario": "duplicate_and_fencing", "result": "PASS", "seconds": 33.826, "attachment_id": "c0e36849-d59d-43ff-9f77-2af03cdd9bcf", "fence": 2, "attempt": 2} +{"scenario": "reorder_in_flight", "result": "PASS", "seconds": 14.563, "attachment_id": "7717096b-a8a2-45d1-ad9c-d178ad545d4d", "active_version": 2} +{"scenario": "late_old_delivery", "result": "PASS", "seconds": 16.117, "attachment_id": "0ee3fe7e-bd09-403d-8f6e-a7881ed2a1f0", "active_version": 2} +{"scenario": "atomic_version_switch", "result": "PASS", "seconds": 16.302, "attachment_id": "3a58381d-2c97-4728-b3f3-4b72457f3e35", "active_version": 2} +{"scenario": "temporary_error", "result": "PASS", "seconds": 8.366, "attachment_id": "7e9d1700-8dee-4622-9946-dc6bd2527a3b", "attempt": 2} +{"scenario": "permanent_error", "result": "PASS", "seconds": 9.06, "attachment_id": "7853eedb-f16a-421d-870e-f72496843dc3", "attempt": 2} +{"scenario": "restart_with_queued_attachment", "result": "PASS", "seconds": 23.39, "attachment_id": "e1baa3d5-bed2-48f7-ac29-5713b91b56d2", "attempt": 1, "message_link_preserved": true, "staging_cleaned": true} diff --git a/infra/app-start.sh b/infra/app-start.sh new file mode 100644 index 00000000..756eed01 --- /dev/null +++ b/infra/app-start.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +# Compose file-based secrets inherit host ownership. Copy only the secrets +# mounted for this role before dropping privileges. +install -d -m 0700 -o appuser -g appuser /run/app-secrets +for key in DATABASE_URL_FILE S3_ACCESS_KEY_ID_FILE S3_SECRET_ACCESS_KEY_FILE BROKER_URL_FILE; do + source_file=$(printenv "$key") || continue + target="/run/app-secrets/$key" + install -m 0400 -o appuser -g appuser "$source_file" "$target" + export "$key=$target" +done +exec setpriv --reuid=appuser --regid=appuser --init-groups -- "$@" diff --git a/infra/broker-start.sh b/infra/broker-start.sh new file mode 100644 index 00000000..c34afdf1 --- /dev/null +++ b/infra/broker-start.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)" +exec docker-entrypoint.sh rabbitmq-server diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..6f726a02 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 60s + timeout server 60s + timeout check 1s + # Never replay a potentially committed POST automatically. + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 1-10 api:8080 check diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 00000000..a0e5f3ff --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments +mc anonymous set none local/messenger-attachments +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy create local attachments-api /policy.json +mc admin policy attach local attachments-api --user "$(cat /run/secrets/s3_access_key_id)" diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..cc1d04db --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,34 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/results/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ], + "Condition": { + "StringLike": { + "s3:prefix": [ + "originals/*", "staging/*", "results/*" + ] + } + } + } + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..bde1ac87 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.38.12 +Pillow==11.2.1 + +pika==1.3.2 diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 00000000..12cc5ed7 --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then + export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH" +fi +exec docker compose --env-file .env "$@" diff --git a/scripts/init-local.py b/scripts/init-local.py new file mode 100644 index 00000000..22be0062 --- /dev/null +++ b/scripts/init-local.py @@ -0,0 +1,26 @@ +"""Generate local credentials once; never overwrite an existing secret.""" +from pathlib import Path +import secrets +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(mode=0o700, exist_ok=True) +values = {'broker_password': secrets.token_hex(24), 'postgres_password': secrets.token_hex(24), 's3_root_user': 'lab5root', 's3_root_password': secrets.token_hex(24), 's3_access_key_id': 'lab5api', 's3_secret_access_key': secrets.token_hex(24)} +for name, value in values.items(): + path = directory / name + if not path.exists(): + path.write_text(value + '\n') + path.chmod(0o600) +path = directory / 'database_url' +if not path.exists(): + password = (directory / 'postgres_password').read_text().strip() + path.write_text(f'postgresql://messenger:{password}@postgres:5432/messenger\n') + path.chmod(0o600) +path = directory / 'broker_url' +if not path.exists(): + password = (directory / 'broker_password').read_text().strip() + path.write_text(f'amqp://messenger:{password}@broker:5672/%2F\n') + path.chmod(0o600) +path = root / '.env' +if not path.exists(): + path.write_text('COMPOSE_PROJECT_NAME=mtusi2026lab5\nPUBLIC_PORT=8080\n') +print('Local configuration ready (secret values suppressed)') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..da8923aa --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +containers=$(./scripts/compose.sh ps -aq postgres redis s3 broker api lb publisher worker recovery) +docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-before.txt +./scripts/compose.sh restart postgres redis s3 broker api lb publisher worker recovery +docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-after.txt diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..081b737f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}" +for container in $(./scripts/compose.sh ps -q api); do + instance=$(docker exec "$container" python -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])') + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 2 "$container" >/dev/null + exit 0 + fi +done +exit 1 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..725cde35 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,149 @@ +import asyncio +import hashlib +import subprocess +import sys +import tempfile +import unicodedata +from uuid import UUID, uuid4 + +from botocore.exceptions import BotoCoreError, ClientError +from fastapi import HTTPException, Request +from fastapi.responses import StreamingResponse +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from .database import get_connection +from .jobs import enqueue +from .storage import s3, BUCKET + +MAX_BYTES = 10 * 1024**2 +upload_slots = asyncio.Semaphore(1) + + +def serialize(row): + result = {key: str(row[key]) for key in ('id', 'chat_id', 'owner_id')} + result.update({key: row[key] for key in ('filename', 'content_type', 'size_bytes', 'sha256', 'status')}) + result.update({key: row[key].isoformat().replace('+00:00', 'Z') for key in ('created_at', 'updated_at')}) + manifest = row.get('manifest') or {} + variants = [ + dict(name=name, **{field: obj[field] for field in + ('content_type', 'size_bytes', 'sha256', 'width', 'height')}) + for name, obj in sorted(manifest.items()) if name != 'original' + ] + return dict(result, variants=variants, error_code=row['error_code']) + + +def persist(path, chat, owner, filename, mime, size, checksum): + attachment_id = uuid4() + key = 'staging/' + str(attachment_id) + # The same lock is taken by cleanup before its final DB check/deletion. + # A process crash releases the transaction; an uncommitted object is later cleaned. + with get_connection() as conn: + with conn.transaction(): + conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (key,)) + with open(path, 'rb') as source: + s3.put_object(Bucket=BUCKET, Key=key, Body=source, ContentLength=size, ContentType=mime) + row = conn.execute('''INSERT INTO attachments + (id, chat_id, owner_id, filename, object_key, size_bytes, content_type, sha256, staging_key, status) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *''', + (attachment_id, UUID(chat), UUID(owner), filename, key, size, mime, checksum, key)).fetchone() + enqueue(conn, attachment_id, 1) + # Exiting transaction above commits before returning 202. + return serialize(row) + + +def accessible(attachment_id, user_id): + try: + aid = UUID(attachment_id) + except ValueError: + raise HTTPException(400, 'Неверный UUID') from None + with get_connection() as conn: + row = conn.execute('''SELECT a.*, j.manifest FROM attachments a + LEFT JOIN image_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.active_version + JOIN chat_members cm ON cm.chat_id=a.chat_id + WHERE a.id=%s AND cm.user_id=%s''', (aid, UUID(user_id))).fetchone() + if row is None: + raise HTTPException(404, 'Вложение не найдено') + return row + + +def register(app, current_user, get_chat): + @app.post('/api/v1/chats/{chat_id}/attachments', status_code=202) + async def upload(chat_id: str, request: Request): + user = await run_in_threadpool(current_user, request) + chat = await run_in_threadpool(get_chat, chat_id, request) + filename = request.query_params.get('filename', '') + if (not 1 <= len(filename) <= 255 or not filename.strip() + or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename)): + raise HTTPException(400, 'Недопустимое имя файла') + mime = request.headers.get('content-type', '').split(';')[0].strip().lower() + if mime not in ('image/png', 'image/jpeg'): + raise HTTPException(415, 'Ожидается PNG или JPEG') + length = request.headers.get('content-length') + if length is not None: + if not length.isascii() or not length.isdecimal() or len(length) > 12: + raise HTTPException(400, 'Неверный размер') + if int(length) > MAX_BYTES: + raise HTTPException(413, 'Файл больше 10 MiB') + try: + await asyncio.wait_for(upload_slots.acquire(), timeout=30) + except asyncio.TimeoutError: + raise HTTPException(503, 'Загрузчик занят') from None + try: + with tempfile.NamedTemporaryFile(prefix='lab5-') as target: + size = 0 + digest = hashlib.sha256() + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413, 'Файл больше 10 MiB') + target.write(chunk) + digest.update(chunk) + except ClientDisconnect: + raise HTTPException(400, 'Загрузка прервана') from None + except TimeoutError: + raise HTTPException(400, 'Истекло время загрузки') from None + target.flush() + def validate(): + try: + return subprocess.run([sys.executable, '-m', 'src.image_check', target.name, mime], + timeout=10, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode + except subprocess.TimeoutExpired: + return 15 + code = await run_in_threadpool(validate) + if code != 0: + raise HTTPException(413 if code == 13 else 415, 'Недопустимое изображение') + return await run_in_threadpool(persist, target.name, chat['id'], user['id'], filename, mime, size, digest.hexdigest()) + finally: + upload_slots.release() + + @app.get('/api/v1/attachments/{attachment_id}') + def metadata(attachment_id: str, request: Request): + return serialize(accessible(attachment_id, current_user(request)['id'])) + + @app.get('/api/v1/attachments/{attachment_id}/content') + def content(attachment_id: str, request: Request): + row = accessible(attachment_id, current_user(request)['id']) + if row['status'] != 'ready': + raise HTTPException(409, 'Вложение не готово') + variant = row['manifest'].get(request.query_params.get('variant', 'original')) + if variant is None: + raise HTTPException(404, 'Вариант не найден') + obj = s3.get_object(Bucket=BUCKET, Key=variant['key']) + def chunks(): + try: + for chunk in obj['Body'].iter_chunks(64 * 1024): + yield chunk + finally: + obj['Body'].close() + return StreamingResponse(chunks(), media_type=variant['content_type'], headers={ + 'Content-Length': str(variant['size_bytes']), 'X-Content-Type-Options': 'nosniff', + 'Content-Disposition': 'attachment', 'Cache-Control': 'private, no-store'}) + + @app.exception_handler(BotoCoreError) + @app.exception_handler(ClientError) + async def storage_error(request, exc): + from .main import error_response + return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно') diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..6bb7c6cb --- /dev/null +++ b/src/auth.py @@ -0,0 +1,91 @@ +import base64 +import binascii +import hashlib +import os +import re +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + + +password_hasher = PasswordHasher( + time_cost=3, + memory_cost=65536, + parallelism=4, +) + + +def hash_password(password: str) -> str: + return password_hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return password_hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError): + return False + + +def parse_basic(authorization: str | None) -> tuple[str, str] | None: + if not authorization: + return None + + scheme, separator, credentials = authorization.partition(" ") + + if not separator or scheme.lower() != "basic": + return None + + try: + raw = base64.b64decode(credentials, validate=True) + decoded = raw.decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + return None + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + return None + + if not 12 <= len(password) <= 128: + return None + + return username, password + + +def parse_bearer(authorization: str | None) -> str | None: + if not authorization: + return None + + scheme, separator, token = authorization.partition(" ") + + if not separator or scheme.lower() != "bearer": + return None + + if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512: + return None + + return token + + +def create_token() -> str: + return secrets.token_urlsafe(32) + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("ascii")).hexdigest() + + +def session_expiration() -> datetime: + ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + + if ttl < 1: + raise ValueError("SESSION_TTL_SECONDS must be positive") + + return datetime.now(timezone.utc) + timedelta(seconds=ttl) + + +def format_datetime(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..a88b3762 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,33 @@ +"""Persistent RabbitMQ transport with explicit publisher confirms.""" +import json +import os +from contextlib import contextmanager +from pathlib import Path + +import pika + +QUEUE = 'attachment.process.v1' + + +@contextmanager +def channel(): + params = pika.URLParameters(Path(os.environ['BROKER_URL_FILE']).read_text().strip()) + params.connection_attempts = 1 + params.socket_timeout = 4 + params.stack_timeout = 8 + params.blocked_connection_timeout = 10 + params.heartbeat = 120 + connection = pika.BlockingConnection(params) + try: + ch = connection.channel() + ch.queue_declare(QUEUE, durable=True, arguments={'x-queue-type': 'quorum', 'x-delivery-limit': -1}) + yield connection, ch + finally: + if connection.is_open: + connection.close() + + +def send(ch, message): + ch.basic_publish('', QUEUE, json.dumps(message).encode(), mandatory=True, + properties=pika.BasicProperties(content_type='application/json', delivery_mode=2, + message_id=message['job_id'])) diff --git a/src/chat_repository.py b/src/chat_repository.py new file mode 100644 index 00000000..ae5acf34 --- /dev/null +++ b/src/chat_repository.py @@ -0,0 +1,254 @@ +from uuid import UUID, uuid4 + +from .database import get_connection + + +def serialize_chat(row): + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def serialize_message(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": [str(a) for a in row["attachment_ids"]], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def create_chat(creator_id, title, member_ids): + chat_id = uuid4() + members = [UUID(member_id) for member_id in member_ids] + + with get_connection() as conn: + with conn.transaction(): + with conn.cursor() as cur: + cur.execute( + """ + SELECT id + FROM users + WHERE id = ANY(%s::uuid[]) + """, + (members,), + ) + existing = {row["id"] for row in cur.fetchall()} + + if len(existing) != len(members): + return None + + cur.execute( + """ + INSERT INTO chats (id, title, created_by) + VALUES (%s, %s, %s) + RETURNING id, title, created_at + """, + (chat_id, title, UUID(creator_id)), + ) + chat = cur.fetchone() + + cur.executemany( + """ + INSERT INTO chat_members (chat_id, user_id) + VALUES (%s, %s) + """, + [(chat_id, member_id) for member_id in members], + ) + + return { + "id": str(chat["id"]), + "title": chat["title"], + "member_ids": member_ids, + "created_at": chat["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def get_chat(chat_id, user_id): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + c.id, + c.title, + c.created_at, + ARRAY( + SELECT cm2.user_id + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE c.id = %s + AND cm.user_id = %s + """, + (UUID(chat_id), UUID(user_id)), + ) + row = cur.fetchone() + return serialize_chat(row) if row else None + + +def list_chats(user_id, limit, last_chat_id=None): + with get_connection() as conn: + with conn.cursor() as cur: + last_key = None + + if last_chat_id is not None: + cur.execute( + """ + SELECT c.created_at, c.id + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE c.id = %s + AND cm.user_id = %s + """, + (UUID(last_chat_id), UUID(user_id)), + ) + last_key = cur.fetchone() + + if last_key is None: + return None + + cur.execute( + """ + SELECT + c.id, + c.title, + c.created_at, + ARRAY( + SELECT cm2.user_id + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE cm.user_id = %s + AND ( + %s::timestamptz IS NULL + OR (c.created_at, c.id) > + (%s::timestamptz, %s::uuid) + ) + ORDER BY c.created_at, c.id + LIMIT %s + """, + ( + UUID(user_id), + last_key["created_at"] if last_key else None, + last_key["created_at"] if last_key else None, + last_key["id"] if last_key else None, + limit + 1, + ), + ) + rows = cur.fetchall() + + return [serialize_chat(row) for row in rows] + + +def create_message(chat_id, sender_id, text, key, attachment_ids): + # INSERT ON CONFLICT waits for a concurrent transaction on the same key. + # READ COMMITTED's next statement sees its committed result. Both the + # message and deduplication record commit together, or both roll back. + with get_connection() as conn: + with conn.transaction(): + attachments = [UUID(a) for a in attachment_ids] + scope = (UUID(chat_id), UUID(sender_id), key) + conn.execute(""" + DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s + AND created_at <= clock_timestamp() - interval '24 hours' + """, scope) + row = conn.execute( + """INSERT INTO message_requests(chat_id, sender_id, key, text) + VALUES (%s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, key) DO NOTHING + RETURNING key""", (*scope, text) + ).fetchone() + if row is None: + previous = conn.execute( + """SELECT m.* FROM message_requests r + JOIN messages m ON m.id = r.message_id + WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""", + scope, + ).fetchone() + if previous["text"] != text or previous["attachment_ids"] != attachments: + return "conflict" + return serialize_message(previous) + rows = conn.execute( + 'SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) ' + 'ORDER BY id FOR SHARE', (UUID(chat_id), attachments) + ).fetchall() + if len(rows) != len(attachments) or any(row['status'] == 'failed' for row in rows): + # No unfinished idempotency record may survive validation failure. + conn.execute('DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s', scope) + return None if len(rows) != len(attachments) else "attachment_failed" + message = conn.execute( + """INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids) + VALUES (%s, %s, %s, %s, %s) RETURNING *""", + (uuid4(), UUID(chat_id), UUID(sender_id), text, attachments), + ).fetchone() + conn.execute( + """UPDATE message_requests SET message_id=%s + WHERE chat_id=%s AND sender_id=%s AND key=%s""", + (message["id"], *scope), + ) + return serialize_message(message) + + +def list_messages(chat_id, limit, last_message_id=None): + with get_connection() as conn: + with conn.cursor() as cur: + last_key = None + + if last_message_id is not None: + cur.execute( + """ + SELECT created_at, id + FROM messages + WHERE chat_id = %s + AND id = %s + """, + (UUID(chat_id), UUID(last_message_id)), + ) + last_key = cur.fetchone() + + if last_key is None: + return None + + # Разные формы SQL сохраняют индексный range scan и для generic plan. + # В SQL подставляется только константа; значения остаются параметрами. + cursor_condition = "" + params = [UUID(chat_id)] + if last_key is not None: + cursor_condition = "AND (created_at, id) > (%s, %s)" + params.extend([last_key["created_at"], last_key["id"]]) + params.append(limit + 1) + cur.execute( + f""" + SELECT id, chat_id, sender_id, text, attachment_ids, created_at + FROM messages + WHERE chat_id = %s + {cursor_condition} + ORDER BY created_at, id + LIMIT %s + """, + params, + ) + rows = cur.fetchall() + + return [serialize_message(row) for row in rows] diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..758eff55 --- /dev/null +++ b/src/database.py @@ -0,0 +1,48 @@ +import os +from contextlib import contextmanager +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def get_database_url() -> str: + secret_file = os.getenv("DATABASE_URL_FILE") + + if secret_file: + return Path(secret_file).read_text().strip() + + database_url = os.getenv("DATABASE_URL") + + if database_url: + return database_url + + raise RuntimeError("Database connection is not configured") + + +pool = ConnectionPool( + conninfo=get_database_url(), + min_size=1, + max_size=10, + timeout=5, + open=False, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": "-c statement_timeout=5000", + }, +) + + +def open_database(): + pool.open(wait=True, timeout=15) + + +def close_database(): + pool.close() + + +@contextmanager +def get_connection(): + with pool.connection() as connection: + yield connection diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 00000000..5c9b8b49 --- /dev/null +++ b/src/faults.py @@ -0,0 +1,42 @@ +"""One-shot fault controls, enabled only by the operator on a disposable stand.""" +import os +import time + +from .database import get_connection + +ENABLED = os.getenv('PIPELINE_FAULTS') == '1' + + +def take(conn, attachment_id, version, point): + if not ENABLED: + return None + row = conn.execute(""" + UPDATE pipeline_faults SET hit_at=clock_timestamp() + WHERE attachment_id=%s AND pipeline_version=%s AND point=%s AND hit_at IS NULL + RETURNING action + """, (attachment_id, version, point)).fetchone() + return row['action'] if row else None + + +def checkpoint(job, point): + if not ENABLED: + return None + key = (job['attachment_id'], job['pipeline_version'], point) + with get_connection() as conn: + action = take(conn, *key) + if action == 'crash': + print(f'fault {point}: crash {job["id"]}', flush=True) + os._exit(86) + if action == 'pause': + print(f'fault {point}: pause {job["id"]}', flush=True) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + with get_connection() as conn: + row = conn.execute(""" + SELECT released FROM pipeline_faults + WHERE attachment_id=%s AND pipeline_version=%s AND point=%s + """, key).fetchone() + if row is None or row['released']: + break + time.sleep(0.1) + return action diff --git a/src/image_check.py b/src/image_check.py new file mode 100644 index 00000000..d57527e3 --- /dev/null +++ b/src/image_check.py @@ -0,0 +1,25 @@ +"""Isolated image decoder: bounded address space, CPU and parent wall timeout.""" +import resource +import sys +import warnings + +if sys.platform.startswith('linux'): + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2)) +resource.setrlimit(resource.RLIMIT_CPU, (8, 8)) +from PIL import Image +Image.MAX_IMAGE_PIXELS = 40_000_000 +warnings.simplefilter('error', Image.DecompressionBombWarning) +try: + with Image.open(sys.argv[1]) as image: + if image.width * image.height > 40_000_000: + sys.exit(13) + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) + if actual != sys.argv[2] or getattr(image, 'n_frames', 1) != 1: + sys.exit(15) + image.verify() + with Image.open(sys.argv[1]) as image: + image.load() +except (Image.DecompressionBombError, Image.DecompressionBombWarning): + sys.exit(13) +except Exception: + sys.exit(15) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..8786f71b --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,154 @@ +"""PostgreSQL owns execution state and the publication pointer.""" +import os +import random + +from psycopg.types.json import Jsonb + +from .database import get_connection + +MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5')) +LEASE_SECONDS = int(os.getenv('JOB_TIMEOUT_SECONDS', '60')) + + +def enqueue(conn, attachment_id, version): + job = conn.execute(''' + INSERT INTO image_jobs(attachment_id,pipeline_version) VALUES (%s,%s) RETURNING * + ''', (attachment_id, version)).fetchone() + schedule(conn, job['id']) + return job + + +def schedule(conn, job_id, delay=0): + conn.execute(''' + INSERT INTO job_outbox(job_id,publish_after) + VALUES (%s, clock_timestamp() + %s * interval '1 second') + ON CONFLICT (job_id) DO UPDATE SET publish_after=excluded.publish_after + ''', (job_id, delay)) + + +def event(conn, job, name, detail=None): + conn.execute('INSERT INTO job_events(job_id,attempt,event,detail) VALUES (%s,%s,%s,%s)', + (job['id'], job['attempt'], name, detail)) + + +def carrier(job): + return dict(schema_version=1, type='attachment.process.v1', job_id=str(job['id']), + attachment_id=str(job['attachment_id']), pipeline_version=job['pipeline_version'], + attempt=job['attempt'], created_at=job['created_at'].isoformat().replace('+00:00', 'Z')) + + +def claim(message): + with get_connection() as conn: + job = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE', + (message['job_id'],)).fetchone() + if (job is None or job['attachment_id'] != message['attachment_id'] + or job['pipeline_version'] != message['pipeline_version'] + or job['attempt'] != message['attempt']): + return None + if job['state'] != 'queued': + event(conn, job, 'delivery_ignored', job['state']) + return None + attachment = lock_attachment(conn, job) + if attachment['desired_version'] != job['pipeline_version']: + supersede(conn, job) + return None + job = conn.execute(''' + UPDATE image_jobs SET state='processing', fence=fence+1, + lease_until=clock_timestamp() + %s * interval '1 second' + WHERE id=%s RETURNING * + ''', (LEASE_SECONDS, job['id'])).fetchone() + update_attachment(conn, job, 'processing') + event(conn, job, 'claimed', str(job['fence'])) + return job, attachment + + +def lock_attachment(conn, job): + return conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE', + (job['attachment_id'],)).fetchone() + + +def update_attachment(conn, job, state, error=None): + # While building a new version, continue serving the previously published version. + conn.execute(''' + UPDATE attachments SET status=%s, error_code=%s, updated_at=clock_timestamp() + WHERE id=%s AND desired_version=%s AND active_version IS NULL + ''', (state, error, job['attachment_id'], job['pipeline_version'])) + + +def supersede(conn, job): + conn.execute("UPDATE image_jobs SET state='superseded', lease_until=NULL WHERE id=%s", (job['id'],)) + event(conn, job, 'superseded') + + +def owned(conn, job): + current = conn.execute(''' + SELECT *, lease_until > clock_timestamp() AS live FROM image_jobs WHERE id=%s FOR NO KEY UPDATE + ''', (job['id'],)).fetchone() + return current['state'] == 'processing' and current['fence'] == job['fence'] and current['live'] + + +def finish(job, manifest): + with get_connection() as conn: + if not owned(conn, job): + event(conn, job, 'fence_rejected', str(job['fence'])) + return + attachment = lock_attachment(conn, job) + if not owned(conn, job): + event(conn, job, 'fence_rejected', str(job['fence'])) + return + if attachment['desired_version'] != job['pipeline_version']: + supersede(conn, job) + return + conn.execute(""" + UPDATE image_jobs SET state='ready', manifest=%s, lease_until=NULL,error_code=NULL WHERE id=%s + """, (Jsonb(manifest), job['id'])) + conn.execute(""" + UPDATE attachments SET status='ready', active_version=%s, error_code=NULL, + object_key=%s, updated_at=clock_timestamp() WHERE id=%s + """, (job['pipeline_version'], manifest['original']['key'], attachment['id'])) + event(conn, job, 'published') + + +def fail_locked(conn, job, reason, permanent=False): + attachment = lock_attachment(conn, job) + if attachment['desired_version'] != job['pipeline_version']: + supersede(conn, job) + return + failures = job['failures'] + 1 + if permanent or failures >= MAX_ATTEMPTS: + code = reason if permanent else 'retry_exhausted' + conn.execute(""" + UPDATE image_jobs SET state='failed', error_code=%s, lease_until=NULL, failures=%s WHERE id=%s + """, (code, failures, job['id'])) + conn.execute('INSERT INTO job_dead_letters(job_id,attempt,error_code) VALUES (%s,%s,%s)', + (job['id'], job['attempt'], code)) + update_attachment(conn, job, 'failed', code) + event(conn, job, 'dead_letter', reason) + return + delay = min(30, 2 ** (failures - 1)) * random.uniform(0.75, 1.25) + conn.execute(""" + UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=%s, + lease_until=NULL,error_code=%s WHERE id=%s + """, (failures, reason, job['id'])) + schedule(conn, job['id'], delay) + update_attachment(conn, job, 'queued') + event(conn, job, 'retry_scheduled', reason) + + +def fail(job, reason, permanent=False): + with get_connection() as conn: + if owned(conn, job): + fail_locked(conn, job, reason, permanent) + else: + event(conn, job, 'fence_rejected', str(job['fence'])) + + +def recover_one(): + with get_connection() as conn: + job = conn.execute(""" + SELECT * FROM image_jobs WHERE state='processing' AND lease_until <= clock_timestamp() + ORDER BY lease_until LIMIT 1 FOR NO KEY UPDATE SKIP LOCKED + """).fetchone() + if job: + fail_locked(conn, job, 'lease_expired') + return job is not None diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..b55b0d93 --- /dev/null +++ b/src/main.py @@ -0,0 +1,519 @@ +"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили.""" + +import base64 +import binascii +import json +from pathlib import Path +import logging +import re +from uuid import UUID, uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + + + +from psycopg import OperationalError, Error as DatabaseError +from psycopg_pool import PoolTimeout + +from .auth import ( + create_token, + format_datetime, + hash_password, + hash_token, + parse_basic, + parse_bearer, + session_expiration, + verify_password, +) +from .database import open_database, close_database, get_connection +from .user_repository import ( + create_user as insert_user, + create_session, + find_session_user_cached, + find_user_by_username, + revoke_session, + serialize_user, +) + +from .chat_repository import ( + create_chat as insert_chat, + get_chat as find_chat, + list_chats as find_chats, + create_message as insert_message, + list_messages as find_messages, +) + +app = FastAPI(title="MTUSI Messenger", version="1.5.0") +# Serve the mandatory contract in Swagger, including auth and request schemas. +OPENAPI_SCHEMA = json.loads( + (Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text() +) +app.openapi = lambda: OPENAPI_SCHEMA + +INSTANCE_ID = str(uuid4()) + +logger = logging.getLogger("uvicorn.error") + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 500: "internal_error", + 503: "unavailable", +} + + +def error_response(request, status_code, code, message): + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request.state.request_id, + } + }, + ) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + response = await call_next(request) + except Exception: + # Логируем путь, метод и request_id, но НЕ тело сообщения. + await run_in_threadpool( + logger.error, + "Unhandled error: %s %s request_id=%s", + request.method, + request.url.path, + request.state.request_id, + ) + response = error_response( + request, 500, "internal_error", "Внутренняя ошибка сервера" + ) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(request: Request, exc: StarletteHTTPException): + response = error_response( + request, + exc.status_code, + ERROR_CODES.get(exc.status_code, "invalid_request"), + str(exc.detail), + ) + if exc.headers: + response.headers.update(exc.headers) + return response + + +@app.exception_handler(DatabaseError) +@app.exception_handler(PoolTimeout) +async def database_error_handler(request: Request, exc): + return error_response(request, 503, "unavailable", "База данных временно недоступна") + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Некорректные данные запроса") + + +async def read_json(request: Request): + """415 — только для неправильного Content-Type; 400 — для неверного JSON.""" + content_type = ( + request.headers.get("content-type", "").split(";")[0].strip().lower() + ) + if content_type != "application/json": + raise HTTPException(415, "Неподдерживаемый Content-Type") + + try: + raw = bytearray() + async for chunk in request.stream(): + if len(raw) + len(chunk) > 65536: + raise HTTPException(413, "JSON слишком большой") + raw.extend(chunk) + data = json.loads(raw) + except (ValueError, UnicodeError): + raise HTTPException(400, "Неверный JSON") from None + + # [], строка, число, true, false, null — это корректный JSON, + # но НЕ объект, обязательный по контракту. + if not isinstance(data, dict): + raise HTTPException(400, "Ожидается JSON-объект") + return data + + +def get_current_user(request: Request): + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise HTTPException( + 401, + "Требуется действующий Bearer-токен", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + try: + user = find_session_user_cached(hash_token(token)) + except (OperationalError, PoolTimeout): + raise HTTPException( + 503, + "База данных временно недоступна", + ) from None + if user is None: + raise HTTPException( + 401, + "Недействительная или истёкшая сессия", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + return user + + +def get_limit(request: Request): + """Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500.""" + limit_raw = request.query_params.get("limit", "50") + if not limit_raw.isascii() or not limit_raw.isdecimal(): + raise HTTPException(400, "Неверный limit") + + significant = limit_raw.lstrip("0") + if not significant or len(significant) > 3: + raise HTTPException(400, "Неверный limit") + + limit = int(significant) # Теперь строка содержит максимум 3 цифры. + if not 1 <= limit <= 100: + raise HTTPException(400, "limit должен быть от 1 до 100") + return limit + + +def encode_cursor(first_id, last_id): + raw = f"{first_id}|{last_id}".encode("ascii") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor): + try: + if not cursor or len(cursor) > 256: + raise ValueError("empty or too long") + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.b64decode( + padded, altchars=b"-_", validate=True + ).decode("ascii") + first_id, last_id = raw.split("|") + return str(UUID(first_id)), str(UUID(last_id)) + except (ValueError, UnicodeError, binascii.Error): + raise HTTPException(400, "Неверный cursor") from None + + +# ---- Health ---- + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + try: + with get_connection() as conn: + conn.execute("SELECT 1") + from .storage import s3, BUCKET + s3.list_objects_v2(Bucket=BUCKET, Prefix="originals/", MaxKeys=1) + except (OperationalError, PoolTimeout): + raise HTTPException(503, "База данных недоступна") from None + return {"status": "ok"} + + +# ---- Users ---- + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await read_json(request) + + if set(data) != {"username", "display_name", "password"}: + raise HTTPException(400, "Неверные поля пользователя") + + username = data["username"] + display_name = data["display_name"] + password = data["password"] + + if not isinstance(username, str) or not re.fullmatch( + r"[a-z0-9_]{3,32}", username + ): + raise HTTPException(400, "Недопустимый username") + + if ( + not isinstance(display_name, str) + or not 1 <= len(display_name) <= 100 + or not display_name.strip() + ): + raise HTTPException(400, "Недопустимое имя пользователя") + + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise HTTPException(400, "Пароль должен содержать 12–128 символов") + + password_hash = await run_in_threadpool(hash_password, password) + user = await run_in_threadpool(insert_user, username, display_name, password_hash) + if user is None: + raise HTTPException(409, "Username уже занят") + + return user + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return get_current_user(request) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(request: Request): + credentials = parse_basic(request.headers.get("Authorization")) + + if credentials is None: + raise HTTPException( + 401, + "Неверные учётные данные", + headers={ + "WWW-Authenticate": + 'Basic realm="messenger", charset="UTF-8"' + }, + ) + + username, password = credentials + user_row = find_user_by_username(username) + + if user_row is None or not verify_password( + user_row["password_hash"], password + ): + raise HTTPException( + 401, + "Неверные учётные данные", + headers={ + "WWW-Authenticate": + 'Basic realm="messenger", charset="UTF-8"' + }, + ) + + token = create_token() + expires_at = session_expiration() + + create_session(user_row["id"], hash_token(token), expires_at) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": format_datetime(expires_at), + "user": serialize_user(user_row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + get_current_user(request) + token = parse_bearer(request.headers.get("Authorization")) + + from .session_cache import invalidate_session + + if not revoke_session(hash_token(token)): + raise HTTPException( + 401, + "Недействительная сессия", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + invalidate_session(hash_token(token)) + + from fastapi import Response + return Response(status_code=204) + + + +# ---- Chats ---- + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await run_in_threadpool(get_current_user, request) + data = await read_json(request) + + if set(data) != {"title", "member_ids"}: + raise HTTPException(400, "Неверные поля чата") + + title = data["title"] + member_ids = data["member_ids"] + + if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip(): + raise HTTPException(400, "Недопустимое название чата") + + if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100: + raise HTTPException(400, "Неверный список участников") + + normalized_ids = [] + + for member_id in member_ids: + if not isinstance(member_id, str): + raise HTTPException(400, "Неверный ID участника") + + try: + normalized_ids.append(str(UUID(member_id))) + except ValueError: + raise HTTPException(400, "Неверный UUID участника") from None + + if len(normalized_ids) != len(set(normalized_ids)): + raise HTTPException(400, "Участники повторяются") + + if user["id"] not in normalized_ids: + raise HTTPException(400, "Создатель должен быть участником") + + chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids) + + if chat is None: + raise HTTPException(400, "Неизвестный участник") + + return chat + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + user = get_current_user(request) + limit = get_limit(request) + cursor = request.query_params.get("cursor") + + last_chat_id = None + + if cursor is not None: + cursor_user_id, last_chat_id = decode_cursor(cursor) + + if cursor_user_id != user["id"]: + raise HTTPException(400, "Неверный cursor") + + rows = find_chats(user["id"], limit, last_chat_id) + + if rows is None: + raise HTTPException(400, "Неверный cursor") + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + encode_cursor(user["id"], page[-1]["id"]) + if has_more and page + else None + ) + + return {"items": page, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + user = get_current_user(request) + + try: + chat_id = str(UUID(chat_id)) + except ValueError: + raise HTTPException(400, "Неверный UUID чата") from None + + chat = find_chat(chat_id, user["id"]) + + if chat is None: + raise HTTPException(404, "Чат не найден") + + return chat + + +# ---- Messages ---- + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, request: Request): + user = await run_in_threadpool(get_current_user, request) + chat = await run_in_threadpool(get_chat, chat_id, request) + data = await read_json(request) + + if "text" not in data or set(data) - {"text", "attachment_ids"}: + raise HTTPException(400, "Неверные поля сообщения") + + text = data["text"] + + attachment_ids = data.get("attachment_ids", []) + if not isinstance(attachment_ids, list) or len(attachment_ids) > 10: + raise HTTPException(400, "Неверный список вложений") + try: + if any(not isinstance(a, str) for a in attachment_ids): + raise ValueError() + attachment_ids = [str(UUID(a)) for a in attachment_ids] + if len(set(attachment_ids)) != len(attachment_ids): + raise ValueError() + except ValueError: + raise HTTPException(400, "Неверные ID вложений") from None + if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids): + raise HTTPException(400, "Недопустимый текст сообщения") + + key = request.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400, "Неверный Idempotency-Key") + message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key, attachment_ids) + if message == "attachment_failed": + raise HTTPException(409, "Обработка вложения завершилась ошибкой") + if message == "conflict": + raise HTTPException(409, "Ключ использован для другого сообщения") + + if message is None: + raise HTTPException(404, "Чат не найден") + + return message + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + chat = get_chat(chat_id, request) + limit = get_limit(request) + cursor = request.query_params.get("cursor") + + last_message_id = None + + if cursor is not None: + cursor_chat_id, last_message_id = decode_cursor(cursor) + + if cursor_chat_id != chat["id"]: + raise HTTPException(400, "Неверный cursor") + + rows = find_messages(chat["id"], limit, last_message_id) + + if rows is None: + raise HTTPException(400, "Неверный cursor") + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + encode_cursor(chat["id"], page[-1]["id"]) + if has_more and page + else None + ) + + return {"items": page, "next_cursor": next_cursor} + + +@app.on_event("startup") +async def start_database_pool(): + await run_in_threadpool(open_database) + + +@app.on_event("shutdown") +async def stop_database_pool(): + await run_in_threadpool(close_database) + +from .attachments import register +register(app, get_current_user, get_chat) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..b98e6faf --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,16 @@ +"""One-shot migrations, serialized and recorded in PostgreSQL.""" +from pathlib import Path +import psycopg +from .database import get_database_url + +with psycopg.connect(get_database_url()) as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(310026)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())") + for path in sorted((Path(__file__).resolve().parents[1] / "db/migrations").glob("*.sql")): + if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,)) + print("Applied", path.name, flush=True) + else: + print("Already applied", path.name, flush=True) diff --git a/src/operator.py b/src/operator.py new file mode 100644 index 00000000..3c3cbcbb --- /dev/null +++ b/src/operator.py @@ -0,0 +1,91 @@ +"""Local administrative commands. Run from the trusted Compose project.""" +import argparse +import json + +from .broker import channel, send +from .database import close_database, get_connection, open_database +from .jobs import carrier, enqueue, event, schedule + + +def reprocess(attachment_id, version): + with get_connection() as conn: + row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE', (attachment_id,)).fetchone() + if not row or version <= row['desired_version']: + raise ValueError('Version must increase for an existing attachment') + if row['active_version'] is None and row['staging_key'] is None and row['object_key'].startswith('staging/'): + raise ValueError('Original expired') + conn.execute(""" + UPDATE attachments SET desired_version=%s, + status=CASE WHEN active_version IS NULL THEN 'queued' ELSE status END, + error_code=NULL,updated_at=clock_timestamp() WHERE id=%s + """, (version, attachment_id)) + job = enqueue(conn, attachment_id, version) + event(conn, job, 'reprocess_requested') + return carrier(job) + + +def replay(job_id): + with get_connection() as conn: + row = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE', (job_id,)).fetchone() + if not row or row['state'] != 'failed': + raise ValueError('Replay requires a failed job') + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE', + (row['attachment_id'],)).fetchone() + if attachment['desired_version'] != row['pipeline_version']: + raise ValueError('Replay requires the current pipeline version') + if attachment['staging_key'] is None and attachment['object_key'].startswith('staging/'): + raise ValueError('Staging retention expired') + conn.execute(""" + UPDATE job_dead_letters SET replayed_at=clock_timestamp() WHERE job_id=%s AND attempt=%s + """, (job_id, row['attempt'])) + updated = conn.execute(""" + UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=0, + error_code=NULL WHERE id=%s RETURNING * + """, (job_id,)).fetchone() + conn.execute(""" + UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() + WHERE id=%s AND active_version IS NULL + """, (row['attachment_id'],)) + schedule(conn, job_id) + event(conn, updated, 'operator_replay') + return carrier(updated) + + +def redeliver(job_id): + with get_connection() as conn: + row = conn.execute('SELECT * FROM image_jobs WHERE id=%s', (job_id,)).fetchone() + if not row: + raise ValueError('Job not found') + message = carrier(row) + with channel() as (_, ch): + ch.confirm_delivery() + send(ch, message) + return message + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + new = commands.add_parser('reprocess') + new.add_argument('attachment_id') + new.add_argument('version', type=int, choices=[1, 2]) + commands.add_parser('replay').add_argument('job_id') + commands.add_parser('redeliver').add_argument('job_id') + args = parser.parse_args() + open_database() + try: + if args.command == 'reprocess': + result = reprocess(args.attachment_id, args.version) + elif args.command == 'replay': + result = replay(args.job_id) + else: + result = redeliver(args.job_id) + print(json.dumps(result)) + except ValueError as exc: + parser.exit(2, f'{exc}\n') + finally: + close_database() + + +if __name__ == '__main__': + main() diff --git a/src/pipeline_health.py b/src/pipeline_health.py new file mode 100644 index 00000000..59d41f2d --- /dev/null +++ b/src/pipeline_health.py @@ -0,0 +1,29 @@ +"""Check recent progress by the service loop in this container.""" +from pathlib import Path +import time + +PROGRESS = Path('/tmp/pipeline-progress') +MAX_IDLE_SECONDS = 60 +_last_write = float('-inf') + + +def mark_progress(): + global _last_write + now = time.monotonic() + if now - _last_write >= 1: + PROGRESS.touch() + _last_write = now + + +def clear_progress(): + global _last_write + PROGRESS.unlink(missing_ok=True) + _last_write = float('-inf') + + +if __name__ == '__main__': + try: + healthy = time.time() - PROGRESS.stat().st_mtime < MAX_IDLE_SECONDS + except FileNotFoundError: + healthy = False + raise SystemExit(0 if healthy else 1) diff --git a/src/process.py b/src/process.py new file mode 100644 index 00000000..022d5551 --- /dev/null +++ b/src/process.py @@ -0,0 +1,31 @@ +import logging +import signal +from threading import Event + +import pika +from psycopg import OperationalError +from psycopg_pool import PoolTimeout +from botocore.exceptions import BotoCoreError, ClientError + +from .pipeline_health import mark_progress, clear_progress +from .database import open_database, close_database + + +def serve(loop): + logging.basicConfig(level=logging.WARNING, format='%(asctime)s %(levelname)s %(message)s') + stopping = Event() + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: stopping.set()) + clear_progress() + open_database() + try: + while not stopping.is_set(): + try: + loop(stopping, mark_progress) + except (OperationalError, PoolTimeout, BotoCoreError, ClientError, pika.exceptions.AMQPError, OSError) as exc: + clear_progress() + logging.warning('dependency failure: %s', type(exc).__name__) + stopping.wait(1) + finally: + clear_progress() + close_database() diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 00000000..dbb3e751 --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,50 @@ +"""Drain committed outbox rows, deleting only after a positive broker confirm.""" +import logging +import os + +from .broker import channel, send +from .database import get_connection +from .faults import take +from .jobs import carrier, event +from .process import serve + + +def publish_one(ch): + with get_connection() as conn: + job = conn.execute(""" + SELECT j.* FROM job_outbox o JOIN image_jobs j ON j.id=o.job_id + WHERE o.publish_after <= clock_timestamp() + ORDER BY o.publish_after LIMIT 1 FOR UPDATE OF o SKIP LOCKED + """).fetchone() + if job is None: + return False + if job['state'] != 'queued': + conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],)) + return True + args = (job['attachment_id'], job['pipeline_version']) + crash_before = take(conn, *args, 'before_publish') == 'crash' + if not crash_before: + send(ch, carrier(job)) + crash_after = take(conn, *args, 'after_confirm') == 'crash' + if not crash_after: + conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],)) + event(conn, job, 'broker_confirmed') + if crash_before or crash_after: + point = 'before_publish' if crash_before else 'after_confirm' + logging.warning('fault %s: crash %s', point, job['id']) + os._exit(86) + return True + + +def loop(stopping, progress): + with channel() as (connection, ch): + ch.confirm_delivery() + while not stopping.is_set(): + if not publish_one(ch): + connection.process_data_events(time_limit=0.2) + stopping.wait(0.1) + progress() + + +if __name__ == '__main__': + serve(loop) diff --git a/src/recovery.py b/src/recovery.py new file mode 100644 index 00000000..849cb0aa --- /dev/null +++ b/src/recovery.py @@ -0,0 +1,50 @@ +"""Reclaim expired jobs and remove staging after safe publication or retention.""" +from datetime import datetime, timedelta, timezone +import os +import time + +from .database import get_connection +from .jobs import recover_one +from .process import serve +from .storage import BUCKET, s3 + +RETENTION = int(os.getenv('STAGING_RETENTION_SECONDS', '86400')) + + +def cleanup(): + cutoff = datetime.now(timezone.utc) - timedelta(seconds=RETENTION) + for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='staging/'): + for obj in page.get('Contents', []): + key = obj['Key'] + with get_connection() as conn: + locked = conn.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS ok', + (key,)).fetchone()['ok'] + if not locked: + continue + row = conn.execute(""" + SELECT id,status,active_version,updated_at FROM attachments WHERE staging_key=%s FOR UPDATE + """, (key,)).fetchone() + if row: + if not (row['active_version'] or (row['status'] == 'failed' and row['updated_at'] < cutoff)): + continue + elif obj['LastModified'] >= cutoff: + continue + s3.delete_object(Bucket=BUCKET, Key=key) + if row: + conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s', (row['id'],)) + + +def loop(stopping, progress): + next_cleanup = 0 + while not stopping.is_set(): + while not stopping.is_set() and recover_one(): + progress() + if time.monotonic() >= next_cleanup: + cleanup() + next_cleanup = time.monotonic() + 15 + progress() + stopping.wait(0.5) + + +if __name__ == '__main__': + serve(loop) diff --git a/src/render_image.py b/src/render_image.py new file mode 100644 index 00000000..8f262be9 --- /dev/null +++ b/src/render_image.py @@ -0,0 +1,49 @@ +"""Bounded subprocess for untrusted image decoding and resizing.""" +import resource +import sys +import warnings +from pathlib import Path + +from PIL import Image, ImageOps + +# Linux runtime. macOS does not implement RLIMIT_AS with the same semantics. +if sys.platform.startswith('linux'): + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2)) +resource.setrlimit(resource.RLIMIT_CPU, (15, 16)) +Image.MAX_IMAGE_PIXELS = 40_000_000 +warnings.simplefilter('error', Image.DecompressionBombWarning) + + +def fit(image, bound): + width, height = image.size + largest = max(width, height) + if largest <= bound: + return image + return image.resize((max(1, width * bound // largest), max(1, height * bound // largest)), + Image.Resampling.LANCZOS) + + +def render(directory, version): + with Image.open(directory / 'original') as source: + image = ImageOps.exif_transpose(source) + if 'A' in image.getbands() or 'transparency' in image.info: + rgba = image.convert('RGBA') + del image + image = Image.new('RGB', rgba.size, 'white') + image.paste(rgba, mask=rgba.getchannel('A')) + del rgba + else: + image = image.convert('RGB') + image.info.clear() + jpeg_quality, webp_quality = {1: (85, 82), 2: (90, 88)}[version] + fit(image, 256).save(directory / 'thumbnail', format='JPEG', quality=jpeg_quality) + ImageOps.fit(image, (128, 128), Image.Resampling.LANCZOS).save( + directory / 'crop', format='JPEG', quality=jpeg_quality) + fit(image, 1280).save(directory / 'optimized', format='WEBP', quality=webp_quality, method=4) + + +if __name__ == '__main__': + try: + render(Path(sys.argv[1]), int(sys.argv[2])) + except (OSError, ValueError, Image.DecompressionBombError, Image.DecompressionBombWarning): + sys.exit(15) diff --git a/src/session_cache.py b/src/session_cache.py new file mode 100644 index 00000000..766a1a77 --- /dev/null +++ b/src/session_cache.py @@ -0,0 +1,48 @@ +import json +import os +from pathlib import Path + +import redis + + +def get_cache_url(): + secret_file = os.getenv("CACHE_URL_FILE") + if secret_file: + return Path(secret_file).read_text().strip() + return os.getenv("CACHE_URL", "redis://redis:6379/0") + + +client = redis.Redis.from_url( + get_cache_url(), + socket_connect_timeout=1, + socket_timeout=1, + decode_responses=True, +) + + +def get_cached_user(token_hash): + try: + value = client.get("session:" + token_hash) + return json.loads(value) if value else None + except (redis.RedisError, ValueError): + return None + + +def cache_user(token_hash, user, ttl): + if ttl <= 0: + return + try: + client.setex( + "session:" + token_hash, + ttl, + json.dumps(user), + ) + except redis.RedisError: + pass + + +def invalidate_session(token_hash): + try: + client.delete("session:" + token_hash) + except redis.RedisError: + pass diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..a00b1bf5 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,19 @@ +"""Private S3 access; no automatic PUT retries, explicit bounded timeouts.""" +import os +from pathlib import Path +import boto3 +from botocore.config import Config + + +def secret(name): + return Path(os.environ[name + '_FILE']).read_text().strip() + + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +s3 = boto3.client( + 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'), + aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'), + config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}), +) diff --git a/src/user_repository.py b/src/user_repository.py new file mode 100644 index 00000000..13d732b7 --- /dev/null +++ b/src/user_repository.py @@ -0,0 +1,136 @@ +from uuid import uuid4 + +from psycopg.errors import UniqueViolation + +from .database import get_connection + + +def serialize_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def create_user(username, display_name, password_hash): + with get_connection() as conn: + with conn.cursor() as cur: + try: + cur.execute( + """ + INSERT INTO users ( + id, username, display_name, password_hash + ) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ) + return serialize_user(cur.fetchone()) + except UniqueViolation: + conn.rollback() + return None + + +def find_user_by_username(username): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, username, display_name, + password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ) + return cur.fetchone() + + +def create_session(user_id, token_hash, expires_at): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO sessions ( + id, user_id, token_hash, expires_at + ) + VALUES (%s, %s, %s, %s) + """, + (uuid4(), user_id, token_hash, expires_at), + ) + + +def revoke_session(token_hash): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE sessions + SET revoked_at = NOW() + WHERE token_hash = %s + AND revoked_at IS NULL + AND expires_at > NOW() + RETURNING id + """, + (token_hash,), + ) + return cur.fetchone() is not None + + +def find_session_user_cached(token_hash): + from datetime import datetime, timezone + from .session_cache import cache_user, get_cached_user + + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + s.user_id, + s.expires_at + FROM sessions s + WHERE s.token_hash = %s + AND s.revoked_at IS NULL + AND s.expires_at > NOW() + """, + (token_hash,), + ) + session = cur.fetchone() + + if session is None: + return None + + cached = get_cached_user(token_hash) + + if cached is not None and str(cached.get("id")) == str(session["user_id"]): + return cached + + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (session["user_id"],), + ) + row = cur.fetchone() + + if row is None: + return None + + user = serialize_user(row) + ttl = int( + (session["expires_at"] - datetime.now(timezone.utc)).total_seconds() + ) + + if ttl > 0: + cache_user(token_hash, user, ttl) + + return user diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..1125f1ec --- /dev/null +++ b/src/worker.py @@ -0,0 +1,124 @@ +"""One in-flight delivery per worker process. ACK follows the durable decision.""" +from datetime import datetime +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +from uuid import UUID + +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image + +from .broker import QUEUE, channel +from .faults import checkpoint +from .jobs import claim, fail, finish +from .process import serve +from .storage import BUCKET, s3 + +VARIANTS = {'thumbnail': 'image/jpeg', 'crop': 'image/jpeg', 'optimized': 'image/webp'} + + +class PermanentError(Exception): + pass + + +def original(attachment): + key = attachment['object_key'] + response = s3.get_object(Bucket=BUCKET, Key=key) + body = response['Body'] + try: + data = body.read(10 * 1024**2 + 1) + finally: + body.close() + if (len(data) != attachment['size_bytes'] + or hashlib.sha256(data).hexdigest() != attachment['sha256']): + raise PermanentError('source_checksum_mismatch') + return data + + +def put(job, variant, data, mime, dimensions=None): + checksum = hashlib.sha256(data).hexdigest() + key = f'results/{job["attachment_id"]}/v{job["pipeline_version"]}/{variant}/{checksum}' + s3.put_object(Bucket=BUCKET, Key=key, Body=data, ContentType=mime) + entry = dict(key=key, sha256=checksum, size_bytes=len(data), content_type=mime) + if dimensions: + entry.update(width=dimensions[0], height=dimensions[1]) + return entry + + +def process(job, attachment): + data = original(attachment) + with tempfile.TemporaryDirectory(prefix='pipeline-') as target: + directory = Path(target) + (directory / 'original').write_bytes(data) + proc = subprocess.run([sys.executable, '-m', 'src.render_image', target, + str(job['pipeline_version'])], timeout=20, check=False) + if proc.returncode: + raise PermanentError('invalid_image' if proc.returncode == 15 else 'processing_limit') + action = checkpoint(job, 'before_s3') + if action == 'permanent': + raise PermanentError('invalid_image') + if action == 'temporary': + raise OSError('injected transient error') + manifest = {'original': put(job, 'original', data, attachment['content_type'])} + for variant, mime in VARIANTS.items(): + path = directory / variant + with Image.open(path) as image: + dimensions = image.size + manifest[variant] = put(job, variant, path.read_bytes(), mime, dimensions) + checkpoint(job, 'after_s3') + return manifest + + +def handle(ch, method, body): + try: + message = json.loads(body) + if (type(message['schema_version']) is not int or message['schema_version'] != 1 + or message['type'] != QUEUE + or any(type(message[field]) is not int or message[field] < 1 + for field in ('pipeline_version', 'attempt'))): + raise ValueError('Invalid carrier') + if datetime.fromisoformat(message['created_at']).tzinfo is None: + raise ValueError('Missing UTC offset') + for field in ('job_id', 'attachment_id'): + message[field] = UUID(message[field]) + except (ValueError, TypeError, KeyError, AttributeError): + ch.basic_reject(method.delivery_tag, requeue=False) + return + claimed = claim(message) + if claimed: + job, attachment = claimed + try: + manifest = process(job, attachment) + except PermanentError as exc: + fail(job, str(exc), permanent=True) + except subprocess.TimeoutExpired: + fail(job, 'processing_limit', permanent=True) + except ClientError as exc: + code = exc.response['Error']['Code'] + fail(job, 'source_missing' if code in ('NoSuchKey', '404') else 'storage_unavailable', + permanent=code in ('NoSuchKey', '404')) + except (BotoCoreError, OSError): + fail(job, 'temporary_failure') + else: + finish(job, manifest) + checkpoint(job, 'after_commit') + ch.basic_ack(method.delivery_tag) + + +def loop(stopping, progress): + with channel() as (connection, ch): + ch.basic_qos(prefetch_count=1) + while not stopping.is_set(): + method, _, body = ch.basic_get(QUEUE, auto_ack=False) + if method: + handle(ch, method, body) + else: + connection.process_data_events(time_limit=0.2) + progress() + + +if __name__ == '__main__': + serve(loop) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..2d7f0976 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Destructive fault injection for a disposable, dedicated LAB5 Compose project.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check +from image_probe import jpeg_size + +COMPOSE = str(ROOT / 'scripts/compose.sh') + + +def compose(*args): + return subprocess.run([COMPOSE, *args], check=True, text=True, capture_output=True, timeout=120).stdout.strip() + + +def sql(query): + return compose('exec', '-T', 'postgres', 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1', + '-U', 'messenger', '-d', 'messenger', '-c', query) + + +def job(attachment, version=1): + return json.loads(sql(f"""SELECT row_to_json(j) FROM image_jobs j + WHERE attachment_id='{attachment['id']}' AND pipeline_version={version}""")) + + +def events(job_id, name): + return int(sql(f"SELECT count(*) FROM job_events WHERE job_id='{job_id}' AND event='{name}'")) + + +def operator(command, identifier, *args): + return json.loads(compose('exec', '-T', 'recovery', 'python', '-m', 'src.operator', command, + identifier, *args)) + + +def wait(predicate, description, timeout=105): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.2) + raise AssertionError(f'timeout: {description}') + + +def arm(attachment, point, action, version=1): + sql(f"""INSERT INTO pipeline_faults(attachment_id,pipeline_version,point,action) + VALUES ('{attachment['id']}',{version},'{point}','{action}')""") + + +def hit(attachment, point, release=False): + scope = f"attachment_id='{attachment['id']}' AND pipeline_version=1 AND point='{point}'" + wait(lambda: sql(f'SELECT hit_at IS NOT NULL FROM pipeline_faults WHERE {scope}') == 't', point) + if release: + sql(f'UPDATE pipeline_faults SET released=true WHERE {scope}') + + +def objects(attachment, version): + prefix = f"results/{attachment['id']}/v{version}/" + code = ("import json; from src.storage import s3,BUCKET; " + f"print(json.dumps(sorted(x['Key'] for x in s3.list_objects_v2(Bucket=BUCKET,Prefix={prefix!r})" + ".get('Contents',[]))))") + return json.loads(compose('exec', '-T', 'recovery', 'python', '-c', code)) + + +def report(name, started, attachment, **metrics): + print(json.dumps(dict(scenario=name, result='PASS', seconds=round(time.monotonic() - started, 3), + attachment_id=attachment['id'], **metrics), ensure_ascii=False), flush=True) + + +def service_health(service): + return [json.loads(line)['Health'] for line in + compose('ps', '--format', 'json', service).splitlines()] + + +def experiments(client): + alice, bob, eve, chat = client.bootstrap() + worker_count = compose('ps', '-q', 'worker').splitlines() + check(len(worker_count) >= 2, 'two workers are required') + for service in ('publisher', 'worker'): + enabled = compose('exec', '-T', service, 'printenv', 'PIPELINE_FAULTS') + check(enabled == '1', f'fault injection disabled on {service}') + + def upload(): + return client.attachment(alice, chat) + + def meta(attachment, status=200): + return client.call('GET', '/api/v1/attachments/' + attachment['id'], status, + actor=bob, schema='Attachment')['json'] + + def verify(attachment, content, version=1): + wait(lambda: job(attachment, version)['state'] == 'ready', f'job v{version} ready') + ready = meta(attachment) + check(ready['status'] == 'ready', 'attachment is not ready') + row = job(attachment, version) + active = int(sql(f"SELECT active_version FROM attachments WHERE id='{attachment['id']}'")) + check(active == version, 'wrong active version') + check(events(row['id'], 'published') == 1, 'expected one published manifest') + check(set(row['manifest']) == {'original', 'thumbnail', 'crop', 'optimized'}, 'manifest incomplete') + check(set(objects(attachment, version)) == {v['key'] for v in row['manifest'].values()}, + 'unexpected keys or duplicated side effects') + path = '/api/v1/attachments/' + attachment['id'] + '/content?variant=' + check({v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'}, 'wrong API variants') + for variant, entry in row['manifest'].items(): + downloaded = client.call('GET', path + variant, actor=bob) + data = downloaded['bytes'] + check(downloaded['headers'].get_content_type() == entry['content_type'], 'wrong MIME') + check(len(data) == entry['size_bytes'], 'wrong size') + check(hashlib.sha256(data).hexdigest() == entry['sha256'], 'wrong checksum') + if variant == 'original': + check(data == content, 'original changed') + elif variant == 'optimized': + check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature') + check((entry['width'], entry['height']) == (320, 200), 'WebP must not be enlarged') + else: + expected = (256, 160) if variant == 'thumbnail' else (128, 128) + check(jpeg_size(data) == expected, 'wrong JPEG dimensions') + return row + + started = time.monotonic() + key = 'concurrent-message-' + chat['id'] + def post_message(_): + return client.message(alice, chat, 'concurrent', key=key) + with ThreadPoolExecutor(max_workers=6) as executor: + responses = list(executor.map(post_message, range(6))) + check(len({response['json']['id'] for response in responses}) == 1, 'concurrent writes created duplicates') + check(len({response['headers']['X-Instance-Id'] for response in responses}) >= 2, + 'concurrent requests did not reach two API instances') + report('concurrent_message', started, {'id': None}, requests=6) + + started = time.monotonic() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png' + fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes() + client.call('POST', path, 404, actor=eve, raw=fixture, headers={'Content-Type': 'image/png'}) + client.call('POST', path, 415, actor=alice, raw=fixture, headers={'Content-Type': 'image/jpeg'}) + client.call('POST', path, 415, actor=alice, raw=b'not a png', headers={'Content-Type': 'image/png'}) + client.call('POST', path, 413, actor=alice, raw=b'', + headers={'Content-Type': 'image/png', 'Content-Length': '10485761'}) + report('upload_validation', started, {'id': None}, statuses=[404, 415, 415, 413]) + + started = time.monotonic() + compose('stop', '-t', '1', 'broker') + try: + attachment, content = upload() + row = job(attachment) + check(meta(attachment)['status'] == 'queued', 'upload did not remain queued') + check(sql(f"SELECT count(*) FROM job_outbox WHERE job_id='{row['id']}'") == '1', + 'lost intent during real broker outage') + wait(lambda: service_health('publisher') == ['unhealthy'] + and service_health('worker') == ['unhealthy'] * len(worker_count), + 'AMQP outage reflected in publisher and worker health', timeout=45) + finally: + compose('start', 'broker') + verify(attachment, content) + wait(lambda: service_health('publisher') == ['healthy'] + and service_health('worker') == ['healthy'] * len(worker_count), + 'publisher and all workers became healthy again', timeout=45) + report('broker_unavailable', started, attachment, attempt=1, pipeline_health_recovered=True) + + for point in ('before_publish', 'after_confirm', 'before_s3', 'after_s3', 'after_commit'): + started = time.monotonic() + compose('stop', '-t', '1', 'publisher') + if point in ('before_publish', 'after_confirm'): + compose('stop', '-t', '1', 'worker') + attachment, content = upload() + arm(attachment, point, 'crash') + compose('start', 'publisher') + hit(attachment, point) + if point in ('before_publish', 'after_confirm'): + job_id = job(attachment)['id'] + wait(lambda: events(job_id, 'broker_confirmed'), 'outbox drained after publisher crash') + compose('start', 'worker') + if point in ('before_s3', 'after_s3'): + keys = objects(attachment, 1) + check(len(keys) == (0 if point == 'before_s3' else 4), 'wrong S3 effects at crash') + check(job(attachment)['state'] == 'processing', 'manifest was already committed at crash') + row = verify(attachment, content) + wanted = {'before_s3': 2, 'after_s3': 2}.get(point, 1) + check(row['attempt'] == wanted, f'wrong attempt after {point}') + check(f'fault {point}: crash {row["id"]}' in compose('logs', '--no-color', 'publisher', 'worker'), + 'missing crash log') + if point in ('after_commit', 'after_confirm'): + wait(lambda: events(row['id'], 'delivery_ignored'), 'duplicate delivery ignored') + report(point, started, attachment, attempt=row['attempt'], published=1) + + started = time.monotonic() + compose('stop', '-t', '1', 'publisher') + attachment, content = upload() + arm(attachment, 'before_s3', 'pause') + compose('start', 'publisher') + hit(attachment, 'before_s3') + initial = job(attachment) + operator('redeliver', initial['id']) + wait(lambda: events(initial['id'], 'delivery_ignored'), 'parallel duplicate rejected') + check(events(initial['id'], 'claimed') == 1, 'duplicate obtained live lease') + current = verify(attachment, content) + check(current['fence'] > initial['fence'] and current['attempt'] == 2, 'lease was not reclaimed') + hit(attachment, 'before_s3', release=True) + wait(lambda: events(initial['id'], 'fence_rejected'), 'old attempt fenced') + ignored = events(initial['id'], 'delivery_ignored') + operator('redeliver', initial['id']) + wait(lambda: events(initial['id'], 'delivery_ignored') > ignored, 'completed duplicate rejected') + check(verify(attachment, content)['manifest'] == current['manifest'], 'duplicate changed the manifest') + report('duplicate_and_fencing', started, attachment, fence=current['fence'], attempt=current['attempt']) + + started = time.monotonic() + compose('stop', '-t', '1', 'publisher') + attachment, content = upload() + arm(attachment, 'after_s3', 'pause') + compose('start', 'publisher') + hit(attachment, 'after_s3') + operator('reprocess', attachment['id'], '2') + second = verify(attachment, content, 2) + hit(attachment, 'after_s3', release=True) + old = job(attachment) + wait(lambda: job(attachment)['state'] == 'superseded' or events(old['id'], 'fence_rejected'), + 'v1 completed after v2 without publication') + ignored = events(old['id'], 'delivery_ignored') + operator('redeliver', old['id']) + wait(lambda: events(old['id'], 'delivery_ignored') > ignored, 'reordered v1 redelivery ignored') + check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'v1 changed v2') + check(events(old['id'], 'published') == 0, 'v1 published despite newer version') + report('reorder_in_flight', started, attachment, active_version=2) + + started = time.monotonic() + compose('stop', '-t', '1', 'publisher') + attachment, content = upload() + old = job(attachment) + second = operator('reprocess', attachment['id'], '2') + operator('redeliver', second['job_id']) + verify(attachment, content, 2) + check(job(attachment)['state'] == 'queued', 'v1 must still be queued before late delivery') + operator('redeliver', old['id']) + wait(lambda: events(old['id'], 'superseded') == 1, 'late v1 delivery superseded') + check(events(old['id'], 'claimed') == 0, 'late v1 obtained a lease') + compose('start', 'publisher') + check(not objects(attachment, 1), 'late v1 produced S3 side effects') + verify(attachment, content, 2) + report('late_old_delivery', started, attachment, active_version=2) + + started = time.monotonic() + attachment, content = upload() + first = verify(attachment, content) + compose('stop', '-t', '1', 'publisher') + try: + operator('reprocess', attachment['id'], '2') + check(meta(attachment)['status'] == 'ready', 'reprocessing hid the published version') + visible = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=thumbnail', + actor=bob)['bytes'] + check(hashlib.sha256(visible).hexdigest() == first['manifest']['thumbnail']['sha256'], + 'published version changed before v2 commit') + finally: + compose('start', 'publisher') + second = verify(attachment, content, 2) + for name in ('thumbnail', 'crop', 'optimized'): + check(first['manifest'][name]['sha256'] != second['manifest'][name]['sha256'], 'v2 did not change bytes') + operator('redeliver', first['id']) + wait(lambda: events(first['id'], 'delivery_ignored'), 'duplicate of published v1') + check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'old duplicate changed v2') + report('atomic_version_switch', started, attachment, active_version=2) + + for action in ('temporary', 'permanent'): + started = time.monotonic() + compose('stop', '-t', '1', 'publisher') + attachment, content = upload() + arm(attachment, 'before_s3', action) + compose('start', 'publisher') + initial = job(attachment) + if action == 'permanent': + wait(lambda: job(attachment)['state'] == 'failed', 'permanent failure') + check(meta(attachment)['error_code'] == 'invalid_image', 'unsafe or wrong error code') + path = '/api/v1/attachments/' + attachment['id'] + '/content' + client.call('GET', path, 409, actor=bob) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': '', 'attachment_ids': [attachment['id']]}, + headers={'Idempotency-Key': attachment['id']}) + check(sql(f"SELECT error_code FROM job_dead_letters WHERE job_id='{initial['id']}'") == 'invalid_image', + 'missing DLQ entry') + operator('replay', initial['id']) + check(sql(f"SELECT replayed_at IS NOT NULL FROM job_dead_letters WHERE job_id='{initial['id']}'") == 't', + 'missing replay audit') + completed = verify(attachment, content) + check(completed['attempt'] == 2, 'expected two attempts') + if action == 'temporary': + check(events(initial['id'], 'retry_scheduled') == 1, 'missing transient retry') + report(action + '_error', started, attachment, attempt=2) + + stable_attachment, stable_content = attachment, content + stable_manifest = completed['manifest'] + started = time.monotonic() + compose('stop', '-t', '1', 'worker') + attachment, content = upload() + row = job(attachment) + wait(lambda: events(row['id'], 'broker_confirmed'), 'durable queue entry') + check(meta(attachment)['status'] == 'queued', 'worker processed despite stop') + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob) + message = client.message(alice, chat, '', attachments=[attachment['id']])['json'] + compose('restart', 'postgres', 'redis', 's3', 'broker', 'api', 'lb', 'publisher', 'recovery') + wait(lambda: health(client), 'HAProxy readiness') + check(not compose('ps', '-q', 'worker'), 'restart brought workers back too early') + check(meta(attachment)['status'] == 'queued', 'confirmed job changed before worker restart') + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob) + compose('start', 'worker') + verify(attachment, content) + page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=bob, + schema='MessagePage')['json'] + check(any(item['id'] == message['id'] and attachment['id'] in item['attachment_ids'] + for item in page['items']), 'attachment link did not survive restart') + wait(lambda: sql(f"SELECT staging_key IS NULL FROM attachments WHERE id='{attachment['id']}'") == 't', + 'recovery cleaned staging after publication') + wait(lambda: service_health('recovery') == ['healthy'], 'recovery health after restart') + check(verify(stable_attachment, stable_content)['manifest'] == stable_manifest, + 'published attachment changed on restart') + check(job(attachment)['attempt'] == 1, 'broker restart lost the original delivery') + report('restart_with_queued_attachment', started, attachment, attempt=1, + message_link_preserved=True, staging_cleaned=True) + + +def health(client): + try: + client.call('GET', '/health/ready') + except (OSError, AssertionError): + return False + return True + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--disposable', action='store_true', required=True, + help='confirm this is an isolated test project') + args = parser.parse_args() + try: + experiments(Client(args.base_url)) + except (AssertionError, OSError, ValueError, subprocess.SubprocessError) as exc: + # Subprocess output can include internal details. Keep the public summary short. + print('FAIL:', str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/student-tests/render.py b/student-tests/render.py new file mode 100644 index 00000000..0b3a400a --- /dev/null +++ b/student-tests/render.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check image encoders, geometry, EXIF orientation and alpha on local fixtures.""" +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import zlib + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / 'tests/fixtures/sample.png' + + +def render(directory, version): + subprocess.run([sys.executable, '-m', 'src.render_image', str(directory), str(version)], + cwd=ROOT, check=True) + outputs = {} + for variant, format_name in [('thumbnail', 'JPEG'), ('crop', 'JPEG'), ('optimized', 'WEBP')]: + with Image.open(directory / variant) as image: + image.load() + assert image.format == format_name + assert not image.getexif() + outputs[variant] = (image.size, (directory / variant).read_bytes()) + return outputs + + +def validate(path, mime): + return subprocess.run([sys.executable, '-m', 'src.image_check', str(path), mime], + cwd=ROOT, timeout=10, check=False).returncode + + +def main(): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + original = directory / 'original' + data = FIXTURE.read_bytes() + original.write_bytes(data) + assert validate(original, 'image/png') == 0 + assert validate(original, 'image/jpeg') == 15 + original.write_bytes(b'corrupt') + assert validate(original, 'image/png') == 15 + # Rewrite only IHDR, with a valid CRC, to exercise the pre-decode limit. + header = data[12:16] + struct.pack('>II', 10000, 10000) + data[24:29] + original.write_bytes(data[:12] + header + struct.pack('>I', zlib.crc32(header)) + data[33:]) + assert validate(original, 'image/png') == 13 + original.write_bytes(data) + print('PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding') + first = render(directory, 1) + assert first == render(directory, 1) + second = render(directory, 2) + assert first.keys() == second.keys() + for variant, shape in [('thumbnail', (256, 160)), ('crop', (128, 128)), ('optimized', (320, 200))]: + assert first[variant][0] == second[variant][0] == shape + assert first[variant][1] != second[variant][1] + assert original.read_bytes() == data + print('PASS: fixture geometry, original preserved, deterministic v1, different v2') + transparent = Image.new('RGBA', (7, 3), (255, 0, 0, 0)) + transparent.save(original, format='PNG') + alpha = render(directory, 1) + assert alpha['thumbnail'][0] == (7, 3) + with Image.open(directory / 'thumbnail') as thumbnail: + assert thumbnail.getpixel((0, 0)) == (255, 255, 255) + rotated = Image.new('RGB', (17, 31), 'red') + exif = rotated.getexif() + exif[274] = 6 + rotated.save(original, format='JPEG', exif=exif) + assert render(directory, 1)['thumbnail'][0] == (31, 17) + print('PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed') + + +if __name__ == '__main__': + main()