ЛР 5 - БПИ2301 - Дугина - на 5 (трек 2) #103

Open
aleksandra_dugina wants to merge 1 commits from aleksandra_dugina/mtusi2026lab5:solution/BPI2301-Dugina into main
62 changed files with 3799 additions and 38 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
DEFENSE.md
+6 -20
View File
@@ -1,25 +1,11 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Generate the local .env and isolated secret files with scripts/init-local.py.
COMPOSE_PROJECT_NAME=mtusi2026lab5
PUBLIC_PORT=8080
BROKER_CONSOLE_PORT=15685
S3_CONSOLE_PORT=9005
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
PIPELINE_FAULTS=0
+44
View File
@@ -0,0 +1,44 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
integration:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
PIPELINE_FAULTS: '1'
JOB_TIMEOUT_SECONDS: '20'
COMPOSE_PROJECT_NAME: mtusi2026lab5-ci
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Build and wait for readiness
run: |
python scripts/init-local.py
./scripts/compose.sh up -d --build --wait --wait-timeout 150
curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready
- name: Mandatory smoke and recovery
run: |
set -o pipefail
make test 2>&1 | tee evidence/ci-smoke.txt
make test-persistence ACTION_SCRIPT=scripts/restart.sh 2>&1 | tee evidence/ci-persistence.txt
make test-failover ACTION_SCRIPT=scripts/stop-one.sh 2>&1 | tee evidence/ci-failover.txt
./scripts/compose.sh start api
curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready
- name: Track 2 experiments
run: |
set -o pipefail
python student-tests/pipeline.py --disposable 2>&1 | tee evidence/ci-pipeline.jsonl
- uses: actions/upload-artifact@v4
if: always()
with:
name: lab5-evidence
path: |
evidence/ci-*
evidence/restart-*.txt
- name: Stop test project
if: always()
run: ./scripts/compose.sh down
+2
View File
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
/DEFENSE.md
+31
View File
@@ -0,0 +1,31 @@
FROM python:3.11.16-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install --no-cache-dir -r requirements.txt
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
COPY --chown=appuser:appuser src/ ./src/
COPY --chown=appuser:appuser db/ ./db/
COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
COPY infra/app-start.sh /usr/local/bin/app-start
ENTRYPOINT ["/bin/sh", "/usr/local/bin/app-start"]
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2", "--limit-concurrency", "64"]
+191 -18
View File
@@ -1,37 +1,210 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа — Дугина Александра, БПИ2301.
- Целевая оценка — 5, трек 2 «Доказанная идемпотентность pipeline». Проверки выполнены на реальных контейнерах Docker Desktop.
- Использованные источники и помощь — я использовала контракты и тесты курса, собственную реализацию мессенджера из ЛР №4, FastAPI, psycopg, Pillow, boto3 и Pika.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Мне понадобятся Docker Engine или Docker Desktop с доступным daemon, Compose v2,
Python 3.11+ на хосте для проверок и локальные свободные порты 8080, 15685, 9005.
```sh
python3 scripts/init-local.py
./scripts/compose.sh config --quiet
./scripts/compose.sh up -d --build --wait --wait-timeout 150
curl -fsS http://localhost:8080/health/ready
make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
./scripts/compose.sh start api
./scripts/compose.sh up -d --wait --wait-timeout 150
```
`init-local.py` генерирует случайные пароли в игнорируемом каталоге `secrets/`
и создаёт локальный `.env`. При повторном запуске я сохраняю существующие значения.
Порты можно заменить в `.env`. При изменении `PUBLIC_PORT` я передаю
`BASE_URL=http://localhost:НОВЫЙ_ПОРТ` в Make и `--base-url` в тестовые скрипты.
Swagger открывается на <http://localhost:8080/docs>. Веб-консоли RabbitMQ и MinIO
доступны на <http://localhost:15685> и <http://localhost:9005>. Сетевые порты
привязаны к loopback. Остановить стенд с сохранением данных можно через
`./scripts/compose.sh down`. Для намеренного удаления **своих** учебных данных
существует `./scripts/compose.sh down --volumes`. Перед этим я проверяю имя
Compose-проекта и убеждаюсь, что его volumes больше не нужны.
| Роль | Compose и версия | Готовность и данные | Доступ |
| --- | --- | --- | --- |
| Балансировщик | `lb`, HAProxy 3.0.9-alpine | HTTP-проверки двух реплик, локальных данных нет | 8080 через loopback |
| API и миграции | `api` ×2, `migrate`, Python 3.11.16-slim-bookworm | `/health/ready` проверяет БД и доступ к bucket, миграции одноразовые | Внутренняя сеть, secrets с DSN и доступом к S3 |
| База | `postgres`, PostgreSQL 16.8-bookworm | TCP `pg_isready`, `postgres_data` | Внутренняя сеть, файл пароля |
| Кэш профилей | `redis`, Redis 7.4.2-alpine | `redis-cli ping`, AOF в `redis_data` | Внутренняя сеть |
| Объекты | `s3`, MinIO 2025.7.23 с digest из Compose | MinIO live, `s3_data` | Приватный bucket, ограниченная policy приложения, локальная консоль 9005 |
| Брокер | `broker`, RabbitMQ 4.1.4-management | `rabbitmq-diagnostics check_running`, `broker_data` | Внутренний AMQP, консоль 15685, секрет аутентификации |
| Фоновые процессы | `publisher`, `worker` ×2, `recovery`, образ приложения | Проверка продвижения рабочего цикла, reconnect, restart policy | Нет публичных портов |
Compose дожидается PostgreSQL и применяет SQL-миграции под advisory lock. Список
применённых файлов хранится в `schema_migrations`. MinIO bootstrap создаёт
приватный bucket, сервисную учётную запись и policy. Publisher и worker объявляют
одну и ту же durable quorum queue при подключении. `.env.example` показывает
переменные портов, TTL и pipeline, фактически используемые в Compose. Секреты
подключаются файлами, а entrypoint приложения копирует нужные файлы перед
запуском процессов от непривилегированного пользователя.
Все три фоновые роли обновляют метку прогресса в собственном tmpfs.
Worker обновляет её после обработки доставки или успешного опроса очереди,
publisher — после итерации outbox через канал с publisher confirms, recovery —
после итерации проверки lease и плановой очистки. Метка появляется после
первой успешной итерации, сбрасывается, если ошибка зависимости прервала цикл, и при завершении
процесса. Она истекает через 60 секунд без прогресса. Compose опрашивает
проверку раз в 3 секунды и выставляет unhealthy после трёх неуспехов.
Миграции и S3 bootstrap остаются одноразовыми: Compose ждёт их успешного
завершения.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Моя цель — отвечать на upload после надёжного приёма файла и обрабатывать его
в отдельных процессах. PostgreSQL проверяет сессию при каждом
запросе. Redis ускоряет получение профиля. При конкурентных вставках keyset
пагинация видит записи после курсора с ключом `(created_at, id)`. Между
страницами нет общей snapshot-транзакции.
1. API принимает PNG/JPEG потоком с лимитом 10 MiB и проверяет настоящий формат,
декодируемость и лимит 40 Мп в отдельном процессе. Временный файл нужен для
проверки и записи. Под advisory lock API записывает исходник в приватный S3
prefix `staging/`. В одной PostgreSQL-транзакции сохраняются attachment,
job и outbox. После COMMIT API возвращает снимок `202 queued`. В состоянии
queued файл уже можно привязать к сообщению. Download вернёт 409.
2. Publisher читает только зафиксированные outbox-строки через
`FOR UPDATE SKIP LOCKED`, отправляет carrier по учебной JSON-схеме и удаляет
строку после confirm. RabbitMQ сохраняет сообщения в durable quorum queue.
При потере связи или аварии после confirm до удаления отправка может
повториться. В AMQP передаются идентификаторы и номера, байты остаются в S3.
3. Worker держит одну неподтверждённую доставку, перед работой атомарно
захватывает job, увеличивает fencing token и выставляет lease. По умолчанию
два worker-процесса, lease 60 секунд. Worker берёт серверный ключ из БД,
сверяет байты и SHA-256, применяет EXIF orientation, композитит альфа-канал
на белый. JPEG thumbnail вписывается в 256×256 без увеличения и с
округлением вниз. JPEG crop центрирован и имеет размер 128×128.
WebP вписывается в 1280×1280 без увеличения, method 4. Качество JPEG/WebP
для v1 равно 85/82, для v2 — 90/88. Производные сохраняются без исходных
EXIF/ICC. Исходник копируется побайтно. Linux-процесс декодирования worker
ограничен 512 MiB виртуальной памяти, 15 секундами CPU и 20 секундами по
часам родителя. Лимиты контейнера — 1 CPU и 768 MiB на worker.
4. Ключ результата включает attachment, pipeline version, вариант и SHA-256.
Повторная запись того же ключа содержит те же байты. В одной транзакции
worker публикует manifest и переключает активную версию, если lease ещё
действует, fencing token совпадает и версия остаётся желаемой. Затем
отправляет ack. Чтение через API использует только активный manifest.
При переходе с готовой v1 на v2 прежняя версия доступна до атомарной
публикации v2.
5. Временная ошибка или истёкший lease увеличивает номер попытки и возвращает
job в outbox с exponential backoff, jitter 0,75–1,25 от базовой задержки и
пределом базовой задержки 30 секунд. После пяти попыток по умолчанию либо
постоянной ошибки worker/recovery сохраняет failed и запись в PostgreSQL DLQ.
Я использую этот реестр как долговечную очередь ручного replay. Replay
разрешён только для failed job текущей версии и сохраняет UUID и историю
предыдущих попыток.
6. Recovery удаляет staging готовых вложений после commit. Непривязанные и
failed-исходники остаются по умолчанию сутки. Cleanup проверяет метаданные
и берёт тот же lock, что uploader. Во время reprocessing источником
служит опубликованный оригинал. Внутренний ключ бакета в API отсутствует.
```mermaid
flowchart LR
C[Клиент] --> H[HAProxy]
H --> A[Две API-реплики]
A --> S[(MinIO)]
A --> P[(PostgreSQL)]
P --> O[Publisher]
O --> B[(RabbitMQ)]
B --> W[Worker ×2]
W --> S
W --> P
R[Recovery] --> P
R --> S
```
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Docker VM использовала Linux aarch64, 8 CPU и 7,65 GiB RAM.
Хост Darwin arm64, 8 CPU, 16 GiB RAM, тестовый клиент Python 3.13.3.
Строки CPU и памяти в `container-resources.txt` — один снимок после запуска.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Проверка и наблюдение | Доказательство |
| --- | --- | --- |
| Контракты и примеры | `make check` и полная OpenAPI/JSON Schema валидация прошли | [offline](evidence/local-checks.txt) |
| Воспроизводимый запуск | `up -d --build --wait --wait-timeout 150` завершился с кодом 0. Две API-реплики, балансировщик, хранилища, брокер и все фоновые процессы healthy | [запуск](evidence/compose-up.txt), [после перезапуска](evidence/post-restart-health.txt) |
| Уровень 3, публичный API | `make test` прошёл, включая права, SHA-256 оригинала и размер thumbnail | [smoke](evidence/smoke.txt) |
| Уровень 3, сохранность | `make test-persistence` подтвердил сохранность данных и прежней сессии. Изменились timestamps запуска API, HAProxy, хранилищ, RabbitMQ и фоновых процессов | [тест](evidence/persistence.txt), [до](evidence/restart-before.txt), [после](evidence/restart-after.txt) |
| Уровень 3, failover | `make test-failover` прошёл. Поочерёдно остановлены `api-2` и `api-1`. Первая успешная запись через 2,280 и 1,925 с соответственно. В каждом прогоне подтверждено 120 записей | [тест](evidence/failover.txt), [состояние контейнеров](evidence/failover-containers.txt), [повтор](evidence/failover-repeat.txt) |
| Уровни 3–4, обработка изображений | Проверки с Pillow 11.2.1 внутри образа прошли, включая геометрию, детерминированность, EXIF, прозрачность и лимит пикселей | [render](evidence/container-render.txt) |
| Уровни 3–5, отказоустойчивость, retry, DLQ и идемпотентность | Сценарии и результаты приведены ниже | [JSONL](evidence/track2.jsonl) |
Отдельный `make load` выполнил 7583 успешных POST сообщений за 30.037 с, concurrency 4. Ошибок 0, 252.458 успешных запросов/с. Задержки p50/p95/p99 успешных запросов составили 12.714/37.804/73.943 мс. Это один closed-loop замер HTTP-записей сообщений. [Метрики нагрузки](evidence/load.txt) · [снимок CPU и памяти](evidence/container-resources.txt)
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Гипотеза.** В заданной модели отказов принятый upload дойдёт до готового
результата после восстановления компонентов. Повторные доставки не создадут
вторую публикацию того же manifest. Новая версия сохранится, если позднее
возобновить старый worker или доставить старую job.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Эксперимент использует отдельный Compose-проект, две реплики worker,
lease 20 секунд, PNG 320×200 и обычный HTTP API. Для каждого сценария
создаётся новый объект. Время в JSONL измеряется монотонными часами от начала
сценария до последнего assertion и включает команды Compose, рестарты и
ожидание lease. Отдельный сценарий отправляет шесть одновременных POST
сообщений с общим ключом через HAProxy.
```sh
export COMPOSE_PROJECT_NAME=mtusi2026lab5-track2
export PUBLIC_PORT=8085 BROKER_CONSOLE_PORT=15686 S3_CONSOLE_PORT=9006
export PIPELINE_FAULTS=1 WORKER_CONCURRENCY=2 JOB_TIMEOUT_SECONDS=20
./scripts/compose.sh up -d --build --wait --wait-timeout 150
export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1
set -o pipefail
python3 student-tests/pipeline.py --disposable --base-url http://localhost:8085 | tee evidence/track2.jsonl
```
Исполнитель читает PostgreSQL и MinIO и проверяет скачанные байты по HTTP.
Для аварий он ждёт записи о достижении конкретной точки, сверяет журнал
контейнеров и проверяет число попыток, публикаций, ключей S3, состояние и хеши.
Сценарий брокера останавливает реальный контейнер. `crash` завершает процесс
через `os._exit(86)`. `pause` удерживает старого исполнителя до явного
освобождения в эксперименте.
| Воздействие | Наблюдение | Время сценария |
| --- | --- | --- |
| Конкурентные записи и проверка входа | Шесть запросов с одним ключом через две API-реплики дали один ID сообщения. Отклонены чужая загрузка, неверный MIME, повреждённое и слишком большое тело | 0,041 / 0,379 с |
| Остановка RabbitMQ | Принятый upload остался в outbox. Publisher и оба worker стали unhealthy. После старта RabbitMQ они вернулись в healthy, а вложение стало ready с первой попытки | 19,129 с |
| Авария издателя до publish и после confirm | Задание дошло до ready без пересоздания. Повтор после confirm проигнорирован. В каждом сценарии зафиксирована одна публикация manifest | 12,328 / 12,365 с |
| Авария worker до записи S3 | В точке сбоя итоговых объектов не было. После истечения lease вторая попытка опубликовала manifest один раз | 28,879 с |
| Авария после S3 до PostgreSQL | В S3 уже лежали четыре объекта, а job была processing. После второй попытки сохранился один набор из четырёх ключей и одна публикация manifest | 28,390 с |
| Авария после COMMIT результата до ack | Дубль доставки проигнорирован, номер попытки остался 1, manifest опубликован один раз | 7,570 с |
| Дубль при активном lease и возобновление старого worker | При активном lease зафиксирован один захват. После истечения lease вторая попытка получила fencing token 2. Поздняя фиксация старой попытки отклонена | 33,826 с |
| v2 готова раньше приостановленной или запоздалой v1 | API продолжил выдавать v2. Устаревшая v1 не опубликовала manifest. Запоздалая v1 не получила lease и не создала объектов S3 | 14,563 / 16,117 с |
| Повторная обработка готовой v1 | Пока v2 строилась, скачивалась v1. После COMMIT все скачанные варианты соответствовали v2. Дубль v1 не изменил опубликованный manifest | 16,302 с |
| Временная и постоянная ошибки | Временная ошибка отложила повтор. Постоянная дала failed, безопасный error_code и запись DLQ. Явный replay завершил то же задание со второй попытки | 8,366 / 9,060 с |
| Рестарт API, брокера и хранилищ при остановленных worker | Подтверждённое задание осталось queued, скачивание вернуло 409. После запуска worker вложение стало ready с первой попытки, ссылка в сообщении сохранилась, staging очищен | 23,390 с |
Все 15 сценариев завершились с PASS. После эксперимента Compose повторно подтвердил готовность всех долгоживущих сервисов. [Результаты JSONL](evidence/track2.jsonl) · [код завершения](evidence/track2-errors.txt) · [готовность после эксперимента](evidence/track2-final-health.txt)
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Я опираюсь на сохранность volumes PostgreSQL, RabbitMQ и MinIO на одном хосте.
Временная потеря связи поддерживается протоколом, а потеря диска или всего
хоста выходит за пределы эксперимента. Outbox и lease закрывают разные окна
отказа. Publisher confirm подтверждает приём брокером, а consumer ack снимает
ответственность за конкретную доставку после долговечного решения. Между ними
процесс может остановиться, и RabbitMQ может повторить доставку. Операция S3
может выполниться несколько раз. Идемпотентность проявляется в единственной
публикации активного manifest и в стабильном содержимом адресуемых объектов.
Гарантии exactly-once доставки здесь нет.
Старые версии результата и объекты незавершённой устаревшей версии могут
оставаться в закрытом bucket. Для длительной эксплуатации потребуется отдельная
политика их удаления и аудит потребления места. Lease фиксированный, без
продления, с запасом относительно бюджета обработки. RabbitMQ в стенде
одноузловой. Эксперимент измеряет восстановление одного стенда и использует
один профиль входного изображения.
+209
View File
@@ -0,0 +1,209 @@
name: mtusi2026lab5
services:
postgres:
image: postgres:16.8-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -h 127.0.0.1 -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
cpus: 1.0
mem_limit: 512m
restart: unless-stopped
redis:
image: redis:7.4.2-alpine
command: [redis-server, --appendonly, 'yes', --appendfsync, always]
volumes: [redis_data:/data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
restart: unless-stopped
migrate:
build: .
image: mtusi2026lab5-api:local
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
cpus: 0.5
mem_limit: 256m
restart: 'no'
api:
build: .
image: mtusi2026lab5-api:local
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
deploy:
replicas: 2
healthcheck:
test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()']
interval: 2s
timeout: 3s
retries: 3
start_period: 5s
tmpfs:
- /tmp:size=16m,mode=1777
stop_grace_period: 6s
cpus: 0.75
mem_limit: 768m
restart: unless-stopped
broker:
image: rabbitmq:4.1.4-management
entrypoint: [/bin/sh, /broker-start.sh]
environment:
RABBITMQ_DEFAULT_USER: messenger
secrets: [broker_password]
volumes:
- broker_data:/var/lib/rabbitmq
- ./infra/broker-start.sh:/broker-start.sh:ro
hostname: broker
ports: ['127.0.0.1:${BROKER_CONSOLE_PORT:-15685}:15672']
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, check_running]
interval: 4s
timeout: 5s
retries: 20
mem_limit: 768m
cpus: 1.0
restart: unless-stopped
publisher: &pipeline
image: mtusi2026lab5-api:local
command: [python, -m, src.publisher]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
PIPELINE_FAULTS: ${PIPELINE_FAULTS:-0}
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
depends_on:
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: [CMD, python, -m, src.pipeline_health]
interval: 3s
timeout: 2s
retries: 3
start_period: 20s
tmpfs:
- /tmp:size=16m,mode=1777
cpus: 0.25
mem_limit: 256m
restart: unless-stopped
worker:
<<: *pipeline
command: [python, -m, src.worker]
deploy:
replicas: ${WORKER_CONCURRENCY:-2}
init: true
tmpfs:
- /tmp:size=64m,mode=1777
cpus: 1.0
mem_limit: 768m
stop_grace_period: 30s
recovery:
<<: *pipeline
command: [python, -m, src.recovery]
lb:
image: haproxy:3.0.9-alpine
ports: ['127.0.0.1:${PUBLIC_PORT:-8080}:8080']
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
depends_on:
api:
condition: service_healthy
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready']
interval: 5s
timeout: 3s
retries: 3
cpus: 0.5
mem_limit: 128m
restart: unless-stopped
s3:
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
entrypoint: [/opt/bitnami/minio/bin/minio]
user: '0:0'
command: [server, /data, --console-address, ':9001']
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
ports: ['127.0.0.1:${S3_CONSOLE_PORT:-9005}:9001']
healthcheck:
test: [CMD, curl, -f, http://localhost:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
mem_limit: 768m
cpus: 1.0
s3-bootstrap:
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
entrypoint: [/bin/sh, /bootstrap.sh]
volumes:
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
- ./infra/s3-policy.json:/policy.json:ro
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
depends_on:
s3:
condition: service_healthy
restart: 'no'
volumes:
s3_data:
postgres_data:
redis_data:
broker_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
broker_url:
file: ./secrets/broker_url
broker_password:
file: ./secrets/broker_password
+28
View File
@@ -0,0 +1,28 @@
# Операторские операции pipeline
Эти команды выполняются внутри доверенного Compose-проекта. HTTP-контракт
мессенджера остаётся исходным. Идентификаторы берутся из PostgreSQL.
```sh
./scripts/compose.sh exec -T recovery python -m src.operator reprocess ATTACHMENT_UUID 2
./scripts/compose.sh exec -T recovery python -m src.operator replay JOB_UUID
./scripts/compose.sh exec -T recovery python -m src.operator redeliver JOB_UUID
```
- `reprocess` принимает версию выше желаемой. Реализованы 1 и 2. Старая готовая
версия доступна, пока новая строится. Для вложения без готовой версии
используются обычные состояния LAB5.
- `replay` принимает только failed job текущей желаемой версии с сохранённым
исходником. UUID job сохраняется, attempt растёт, бюджет попыток обновляется.
История отказа остаётся в `job_dead_letters`.
- `redeliver` служит для диагностики. Команда посылает текущий carrier через
RabbitMQ и ждёт publisher confirm. Она сама не выдаёт lease и не меняет статус.
В `image_jobs` хранится состояние каждой версии. Журнал `job_events` сохраняет
захват, номер fencing token, подтверждение, retry, DLQ, публикацию результата и
отклонение устаревшего исполнителя. Выход CLI содержит поля carrier.
Внедрение сбоев разрешает только переменная `PIPELINE_FAULTS=1`. Управление
производится непосредственно через PostgreSQL на выделенном тестовом стенде.
Разовые команды в `pipeline_faults` привязаны к UUID вложения и версии. Выключайте
внедрение сбоев в обычном запуске.
+51
View File
@@ -0,0 +1,51 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name VARCHAR(100) NOT NULL
CHECK (length(btrim(display_name)) > 0),
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title VARCHAR(100) NOT NULL
CHECK (length(btrim(title)) > 0),
created_by UUID NOT NULL REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id),
joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL
CHECK (length(text) <= 4000 AND length(btrim(text)) > 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS sessions (
id UUID PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
token_hash TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_chat_members_user_id
ON chat_members (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
ON sessions (expires_at);
+9
View File
@@ -0,0 +1,9 @@
CREATE TABLE message_requests (
chat_id UUID NOT NULL REFERENCES chats(id),
sender_id UUID NOT NULL REFERENCES users(id),
key VARCHAR(128) NOT NULL,
text TEXT NOT NULL,
message_id UUID REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (chat_id, sender_id, key)
);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
filename TEXT NOT NULL,
object_key TEXT NOT NULL UNIQUE,
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
sha256 CHAR(64) NOT NULL,
status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0));
+77
View File
@@ -0,0 +1,77 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
ALTER TABLE attachments ADD CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments
ADD COLUMN desired_version INTEGER NOT NULL DEFAULT 1,
ADD COLUMN active_version INTEGER,
ADD COLUMN staging_key TEXT UNIQUE,
ADD COLUMN error_code TEXT;
CREATE TABLE image_jobs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL CHECK (pipeline_version > 0),
state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt > 0),
failures INTEGER NOT NULL DEFAULT 0,
fence BIGINT NOT NULL DEFAULT 0,
lease_until TIMESTAMPTZ,
error_code TEXT,
manifest JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version),
CHECK ((state = 'processing') = (lease_until IS NOT NULL)),
CHECK ((state = 'ready') = (manifest IS NOT NULL))
);
CREATE INDEX image_jobs_expired ON image_jobs(lease_until) WHERE state = 'processing';
CREATE TABLE job_outbox (
job_id UUID PRIMARY KEY REFERENCES image_jobs(id),
publish_after TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX job_outbox_due ON job_outbox(publish_after);
CREATE TABLE job_events (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id UUID NOT NULL REFERENCES image_jobs(id),
attempt INTEGER NOT NULL,
event TEXT NOT NULL,
detail TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX job_events_lookup ON job_events(job_id, id);
-- Durable operator-visible DLQ. Replays retain the historical failure rows.
CREATE TABLE job_dead_letters (
job_id UUID NOT NULL REFERENCES image_jobs(id),
attempt INTEGER NOT NULL,
error_code TEXT NOT NULL,
failed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
replayed_at TIMESTAMPTZ,
PRIMARY KEY (job_id, attempt)
);
-- Used only by an explicitly enabled synthetic fault-injection stand.
CREATE TABLE pipeline_faults (
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL,
point TEXT NOT NULL,
action TEXT NOT NULL CHECK (action IN ('crash','pause','temporary','permanent')),
hit_at TIMESTAMPTZ,
released BOOLEAN NOT NULL DEFAULT false,
PRIMARY KEY (attachment_id,pipeline_version,point)
);
-- Upgrade persisted lab4 originals, which remain the source until v1 commits.
UPDATE attachments SET status='queued', updated_at=now();
WITH inserted AS (
INSERT INTO image_jobs(attachment_id,pipeline_version)
SELECT id,1 FROM attachments RETURNING id
)
INSERT INTO job_outbox(job_id) SELECT id FROM inserted;
ALTER TABLE attachments
ADD CHECK (desired_version > 0 AND (active_version IS NULL OR active_version <= desired_version)),
ADD CHECK ((status = 'ready') = (active_version IS NOT NULL)),
ADD CHECK ((status = 'failed') = (error_code IS NOT NULL)),
ADD FOREIGN KEY (id, active_version) REFERENCES image_jobs(attachment_id, pipeline_version);
+231
View File
@@ -0,0 +1,231 @@
Compose now can delegate build to bake for better performances
Just set COMPOSE_BAKE=true
#0 building with "desktop-linux" instance using docker driver
#1 [migrate internal] load build definition from Dockerfile
#1 transferring dockerfile: 1.04kB done
#1 DONE 0.0s
#2 [migrate internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
#2 DONE 0.0s
#3 [migrate internal] load .dockerignore
#3 transferring context: 208B done
#3 DONE 0.0s
#4 [migrate internal] load build context
#4 transferring context: 3.49kB done
#4 DONE 0.0s
#5 [migrate 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
#5 DONE 0.0s
#6 [migrate 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt
#6 CACHED
#7 [migrate 3/9] COPY requirements.txt .
#7 CACHED
#8 [migrate 2/9] WORKDIR /app
#8 CACHED
#9 [migrate 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
#9 CACHED
#10 [migrate 6/9] COPY --chown=appuser:appuser src/ ./src/
#10 DONE 0.0s
#11 [migrate 7/9] COPY --chown=appuser:appuser db/ ./db/
#11 DONE 0.0s
#12 [migrate 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
#12 DONE 0.0s
#13 [migrate 9/9] COPY infra/app-start.sh /usr/local/bin/app-start
#13 DONE 0.0s
#14 [migrate] exporting to image
#14 exporting layers 0.1s done
#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44
#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44 done
#14 exporting config sha256:6e878a3e1844f17aa93a4277e04be0bd98ca42f3ba53997976cb1499c4ab5a22 done
#14 exporting attestation manifest sha256:8998f445e472a60a93fd3480f4000dede2d0fa76d5fcb690911d90e747483592 0.0s done
#14 exporting manifest list sha256:404b7aaa2cb3fda8ba8e0ea873f7171be30797c57a39261d9b78ec48f3e477c5 done
#14 naming to docker.io/library/mtusi2026lab5-api:local done
#14 unpacking to docker.io/library/mtusi2026lab5-api:local 0.0s done
#14 DONE 0.2s
#15 [migrate] resolving provenance for metadata file
#15 DONE 0.0s
#16 [api internal] load build definition from Dockerfile
#16 transferring dockerfile: 1.04kB done
#16 DONE 0.0s
#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
#2 DONE 0.0s
#17 [api internal] load .dockerignore
#17 transferring context: 208B done
#17 DONE 0.0s
#18 [api internal] load build context
#18 transferring context: 2.45kB done
#18 DONE 0.0s
#5 [api 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
#5 DONE 0.0s
#19 [api 3/9] COPY requirements.txt .
#19 CACHED
#20 [api 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt
#20 CACHED
#21 [api 7/9] COPY --chown=appuser:appuser db/ ./db/
#21 CACHED
#8 [api 2/9] WORKDIR /app
#8 CACHED
#22 [api 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
#22 CACHED
#23 [api 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
#23 CACHED
#24 [api 6/9] COPY --chown=appuser:appuser src/ ./src/
#24 CACHED
#25 [api 9/9] COPY infra/app-start.sh /usr/local/bin/app-start
#25 CACHED
#26 [api] exporting to image
#26 exporting layers done
#26 exporting manifest sha256:afc88e63bc12b5238aa444e8bdb808d7f36d417557b747036fa9fb8a16d604b3 done
#26 exporting config sha256:c50287970425ce9631e003b98819e176d36d3d36d2b33f9de55894b567a27cc3 done
#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179
#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179 0.0s done
#26 exporting manifest list sha256:a8591065abe521d14b78bd1884238576e83d46e560f228be8faa90ec3e9a677f done
#26 naming to docker.io/library/mtusi2026lab5-api:local done
#26 unpacking to docker.io/library/mtusi2026lab5-api:local done
#26 DONE 0.1s
#27 [api] resolving provenance for metadata file
#27 DONE 0.0s
api Built
migrate Built
Container mtusi2026lab5-s3-1 Running
Container mtusi2026lab5-postgres-1 Running
Container mtusi2026lab5-s3-bootstrap-1 Created
Container mtusi2026lab5-broker-1 Running
Container mtusi2026lab5-redis-1 Running
Container mtusi2026lab5-recovery-1 Stopping
Container mtusi2026lab5-api-2 Stopping
Container mtusi2026lab5-api-1 Stopping
Container mtusi2026lab5-worker-1 Stopping
Container mtusi2026lab5-worker-2 Stopping
Container mtusi2026lab5-publisher-1 Stopping
Container mtusi2026lab5-publisher-1 Stopped
Container mtusi2026lab5-worker-1 Stopped
Container mtusi2026lab5-worker-2 Stopped
Container mtusi2026lab5-api-2 Stopped
Container mtusi2026lab5-api-1 Stopped
Container mtusi2026lab5-recovery-1 Stopped
Container mtusi2026lab5-migrate-1 Recreate
Container mtusi2026lab5-migrate-1 Recreated
Container mtusi2026lab5-publisher-1 Recreate
Container mtusi2026lab5-worker-1 Recreate
Container mtusi2026lab5-worker-2 Recreate
Container mtusi2026lab5-recovery-1 Recreate
Container mtusi2026lab5-worker-1 Recreated
Container mtusi2026lab5-worker-2 Recreated
Container mtusi2026lab5-recovery-1 Recreated
Container mtusi2026lab5-publisher-1 Recreated
Container mtusi2026lab5-lb-1 Stopping
Container mtusi2026lab5-lb-1 Stopped
Container mtusi2026lab5-api-1 Recreate
Container mtusi2026lab5-api-1 Recreated
Container mtusi2026lab5-lb-1 Stopping
Container mtusi2026lab5-lb-1 Stopped
Container mtusi2026lab5-api-2 Recreate
Container mtusi2026lab5-api-2 Recreated
Container mtusi2026lab5-lb-1 Created
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Starting
Container mtusi2026lab5-migrate-1 Starting
Container mtusi2026lab5-migrate-1 Started
Container mtusi2026lab5-s3-bootstrap-1 Started
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-recovery-1 Starting
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-api-2 Starting
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-worker-2 Starting
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-publisher-1 Starting
Container mtusi2026lab5-publisher-1 Started
Container mtusi2026lab5-api-2 Started
Container mtusi2026lab5-api-1 Starting
Container mtusi2026lab5-recovery-1 Started
Container mtusi2026lab5-worker-2 Started
Container mtusi2026lab5-worker-1 Starting
Container mtusi2026lab5-api-1 Started
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-worker-1 Started
Container mtusi2026lab5-api-2 Healthy
Container mtusi2026lab5-api-1 Healthy
Container mtusi2026lab5-lb-1 Starting
Container mtusi2026lab5-lb-1 Started
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-worker-2 Waiting
Container mtusi2026lab5-worker-1 Waiting
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-publisher-1 Waiting
Container mtusi2026lab5-lb-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-recovery-1 Waiting
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-recovery-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-publisher-1 Healthy
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-api-2 Healthy
Container mtusi2026lab5-api-1 Healthy
Container mtusi2026lab5-worker-2 Healthy
Container mtusi2026lab5-worker-1 Healthy
Container mtusi2026lab5-lb-1 Healthy
+3
View File
@@ -0,0 +1,3 @@
PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding
PASS: fixture geometry, original preserved, deterministic v1, different v2
PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed
+11
View File
@@ -0,0 +1,11 @@
mtusi2026lab5-api-1 12.98% 65.53MiB / 768MiB
mtusi2026lab5-api-2 14.53% 65.53MiB / 768MiB
mtusi2026lab5-broker-1 10.98% 92.7MiB / 768MiB
mtusi2026lab5-lb-1 0.37% 11.71MiB / 128MiB
mtusi2026lab5-postgres-1 2.31% 35.48MiB / 512MiB
mtusi2026lab5-publisher-1 0.22% 26MiB / 256MiB
mtusi2026lab5-recovery-1 0.09% 52.61MiB / 256MiB
mtusi2026lab5-redis-1 0.33% 9.324MiB / 128MiB
mtusi2026lab5-s3-1 3.14% 418.5MiB / 768MiB
mtusi2026lab5-worker-1 0.16% 53.38MiB / 768MiB
mtusi2026lab5-worker-2 0.18% 53.38MiB / 768MiB
+4
View File
@@ -0,0 +1,4 @@
/mtusi2026lab5-api-1 running healthy
/mtusi2026lab5-api-2 running healthy
/mtusi2026lab5-api-1 running healthy
/mtusi2026lab5-api-2 exited unhealthy
+9
View File
@@ -0,0 +1,9 @@
Repeat 1
/mtusi2026lab5-api-1 running healthy
/mtusi2026lab5-api-2 running healthy
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 2, "first_recovery_seconds": 1.925}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
/mtusi2026lab5-api-1 exited unhealthy
/mtusi2026lab5-api-2 running healthy
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 3, "first_recovery_seconds": 2.28}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+18
View File
@@ -0,0 +1,18 @@
python3 tests/load.py --base-url "http://localhost:8080"
{
"lab": 5,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30,
"elapsed_seconds": 30.037,
"concurrency": 4,
"successful_requests": 7583,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 252.458,
"successful_latency_ms": {
"p50": 12.714,
"p95": 37.804,
"p99": 73.943
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+15
View File
@@ -0,0 +1,15 @@
Fri Nov 20 15:25:49 UTC 2026
276dbda
Darwin arm64
8
17179869184
Python 3.13.3
Docker Compose version v2.33.1-desktop.1
Engine=28.0.1 OS=linux arch=aarch64 CPUs=8 memory_bytes=8218316800
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding
PASS: fixture geometry, original preserved, deterministic v1, different v2
PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+75
View File
@@ -0,0 +1,75 @@
Container mtusi2026lab5-s3-1 Running
Container mtusi2026lab5-broker-1 Running
Container mtusi2026lab5-postgres-1 Running
Container mtusi2026lab5-redis-1 Running
Container mtusi2026lab5-s3-bootstrap-1 Created
Container mtusi2026lab5-migrate-1 Created
Container mtusi2026lab5-publisher-1 Running
Container mtusi2026lab5-recovery-1 Running
Container mtusi2026lab5-worker-1 Running
Container mtusi2026lab5-api-1 Running
Container mtusi2026lab5-worker-2 Running
Container mtusi2026lab5-api-2 Running
Container mtusi2026lab5-lb-1 Running
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-migrate-1 Starting
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Starting
Container mtusi2026lab5-s3-bootstrap-1 Started
Container mtusi2026lab5-migrate-1 Started
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-api-1 Healthy
Container mtusi2026lab5-api-2 Healthy
Container mtusi2026lab5-recovery-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-broker-1 Waiting
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-publisher-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-worker-1 Waiting
Container mtusi2026lab5-worker-2 Waiting
Container mtusi2026lab5-lb-1 Waiting
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-broker-1 Healthy
Container mtusi2026lab5-recovery-1 Healthy
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-publisher-1 Healthy
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-lb-1 Healthy
Container mtusi2026lab5-worker-1 Healthy
Container mtusi2026lab5-worker-2 Healthy
Container mtusi2026lab5-api-1 Healthy
Container mtusi2026lab5-api-2 Healthy
+11
View File
@@ -0,0 +1,11 @@
/mtusi2026lab5-api-1 2026-11-20T15:26:46.863943835Z running
/mtusi2026lab5-api-2 2026-11-20T15:26:46.921602543Z running
/mtusi2026lab5-broker-1 2026-11-20T15:26:47.614707835Z running
/mtusi2026lab5-lb-1 2026-11-20T15:26:46.466838501Z running
/mtusi2026lab5-postgres-1 2026-11-20T15:26:46.30309871Z running
/mtusi2026lab5-publisher-1 2026-11-20T15:26:46.328859918Z running
/mtusi2026lab5-recovery-1 2026-11-20T15:26:46.960652918Z running
/mtusi2026lab5-redis-1 2026-11-20T15:26:46.267219626Z running
/mtusi2026lab5-s3-1 2026-11-20T15:26:46.428910043Z running
/mtusi2026lab5-worker-1 2026-11-20T15:26:46.488719835Z running
/mtusi2026lab5-worker-2 2026-11-20T15:26:46.475586876Z running
+11
View File
@@ -0,0 +1,11 @@
/mtusi2026lab5-api-1 2026-11-20T15:26:01.213628633Z running
/mtusi2026lab5-api-2 2026-11-20T15:26:01.040028133Z running
/mtusi2026lab5-broker-1 2026-11-20T15:23:46.29031721Z running
/mtusi2026lab5-lb-1 2026-11-20T15:26:06.958527761Z running
/mtusi2026lab5-postgres-1 2026-11-20T15:23:44.823417251Z running
/mtusi2026lab5-publisher-1 2026-11-20T15:26:01.037922008Z running
/mtusi2026lab5-recovery-1 2026-11-20T15:26:01.0402713Z running
/mtusi2026lab5-redis-1 2026-11-20T15:23:44.789881876Z running
/mtusi2026lab5-s3-1 2026-11-20T15:23:45.012262667Z running
/mtusi2026lab5-worker-1 2026-11-20T15:26:01.26324605Z running
/mtusi2026lab5-worker-2 2026-11-20T15:26:01.0369378Z running
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+1
View File
@@ -0,0 +1 @@
pipeline.py exit=0
+75
View File
@@ -0,0 +1,75 @@
Container mtusi2026lab5-track2-postgres-1 Running
Container mtusi2026lab5-track2-s3-1 Running
Container mtusi2026lab5-track2-broker-1 Running
Container mtusi2026lab5-track2-redis-1 Running
Container mtusi2026lab5-track2-s3-bootstrap-1 Created
Container mtusi2026lab5-track2-migrate-1 Created
Container mtusi2026lab5-track2-recovery-1 Running
Container mtusi2026lab5-track2-publisher-1 Running
Container mtusi2026lab5-track2-worker-1 Running
Container mtusi2026lab5-track2-worker-2 Running
Container mtusi2026lab5-track2-api-1 Running
Container mtusi2026lab5-track2-api-2 Running
Container mtusi2026lab5-track2-lb-1 Running
Container mtusi2026lab5-track2-s3-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Healthy
Container mtusi2026lab5-track2-migrate-1 Starting
Container mtusi2026lab5-track2-s3-1 Healthy
Container mtusi2026lab5-track2-s3-bootstrap-1 Starting
Container mtusi2026lab5-track2-s3-bootstrap-1 Started
Container mtusi2026lab5-track2-migrate-1 Started
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-redis-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-redis-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-api-2 Waiting
Container mtusi2026lab5-track2-api-1 Waiting
Container mtusi2026lab5-track2-api-2 Healthy
Container mtusi2026lab5-track2-api-1 Healthy
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-api-2 Waiting
Container mtusi2026lab5-track2-api-1 Waiting
Container mtusi2026lab5-track2-recovery-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Waiting
Container mtusi2026lab5-track2-redis-1 Waiting
Container mtusi2026lab5-track2-worker-1 Waiting
Container mtusi2026lab5-track2-worker-2 Waiting
Container mtusi2026lab5-track2-lb-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-1 Waiting
Container mtusi2026lab5-track2-publisher-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Healthy
Container mtusi2026lab5-track2-publisher-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-redis-1 Healthy
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-lb-1 Healthy
Container mtusi2026lab5-track2-recovery-1 Healthy
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-s3-1 Healthy
Container mtusi2026lab5-track2-api-2 Healthy
Container mtusi2026lab5-track2-api-1 Healthy
Container mtusi2026lab5-track2-worker-1 Healthy
Container mtusi2026lab5-track2-worker-2 Healthy
+112
View File
@@ -0,0 +1,112 @@
Container mtusi2026lab5-track2-postgres-1 Running
Container mtusi2026lab5-track2-broker-1 Running
Container mtusi2026lab5-track2-s3-1 Running
Container mtusi2026lab5-track2-s3-bootstrap-1 Created
Container mtusi2026lab5-track2-redis-1 Running
Container mtusi2026lab5-track2-recovery-1 Stopping
Container mtusi2026lab5-track2-publisher-1 Stopping
Container mtusi2026lab5-track2-api-2 Stopping
Container mtusi2026lab5-track2-api-1 Stopping
Container mtusi2026lab5-track2-worker-1 Stopping
Container mtusi2026lab5-track2-worker-2 Stopping
Container mtusi2026lab5-track2-publisher-1 Stopped
Container mtusi2026lab5-track2-worker-2 Stopped
Container mtusi2026lab5-track2-worker-1 Stopped
Container mtusi2026lab5-track2-api-2 Stopped
Container mtusi2026lab5-track2-recovery-1 Stopped
Container mtusi2026lab5-track2-api-1 Stopped
Container mtusi2026lab5-track2-migrate-1 Recreate
Container mtusi2026lab5-track2-migrate-1 Recreated
Container mtusi2026lab5-track2-recovery-1 Recreate
Container mtusi2026lab5-track2-publisher-1 Recreate
Container mtusi2026lab5-track2-worker-1 Recreate
Container mtusi2026lab5-track2-worker-2 Recreate
Container mtusi2026lab5-track2-publisher-1 Recreated
Container mtusi2026lab5-track2-recovery-1 Recreated
Container mtusi2026lab5-track2-worker-2 Recreated
Container mtusi2026lab5-track2-worker-1 Recreated
Container mtusi2026lab5-track2-lb-1 Stopping
Container mtusi2026lab5-track2-lb-1 Stopped
Container mtusi2026lab5-track2-api-1 Recreate
Container mtusi2026lab5-track2-api-1 Recreated
Container mtusi2026lab5-track2-lb-1 Stopping
Container mtusi2026lab5-track2-lb-1 Stopped
Container mtusi2026lab5-track2-api-2 Recreate
Container mtusi2026lab5-track2-api-2 Recreated
Container mtusi2026lab5-track2-lb-1 Created
Container mtusi2026lab5-track2-s3-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Waiting
Container mtusi2026lab5-track2-postgres-1 Healthy
Container mtusi2026lab5-track2-migrate-1 Starting
Container mtusi2026lab5-track2-s3-1 Healthy
Container mtusi2026lab5-track2-s3-bootstrap-1 Starting
Container mtusi2026lab5-track2-migrate-1 Started
Container mtusi2026lab5-track2-s3-bootstrap-1 Started
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-redis-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-redis-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-api-2 Starting
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-publisher-1 Starting
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-recovery-1 Starting
Container mtusi2026lab5-track2-worker-1 Starting
Container mtusi2026lab5-track2-recovery-1 Started
Container mtusi2026lab5-track2-worker-1 Started
Container mtusi2026lab5-track2-worker-2 Starting
Container mtusi2026lab5-track2-publisher-1 Started
Container mtusi2026lab5-track2-api-2 Started
Container mtusi2026lab5-track2-api-1 Starting
Container mtusi2026lab5-track2-api-1 Started
Container mtusi2026lab5-track2-api-2 Waiting
Container mtusi2026lab5-track2-api-1 Waiting
Container mtusi2026lab5-track2-worker-2 Started
Container mtusi2026lab5-track2-api-2 Healthy
Container mtusi2026lab5-track2-api-1 Healthy
Container mtusi2026lab5-track2-lb-1 Starting
Container mtusi2026lab5-track2-lb-1 Started
Container mtusi2026lab5-track2-lb-1 Waiting
Container mtusi2026lab5-track2-api-2 Waiting
Container mtusi2026lab5-track2-api-1 Waiting
Container mtusi2026lab5-track2-recovery-1 Waiting
Container mtusi2026lab5-track2-redis-1 Waiting
Container mtusi2026lab5-track2-broker-1 Waiting
Container mtusi2026lab5-track2-publisher-1 Waiting
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
Container mtusi2026lab5-track2-s3-1 Waiting
Container mtusi2026lab5-track2-migrate-1 Waiting
Container mtusi2026lab5-track2-worker-1 Waiting
Container mtusi2026lab5-track2-worker-2 Waiting
Container mtusi2026lab5-track2-postgres-1 Waiting
Container mtusi2026lab5-track2-recovery-1 Healthy
Container mtusi2026lab5-track2-broker-1 Healthy
Container mtusi2026lab5-track2-publisher-1 Healthy
Container mtusi2026lab5-track2-postgres-1 Healthy
Container mtusi2026lab5-track2-s3-1 Healthy
Container mtusi2026lab5-track2-migrate-1 Exited
Container mtusi2026lab5-track2-redis-1 Healthy
Container mtusi2026lab5-track2-api-2 Healthy
Container mtusi2026lab5-track2-api-1 Healthy
Container mtusi2026lab5-track2-worker-1 Healthy
Container mtusi2026lab5-track2-worker-2 Healthy
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
Container mtusi2026lab5-track2-lb-1 Healthy
+15
View File
@@ -0,0 +1,15 @@
{"scenario": "concurrent_message", "result": "PASS", "seconds": 0.041, "attachment_id": null, "requests": 6}
{"scenario": "upload_validation", "result": "PASS", "seconds": 0.379, "attachment_id": null, "statuses": [404, 415, 415, 413]}
{"scenario": "broker_unavailable", "result": "PASS", "seconds": 19.129, "attachment_id": "5b35bebd-3c16-431c-9167-1fae38db338d", "attempt": 1, "pipeline_health_recovered": true}
{"scenario": "before_publish", "result": "PASS", "seconds": 12.328, "attachment_id": "ac1de244-d64b-43f2-98c9-d4662503eb35", "attempt": 1, "published": 1}
{"scenario": "after_confirm", "result": "PASS", "seconds": 12.365, "attachment_id": "75602f19-039e-4bbf-bfb9-25c677d39ba9", "attempt": 1, "published": 1}
{"scenario": "before_s3", "result": "PASS", "seconds": 28.879, "attachment_id": "e8fa8c69-04d7-4f51-b644-17b538a4f6c7", "attempt": 2, "published": 1}
{"scenario": "after_s3", "result": "PASS", "seconds": 28.39, "attachment_id": "7c7439af-6b20-4941-afbb-41c8de02bf3c", "attempt": 2, "published": 1}
{"scenario": "after_commit", "result": "PASS", "seconds": 7.57, "attachment_id": "753ad377-cac2-4630-8207-efeffce7522c", "attempt": 1, "published": 1}
{"scenario": "duplicate_and_fencing", "result": "PASS", "seconds": 33.826, "attachment_id": "c0e36849-d59d-43ff-9f77-2af03cdd9bcf", "fence": 2, "attempt": 2}
{"scenario": "reorder_in_flight", "result": "PASS", "seconds": 14.563, "attachment_id": "7717096b-a8a2-45d1-ad9c-d178ad545d4d", "active_version": 2}
{"scenario": "late_old_delivery", "result": "PASS", "seconds": 16.117, "attachment_id": "0ee3fe7e-bd09-403d-8f6e-a7881ed2a1f0", "active_version": 2}
{"scenario": "atomic_version_switch", "result": "PASS", "seconds": 16.302, "attachment_id": "3a58381d-2c97-4728-b3f3-4b72457f3e35", "active_version": 2}
{"scenario": "temporary_error", "result": "PASS", "seconds": 8.366, "attachment_id": "7e9d1700-8dee-4622-9946-dc6bd2527a3b", "attempt": 2}
{"scenario": "permanent_error", "result": "PASS", "seconds": 9.06, "attachment_id": "7853eedb-f16a-421d-870e-f72496843dc3", "attempt": 2}
{"scenario": "restart_with_queued_attachment", "result": "PASS", "seconds": 23.39, "attachment_id": "e1baa3d5-bed2-48f7-ac29-5713b91b56d2", "attempt": 1, "message_link_preserved": true, "staging_cleaned": true}
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
# Compose file-based secrets inherit host ownership. Copy only the secrets
# mounted for this role before dropping privileges.
install -d -m 0700 -o appuser -g appuser /run/app-secrets
for key in DATABASE_URL_FILE S3_ACCESS_KEY_ID_FILE S3_SECRET_ACCESS_KEY_FILE BROKER_URL_FILE; do
source_file=$(printenv "$key") || continue
target="/run/app-secrets/$key"
install -m 0400 -o appuser -g appuser "$source_file" "$target"
export "$key=$target"
done
exec setpriv --reuid=appuser --regid=appuser --init-groups -- "$@"
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)"
exec docker-entrypoint.sh rabbitmq-server
+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 60s
timeout server 60s
timeout check 1s
# Never replay a potentially committed POST automatically.
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
default_backend replicas
backend replicas
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 1-10 api:8080 check
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local attachments-api /policy.json
mc admin policy attach local attachments-api --user "$(cat /run/secrets/s3_access_key_id)"
+34
View File
@@ -0,0 +1,34 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/results/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"originals/*", "staging/*", "results/*"
]
}
}
}
]
}
+11
View File
@@ -0,0 +1,11 @@
fastapi==0.115.12
uvicorn==0.34.2
pydantic==2.11.4
psycopg[binary]==3.2.6
psycopg-pool==3.2.6
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.38.12
Pillow==11.2.1
pika==1.3.2
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
fi
exec docker compose --env-file .env "$@"
+26
View File
@@ -0,0 +1,26 @@
"""Generate local credentials once; never overwrite an existing secret."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
directory = root / 'secrets'
directory.mkdir(mode=0o700, exist_ok=True)
values = {'broker_password': secrets.token_hex(24), 'postgres_password': secrets.token_hex(24), 's3_root_user': 'lab5root', 's3_root_password': secrets.token_hex(24), 's3_access_key_id': 'lab5api', 's3_secret_access_key': secrets.token_hex(24)}
for name, value in values.items():
path = directory / name
if not path.exists():
path.write_text(value + '\n')
path.chmod(0o600)
path = directory / 'database_url'
if not path.exists():
password = (directory / 'postgres_password').read_text().strip()
path.write_text(f'postgresql://messenger:{password}@postgres:5432/messenger\n')
path.chmod(0o600)
path = directory / 'broker_url'
if not path.exists():
password = (directory / 'broker_password').read_text().strip()
path.write_text(f'amqp://messenger:{password}@broker:5672/%2F\n')
path.chmod(0o600)
path = root / '.env'
if not path.exists():
path.write_text('COMPOSE_PROJECT_NAME=mtusi2026lab5\nPUBLIC_PORT=8080\n')
print('Local configuration ready (secret values suppressed)')
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p evidence
containers=$(./scripts/compose.sh ps -aq postgres redis s3 broker api lb publisher worker recovery)
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-before.txt
./scripts/compose.sh restart postgres redis s3 broker api lb publisher worker recovery
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-after.txt
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}"
for container in $(./scripts/compose.sh ps -q api); do
instance=$(docker exec "$container" python -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])')
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop -t 2 "$container" >/dev/null
exit 0
fi
done
exit 1
View File
+149
View File
@@ -0,0 +1,149 @@
import asyncio
import hashlib
import subprocess
import sys
import tempfile
import unicodedata
from uuid import UUID, uuid4
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import HTTPException, Request
from fastapi.responses import StreamingResponse
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from .database import get_connection
from .jobs import enqueue
from .storage import s3, BUCKET
MAX_BYTES = 10 * 1024**2
upload_slots = asyncio.Semaphore(1)
def serialize(row):
result = {key: str(row[key]) for key in ('id', 'chat_id', 'owner_id')}
result.update({key: row[key] for key in ('filename', 'content_type', 'size_bytes', 'sha256', 'status')})
result.update({key: row[key].isoformat().replace('+00:00', 'Z') for key in ('created_at', 'updated_at')})
manifest = row.get('manifest') or {}
variants = [
dict(name=name, **{field: obj[field] for field in
('content_type', 'size_bytes', 'sha256', 'width', 'height')})
for name, obj in sorted(manifest.items()) if name != 'original'
]
return dict(result, variants=variants, error_code=row['error_code'])
def persist(path, chat, owner, filename, mime, size, checksum):
attachment_id = uuid4()
key = 'staging/' + str(attachment_id)
# The same lock is taken by cleanup before its final DB check/deletion.
# A process crash releases the transaction; an uncommitted object is later cleaned.
with get_connection() as conn:
with conn.transaction():
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (key,))
with open(path, 'rb') as source:
s3.put_object(Bucket=BUCKET, Key=key, Body=source, ContentLength=size, ContentType=mime)
row = conn.execute('''INSERT INTO attachments
(id, chat_id, owner_id, filename, object_key, size_bytes, content_type, sha256, staging_key, status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *''',
(attachment_id, UUID(chat), UUID(owner), filename, key, size, mime, checksum, key)).fetchone()
enqueue(conn, attachment_id, 1)
# Exiting transaction above commits before returning 202.
return serialize(row)
def accessible(attachment_id, user_id):
try:
aid = UUID(attachment_id)
except ValueError:
raise HTTPException(400, 'Неверный UUID') from None
with get_connection() as conn:
row = conn.execute('''SELECT a.*, j.manifest FROM attachments a
LEFT JOIN image_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.active_version
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s''', (aid, UUID(user_id))).fetchone()
if row is None:
raise HTTPException(404, 'Вложение не найдено')
return row
def register(app, current_user, get_chat):
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
async def upload(chat_id: str, request: Request):
user = await run_in_threadpool(current_user, request)
chat = await run_in_threadpool(get_chat, chat_id, request)
filename = request.query_params.get('filename', '')
if (not 1 <= len(filename) <= 255 or not filename.strip()
or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename)):
raise HTTPException(400, 'Недопустимое имя файла')
mime = request.headers.get('content-type', '').split(';')[0].strip().lower()
if mime not in ('image/png', 'image/jpeg'):
raise HTTPException(415, 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isascii() or not length.isdecimal() or len(length) > 12:
raise HTTPException(400, 'Неверный размер')
if int(length) > MAX_BYTES:
raise HTTPException(413, 'Файл больше 10 MiB')
try:
await asyncio.wait_for(upload_slots.acquire(), timeout=30)
except asyncio.TimeoutError:
raise HTTPException(503, 'Загрузчик занят') from None
try:
with tempfile.NamedTemporaryFile(prefix='lab5-') as target:
size = 0
digest = hashlib.sha256()
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413, 'Файл больше 10 MiB')
target.write(chunk)
digest.update(chunk)
except ClientDisconnect:
raise HTTPException(400, 'Загрузка прервана') from None
except TimeoutError:
raise HTTPException(400, 'Истекло время загрузки') from None
target.flush()
def validate():
try:
return subprocess.run([sys.executable, '-m', 'src.image_check', target.name, mime],
timeout=10, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
except subprocess.TimeoutExpired:
return 15
code = await run_in_threadpool(validate)
if code != 0:
raise HTTPException(413 if code == 13 else 415, 'Недопустимое изображение')
return await run_in_threadpool(persist, target.name, chat['id'], user['id'], filename, mime, size, digest.hexdigest())
finally:
upload_slots.release()
@app.get('/api/v1/attachments/{attachment_id}')
def metadata(attachment_id: str, request: Request):
return serialize(accessible(attachment_id, current_user(request)['id']))
@app.get('/api/v1/attachments/{attachment_id}/content')
def content(attachment_id: str, request: Request):
row = accessible(attachment_id, current_user(request)['id'])
if row['status'] != 'ready':
raise HTTPException(409, 'Вложение не готово')
variant = row['manifest'].get(request.query_params.get('variant', 'original'))
if variant is None:
raise HTTPException(404, 'Вариант не найден')
obj = s3.get_object(Bucket=BUCKET, Key=variant['key'])
def chunks():
try:
for chunk in obj['Body'].iter_chunks(64 * 1024):
yield chunk
finally:
obj['Body'].close()
return StreamingResponse(chunks(), media_type=variant['content_type'], headers={
'Content-Length': str(variant['size_bytes']), 'X-Content-Type-Options': 'nosniff',
'Content-Disposition': 'attachment', 'Cache-Control': 'private, no-store'})
@app.exception_handler(BotoCoreError)
@app.exception_handler(ClientError)
async def storage_error(request, exc):
from .main import error_response
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
+91
View File
@@ -0,0 +1,91 @@
import base64
import binascii
import hashlib
import os
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerifyMismatchError
password_hasher = PasswordHasher(
time_cost=3,
memory_cost=65536,
parallelism=4,
)
def hash_password(password: str) -> str:
return password_hasher.hash(password)
def verify_password(password_hash: str, password: str) -> bool:
try:
return password_hasher.verify(password_hash, password)
except (VerifyMismatchError, InvalidHashError):
return False
def parse_basic(authorization: str | None) -> tuple[str, str] | None:
if not authorization:
return None
scheme, separator, credentials = authorization.partition(" ")
if not separator or scheme.lower() != "basic":
return None
try:
raw = base64.b64decode(credentials, validate=True)
decoded = raw.decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError, binascii.Error):
return None
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
return None
if not 12 <= len(password) <= 128:
return None
return username, password
def parse_bearer(authorization: str | None) -> str | None:
if not authorization:
return None
scheme, separator, token = authorization.partition(" ")
if not separator or scheme.lower() != "bearer":
return None
if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512:
return None
return token
def create_token() -> str:
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
return hashlib.sha256(token.encode("ascii")).hexdigest()
def session_expiration() -> datetime:
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if ttl < 1:
raise ValueError("SESSION_TTL_SECONDS must be positive")
return datetime.now(timezone.utc) + timedelta(seconds=ttl)
def format_datetime(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(
timespec="microseconds"
).replace("+00:00", "Z")
+33
View File
@@ -0,0 +1,33 @@
"""Persistent RabbitMQ transport with explicit publisher confirms."""
import json
import os
from contextlib import contextmanager
from pathlib import Path
import pika
QUEUE = 'attachment.process.v1'
@contextmanager
def channel():
params = pika.URLParameters(Path(os.environ['BROKER_URL_FILE']).read_text().strip())
params.connection_attempts = 1
params.socket_timeout = 4
params.stack_timeout = 8
params.blocked_connection_timeout = 10
params.heartbeat = 120
connection = pika.BlockingConnection(params)
try:
ch = connection.channel()
ch.queue_declare(QUEUE, durable=True, arguments={'x-queue-type': 'quorum', 'x-delivery-limit': -1})
yield connection, ch
finally:
if connection.is_open:
connection.close()
def send(ch, message):
ch.basic_publish('', QUEUE, json.dumps(message).encode(), mandatory=True,
properties=pika.BasicProperties(content_type='application/json', delivery_mode=2,
message_id=message['job_id']))
+254
View File
@@ -0,0 +1,254 @@
from uuid import UUID, uuid4
from .database import get_connection
def serialize_chat(row):
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(member_id) for member_id in row["member_ids"]],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def serialize_message(row):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"attachment_ids": [str(a) for a in row["attachment_ids"]],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def create_chat(creator_id, title, member_ids):
chat_id = uuid4()
members = [UUID(member_id) for member_id in member_ids]
with get_connection() as conn:
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM users
WHERE id = ANY(%s::uuid[])
""",
(members,),
)
existing = {row["id"] for row in cur.fetchall()}
if len(existing) != len(members):
return None
cur.execute(
"""
INSERT INTO chats (id, title, created_by)
VALUES (%s, %s, %s)
RETURNING id, title, created_at
""",
(chat_id, title, UUID(creator_id)),
)
chat = cur.fetchone()
cur.executemany(
"""
INSERT INTO chat_members (chat_id, user_id)
VALUES (%s, %s)
""",
[(chat_id, member_id) for member_id in members],
)
return {
"id": str(chat["id"]),
"title": chat["title"],
"member_ids": member_ids,
"created_at": chat["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def get_chat(chat_id, user_id):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
c.id,
c.title,
c.created_at,
ARRAY(
SELECT cm2.user_id
FROM chat_members cm2
WHERE cm2.chat_id = c.id
ORDER BY cm2.user_id
) AS member_ids
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE c.id = %s
AND cm.user_id = %s
""",
(UUID(chat_id), UUID(user_id)),
)
row = cur.fetchone()
return serialize_chat(row) if row else None
def list_chats(user_id, limit, last_chat_id=None):
with get_connection() as conn:
with conn.cursor() as cur:
last_key = None
if last_chat_id is not None:
cur.execute(
"""
SELECT c.created_at, c.id
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE c.id = %s
AND cm.user_id = %s
""",
(UUID(last_chat_id), UUID(user_id)),
)
last_key = cur.fetchone()
if last_key is None:
return None
cur.execute(
"""
SELECT
c.id,
c.title,
c.created_at,
ARRAY(
SELECT cm2.user_id
FROM chat_members cm2
WHERE cm2.chat_id = c.id
ORDER BY cm2.user_id
) AS member_ids
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE cm.user_id = %s
AND (
%s::timestamptz IS NULL
OR (c.created_at, c.id) >
(%s::timestamptz, %s::uuid)
)
ORDER BY c.created_at, c.id
LIMIT %s
""",
(
UUID(user_id),
last_key["created_at"] if last_key else None,
last_key["created_at"] if last_key else None,
last_key["id"] if last_key else None,
limit + 1,
),
)
rows = cur.fetchall()
return [serialize_chat(row) for row in rows]
def create_message(chat_id, sender_id, text, key, attachment_ids):
# INSERT ON CONFLICT waits for a concurrent transaction on the same key.
# READ COMMITTED's next statement sees its committed result. Both the
# message and deduplication record commit together, or both roll back.
with get_connection() as conn:
with conn.transaction():
attachments = [UUID(a) for a in attachment_ids]
scope = (UUID(chat_id), UUID(sender_id), key)
conn.execute("""
DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s
AND created_at <= clock_timestamp() - interval '24 hours'
""", scope)
row = conn.execute(
"""INSERT INTO message_requests(chat_id, sender_id, key, text)
VALUES (%s, %s, %s, %s)
ON CONFLICT (chat_id, sender_id, key) DO NOTHING
RETURNING key""", (*scope, text)
).fetchone()
if row is None:
previous = conn.execute(
"""SELECT m.* FROM message_requests r
JOIN messages m ON m.id = r.message_id
WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""",
scope,
).fetchone()
if previous["text"] != text or previous["attachment_ids"] != attachments:
return "conflict"
return serialize_message(previous)
rows = conn.execute(
'SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) '
'ORDER BY id FOR SHARE', (UUID(chat_id), attachments)
).fetchall()
if len(rows) != len(attachments) or any(row['status'] == 'failed' for row in rows):
# No unfinished idempotency record may survive validation failure.
conn.execute('DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s', scope)
return None if len(rows) != len(attachments) else "attachment_failed"
message = conn.execute(
"""INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids)
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
(uuid4(), UUID(chat_id), UUID(sender_id), text, attachments),
).fetchone()
conn.execute(
"""UPDATE message_requests SET message_id=%s
WHERE chat_id=%s AND sender_id=%s AND key=%s""",
(message["id"], *scope),
)
return serialize_message(message)
def list_messages(chat_id, limit, last_message_id=None):
with get_connection() as conn:
with conn.cursor() as cur:
last_key = None
if last_message_id is not None:
cur.execute(
"""
SELECT created_at, id
FROM messages
WHERE chat_id = %s
AND id = %s
""",
(UUID(chat_id), UUID(last_message_id)),
)
last_key = cur.fetchone()
if last_key is None:
return None
# Разные формы SQL сохраняют индексный range scan и для generic plan.
# В SQL подставляется только константа; значения остаются параметрами.
cursor_condition = ""
params = [UUID(chat_id)]
if last_key is not None:
cursor_condition = "AND (created_at, id) > (%s, %s)"
params.extend([last_key["created_at"], last_key["id"]])
params.append(limit + 1)
cur.execute(
f"""
SELECT id, chat_id, sender_id, text, attachment_ids, created_at
FROM messages
WHERE chat_id = %s
{cursor_condition}
ORDER BY created_at, id
LIMIT %s
""",
params,
)
rows = cur.fetchall()
return [serialize_message(row) for row in rows]
+48
View File
@@ -0,0 +1,48 @@
import os
from contextlib import contextmanager
from pathlib import Path
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
def get_database_url() -> str:
secret_file = os.getenv("DATABASE_URL_FILE")
if secret_file:
return Path(secret_file).read_text().strip()
database_url = os.getenv("DATABASE_URL")
if database_url:
return database_url
raise RuntimeError("Database connection is not configured")
pool = ConnectionPool(
conninfo=get_database_url(),
min_size=1,
max_size=10,
timeout=5,
open=False,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": "-c statement_timeout=5000",
},
)
def open_database():
pool.open(wait=True, timeout=15)
def close_database():
pool.close()
@contextmanager
def get_connection():
with pool.connection() as connection:
yield connection
+42
View File
@@ -0,0 +1,42 @@
"""One-shot fault controls, enabled only by the operator on a disposable stand."""
import os
import time
from .database import get_connection
ENABLED = os.getenv('PIPELINE_FAULTS') == '1'
def take(conn, attachment_id, version, point):
if not ENABLED:
return None
row = conn.execute("""
UPDATE pipeline_faults SET hit_at=clock_timestamp()
WHERE attachment_id=%s AND pipeline_version=%s AND point=%s AND hit_at IS NULL
RETURNING action
""", (attachment_id, version, point)).fetchone()
return row['action'] if row else None
def checkpoint(job, point):
if not ENABLED:
return None
key = (job['attachment_id'], job['pipeline_version'], point)
with get_connection() as conn:
action = take(conn, *key)
if action == 'crash':
print(f'fault {point}: crash {job["id"]}', flush=True)
os._exit(86)
if action == 'pause':
print(f'fault {point}: pause {job["id"]}', flush=True)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
with get_connection() as conn:
row = conn.execute("""
SELECT released FROM pipeline_faults
WHERE attachment_id=%s AND pipeline_version=%s AND point=%s
""", key).fetchone()
if row is None or row['released']:
break
time.sleep(0.1)
return action
+25
View File
@@ -0,0 +1,25 @@
"""Isolated image decoder: bounded address space, CPU and parent wall timeout."""
import resource
import sys
import warnings
if sys.platform.startswith('linux'):
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (8, 8))
from PIL import Image
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.simplefilter('error', Image.DecompressionBombWarning)
try:
with Image.open(sys.argv[1]) as image:
if image.width * image.height > 40_000_000:
sys.exit(13)
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
if actual != sys.argv[2] or getattr(image, 'n_frames', 1) != 1:
sys.exit(15)
image.verify()
with Image.open(sys.argv[1]) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
sys.exit(13)
except Exception:
sys.exit(15)
+154
View File
@@ -0,0 +1,154 @@
"""PostgreSQL owns execution state and the publication pointer."""
import os
import random
from psycopg.types.json import Jsonb
from .database import get_connection
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
LEASE_SECONDS = int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))
def enqueue(conn, attachment_id, version):
job = conn.execute('''
INSERT INTO image_jobs(attachment_id,pipeline_version) VALUES (%s,%s) RETURNING *
''', (attachment_id, version)).fetchone()
schedule(conn, job['id'])
return job
def schedule(conn, job_id, delay=0):
conn.execute('''
INSERT INTO job_outbox(job_id,publish_after)
VALUES (%s, clock_timestamp() + %s * interval '1 second')
ON CONFLICT (job_id) DO UPDATE SET publish_after=excluded.publish_after
''', (job_id, delay))
def event(conn, job, name, detail=None):
conn.execute('INSERT INTO job_events(job_id,attempt,event,detail) VALUES (%s,%s,%s,%s)',
(job['id'], job['attempt'], name, detail))
def carrier(job):
return dict(schema_version=1, type='attachment.process.v1', job_id=str(job['id']),
attachment_id=str(job['attachment_id']), pipeline_version=job['pipeline_version'],
attempt=job['attempt'], created_at=job['created_at'].isoformat().replace('+00:00', 'Z'))
def claim(message):
with get_connection() as conn:
job = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE',
(message['job_id'],)).fetchone()
if (job is None or job['attachment_id'] != message['attachment_id']
or job['pipeline_version'] != message['pipeline_version']
or job['attempt'] != message['attempt']):
return None
if job['state'] != 'queued':
event(conn, job, 'delivery_ignored', job['state'])
return None
attachment = lock_attachment(conn, job)
if attachment['desired_version'] != job['pipeline_version']:
supersede(conn, job)
return None
job = conn.execute('''
UPDATE image_jobs SET state='processing', fence=fence+1,
lease_until=clock_timestamp() + %s * interval '1 second'
WHERE id=%s RETURNING *
''', (LEASE_SECONDS, job['id'])).fetchone()
update_attachment(conn, job, 'processing')
event(conn, job, 'claimed', str(job['fence']))
return job, attachment
def lock_attachment(conn, job):
return conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE',
(job['attachment_id'],)).fetchone()
def update_attachment(conn, job, state, error=None):
# While building a new version, continue serving the previously published version.
conn.execute('''
UPDATE attachments SET status=%s, error_code=%s, updated_at=clock_timestamp()
WHERE id=%s AND desired_version=%s AND active_version IS NULL
''', (state, error, job['attachment_id'], job['pipeline_version']))
def supersede(conn, job):
conn.execute("UPDATE image_jobs SET state='superseded', lease_until=NULL WHERE id=%s", (job['id'],))
event(conn, job, 'superseded')
def owned(conn, job):
current = conn.execute('''
SELECT *, lease_until > clock_timestamp() AS live FROM image_jobs WHERE id=%s FOR NO KEY UPDATE
''', (job['id'],)).fetchone()
return current['state'] == 'processing' and current['fence'] == job['fence'] and current['live']
def finish(job, manifest):
with get_connection() as conn:
if not owned(conn, job):
event(conn, job, 'fence_rejected', str(job['fence']))
return
attachment = lock_attachment(conn, job)
if not owned(conn, job):
event(conn, job, 'fence_rejected', str(job['fence']))
return
if attachment['desired_version'] != job['pipeline_version']:
supersede(conn, job)
return
conn.execute("""
UPDATE image_jobs SET state='ready', manifest=%s, lease_until=NULL,error_code=NULL WHERE id=%s
""", (Jsonb(manifest), job['id']))
conn.execute("""
UPDATE attachments SET status='ready', active_version=%s, error_code=NULL,
object_key=%s, updated_at=clock_timestamp() WHERE id=%s
""", (job['pipeline_version'], manifest['original']['key'], attachment['id']))
event(conn, job, 'published')
def fail_locked(conn, job, reason, permanent=False):
attachment = lock_attachment(conn, job)
if attachment['desired_version'] != job['pipeline_version']:
supersede(conn, job)
return
failures = job['failures'] + 1
if permanent or failures >= MAX_ATTEMPTS:
code = reason if permanent else 'retry_exhausted'
conn.execute("""
UPDATE image_jobs SET state='failed', error_code=%s, lease_until=NULL, failures=%s WHERE id=%s
""", (code, failures, job['id']))
conn.execute('INSERT INTO job_dead_letters(job_id,attempt,error_code) VALUES (%s,%s,%s)',
(job['id'], job['attempt'], code))
update_attachment(conn, job, 'failed', code)
event(conn, job, 'dead_letter', reason)
return
delay = min(30, 2 ** (failures - 1)) * random.uniform(0.75, 1.25)
conn.execute("""
UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=%s,
lease_until=NULL,error_code=%s WHERE id=%s
""", (failures, reason, job['id']))
schedule(conn, job['id'], delay)
update_attachment(conn, job, 'queued')
event(conn, job, 'retry_scheduled', reason)
def fail(job, reason, permanent=False):
with get_connection() as conn:
if owned(conn, job):
fail_locked(conn, job, reason, permanent)
else:
event(conn, job, 'fence_rejected', str(job['fence']))
def recover_one():
with get_connection() as conn:
job = conn.execute("""
SELECT * FROM image_jobs WHERE state='processing' AND lease_until <= clock_timestamp()
ORDER BY lease_until LIMIT 1 FOR NO KEY UPDATE SKIP LOCKED
""").fetchone()
if job:
fail_locked(conn, job, 'lease_expired')
return job is not None
+519
View File
@@ -0,0 +1,519 @@
"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили."""
import base64
import binascii
import json
from pathlib import Path
import logging
import re
from uuid import UUID, uuid4
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from psycopg import OperationalError, Error as DatabaseError
from psycopg_pool import PoolTimeout
from .auth import (
create_token,
format_datetime,
hash_password,
hash_token,
parse_basic,
parse_bearer,
session_expiration,
verify_password,
)
from .database import open_database, close_database, get_connection
from .user_repository import (
create_user as insert_user,
create_session,
find_session_user_cached,
find_user_by_username,
revoke_session,
serialize_user,
)
from .chat_repository import (
create_chat as insert_chat,
get_chat as find_chat,
list_chats as find_chats,
create_message as insert_message,
list_messages as find_messages,
)
app = FastAPI(title="MTUSI Messenger", version="1.5.0")
# Serve the mandatory contract in Swagger, including auth and request schemas.
OPENAPI_SCHEMA = json.loads(
(Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text()
)
app.openapi = lambda: OPENAPI_SCHEMA
INSTANCE_ID = str(uuid4())
logger = logging.getLogger("uvicorn.error")
ERROR_CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
500: "internal_error",
503: "unavailable",
}
def error_response(request, status_code, code, message):
return JSONResponse(
status_code=status_code,
content={
"error": {
"code": code,
"message": message,
"request_id": request.state.request_id,
}
},
)
@app.middleware("http")
async def request_id_middleware(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
response = await call_next(request)
except Exception:
# Логируем путь, метод и request_id, но НЕ тело сообщения.
await run_in_threadpool(
logger.error,
"Unhandled error: %s %s request_id=%s",
request.method,
request.url.path,
request.state.request_id,
)
response = error_response(
request, 500, "internal_error", "Внутренняя ошибка сервера"
)
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(request: Request, exc: StarletteHTTPException):
response = error_response(
request,
exc.status_code,
ERROR_CODES.get(exc.status_code, "invalid_request"),
str(exc.detail),
)
if exc.headers:
response.headers.update(exc.headers)
return response
@app.exception_handler(DatabaseError)
@app.exception_handler(PoolTimeout)
async def database_error_handler(request: Request, exc):
return error_response(request, 503, "unavailable", "База данных временно недоступна")
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные данные запроса")
async def read_json(request: Request):
"""415 — только для неправильного Content-Type; 400 — для неверного JSON."""
content_type = (
request.headers.get("content-type", "").split(";")[0].strip().lower()
)
if content_type != "application/json":
raise HTTPException(415, "Неподдерживаемый Content-Type")
try:
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > 65536:
raise HTTPException(413, "JSON слишком большой")
raw.extend(chunk)
data = json.loads(raw)
except (ValueError, UnicodeError):
raise HTTPException(400, "Неверный JSON") from None
# [], строка, число, true, false, null — это корректный JSON,
# но НЕ объект, обязательный по контракту.
if not isinstance(data, dict):
raise HTTPException(400, "Ожидается JSON-объект")
return data
def get_current_user(request: Request):
token = parse_bearer(request.headers.get("Authorization"))
if token is None:
raise HTTPException(
401,
"Требуется действующий Bearer-токен",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
try:
user = find_session_user_cached(hash_token(token))
except (OperationalError, PoolTimeout):
raise HTTPException(
503,
"База данных временно недоступна",
) from None
if user is None:
raise HTTPException(
401,
"Недействительная или истёкшая сессия",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def get_limit(request: Request):
"""Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500."""
limit_raw = request.query_params.get("limit", "50")
if not limit_raw.isascii() or not limit_raw.isdecimal():
raise HTTPException(400, "Неверный limit")
significant = limit_raw.lstrip("0")
if not significant or len(significant) > 3:
raise HTTPException(400, "Неверный limit")
limit = int(significant) # Теперь строка содержит максимум 3 цифры.
if not 1 <= limit <= 100:
raise HTTPException(400, "limit должен быть от 1 до 100")
return limit
def encode_cursor(first_id, last_id):
raw = f"{first_id}|{last_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(cursor):
try:
if not cursor or len(cursor) > 256:
raise ValueError("empty or too long")
padded = cursor + "=" * (-len(cursor) % 4)
raw = base64.b64decode(
padded, altchars=b"-_", validate=True
).decode("ascii")
first_id, last_id = raw.split("|")
return str(UUID(first_id)), str(UUID(last_id))
except (ValueError, UnicodeError, binascii.Error):
raise HTTPException(400, "Неверный cursor") from None
# ---- Health ----
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
try:
with get_connection() as conn:
conn.execute("SELECT 1")
from .storage import s3, BUCKET
s3.list_objects_v2(Bucket=BUCKET, Prefix="originals/", MaxKeys=1)
except (OperationalError, PoolTimeout):
raise HTTPException(503, "База данных недоступна") from None
return {"status": "ok"}
# ---- Users ----
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
data = await read_json(request)
if set(data) != {"username", "display_name", "password"}:
raise HTTPException(400, "Неверные поля пользователя")
username = data["username"]
display_name = data["display_name"]
password = data["password"]
if not isinstance(username, str) or not re.fullmatch(
r"[a-z0-9_]{3,32}", username
):
raise HTTPException(400, "Недопустимый username")
if (
not isinstance(display_name, str)
or not 1 <= len(display_name) <= 100
or not display_name.strip()
):
raise HTTPException(400, "Недопустимое имя пользователя")
if not isinstance(password, str) or not 12 <= len(password) <= 128:
raise HTTPException(400, "Пароль должен содержать 12–128 символов")
password_hash = await run_in_threadpool(hash_password, password)
user = await run_in_threadpool(insert_user, username, display_name, password_hash)
if user is None:
raise HTTPException(409, "Username уже занят")
return user
@app.get("/api/v1/users/me")
def get_me(request: Request):
return get_current_user(request)
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
credentials = parse_basic(request.headers.get("Authorization"))
if credentials is None:
raise HTTPException(
401,
"Неверные учётные данные",
headers={
"WWW-Authenticate":
'Basic realm="messenger", charset="UTF-8"'
},
)
username, password = credentials
user_row = find_user_by_username(username)
if user_row is None or not verify_password(
user_row["password_hash"], password
):
raise HTTPException(
401,
"Неверные учётные данные",
headers={
"WWW-Authenticate":
'Basic realm="messenger", charset="UTF-8"'
},
)
token = create_token()
expires_at = session_expiration()
create_session(user_row["id"], hash_token(token), expires_at)
return {
"token": token,
"token_type": "Bearer",
"expires_at": format_datetime(expires_at),
"user": serialize_user(user_row),
}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
get_current_user(request)
token = parse_bearer(request.headers.get("Authorization"))
from .session_cache import invalidate_session
if not revoke_session(hash_token(token)):
raise HTTPException(
401,
"Недействительная сессия",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
invalidate_session(hash_token(token))
from fastapi import Response
return Response(status_code=204)
# ---- Chats ----
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await run_in_threadpool(get_current_user, request)
data = await read_json(request)
if set(data) != {"title", "member_ids"}:
raise HTTPException(400, "Неверные поля чата")
title = data["title"]
member_ids = data["member_ids"]
if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip():
raise HTTPException(400, "Недопустимое название чата")
if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100:
raise HTTPException(400, "Неверный список участников")
normalized_ids = []
for member_id in member_ids:
if not isinstance(member_id, str):
raise HTTPException(400, "Неверный ID участника")
try:
normalized_ids.append(str(UUID(member_id)))
except ValueError:
raise HTTPException(400, "Неверный UUID участника") from None
if len(normalized_ids) != len(set(normalized_ids)):
raise HTTPException(400, "Участники повторяются")
if user["id"] not in normalized_ids:
raise HTTPException(400, "Создатель должен быть участником")
chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids)
if chat is None:
raise HTTPException(400, "Неизвестный участник")
return chat
@app.get("/api/v1/chats")
def list_chats(request: Request):
user = get_current_user(request)
limit = get_limit(request)
cursor = request.query_params.get("cursor")
last_chat_id = None
if cursor is not None:
cursor_user_id, last_chat_id = decode_cursor(cursor)
if cursor_user_id != user["id"]:
raise HTTPException(400, "Неверный cursor")
rows = find_chats(user["id"], limit, last_chat_id)
if rows is None:
raise HTTPException(400, "Неверный cursor")
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = (
encode_cursor(user["id"], page[-1]["id"])
if has_more and page
else None
)
return {"items": page, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
user = get_current_user(request)
try:
chat_id = str(UUID(chat_id))
except ValueError:
raise HTTPException(400, "Неверный UUID чата") from None
chat = find_chat(chat_id, user["id"])
if chat is None:
raise HTTPException(404, "Чат не найден")
return chat
# ---- Messages ----
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def send_message(chat_id: str, request: Request):
user = await run_in_threadpool(get_current_user, request)
chat = await run_in_threadpool(get_chat, chat_id, request)
data = await read_json(request)
if "text" not in data or set(data) - {"text", "attachment_ids"}:
raise HTTPException(400, "Неверные поля сообщения")
text = data["text"]
attachment_ids = data.get("attachment_ids", [])
if not isinstance(attachment_ids, list) or len(attachment_ids) > 10:
raise HTTPException(400, "Неверный список вложений")
try:
if any(not isinstance(a, str) for a in attachment_ids):
raise ValueError()
attachment_ids = [str(UUID(a)) for a in attachment_ids]
if len(set(attachment_ids)) != len(attachment_ids):
raise ValueError()
except ValueError:
raise HTTPException(400, "Неверные ID вложений") from None
if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids):
raise HTTPException(400, "Недопустимый текст сообщения")
key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise HTTPException(400, "Неверный Idempotency-Key")
message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key, attachment_ids)
if message == "attachment_failed":
raise HTTPException(409, "Обработка вложения завершилась ошибкой")
if message == "conflict":
raise HTTPException(409, "Ключ использован для другого сообщения")
if message is None:
raise HTTPException(404, "Чат не найден")
return message
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request):
chat = get_chat(chat_id, request)
limit = get_limit(request)
cursor = request.query_params.get("cursor")
last_message_id = None
if cursor is not None:
cursor_chat_id, last_message_id = decode_cursor(cursor)
if cursor_chat_id != chat["id"]:
raise HTTPException(400, "Неверный cursor")
rows = find_messages(chat["id"], limit, last_message_id)
if rows is None:
raise HTTPException(400, "Неверный cursor")
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = (
encode_cursor(chat["id"], page[-1]["id"])
if has_more and page
else None
)
return {"items": page, "next_cursor": next_cursor}
@app.on_event("startup")
async def start_database_pool():
await run_in_threadpool(open_database)
@app.on_event("shutdown")
async def stop_database_pool():
await run_in_threadpool(close_database)
from .attachments import register
register(app, get_current_user, get_chat)
+16
View File
@@ -0,0 +1,16 @@
"""One-shot migrations, serialized and recorded in PostgreSQL."""
from pathlib import Path
import psycopg
from .database import get_database_url
with psycopg.connect(get_database_url()) as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(310026)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())")
for path in sorted((Path(__file__).resolve().parents[1] / "db/migrations").glob("*.sql")):
if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
conn.execute(path.read_text())
conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,))
print("Applied", path.name, flush=True)
else:
print("Already applied", path.name, flush=True)
+91
View File
@@ -0,0 +1,91 @@
"""Local administrative commands. Run from the trusted Compose project."""
import argparse
import json
from .broker import channel, send
from .database import close_database, get_connection, open_database
from .jobs import carrier, enqueue, event, schedule
def reprocess(attachment_id, version):
with get_connection() as conn:
row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE', (attachment_id,)).fetchone()
if not row or version <= row['desired_version']:
raise ValueError('Version must increase for an existing attachment')
if row['active_version'] is None and row['staging_key'] is None and row['object_key'].startswith('staging/'):
raise ValueError('Original expired')
conn.execute("""
UPDATE attachments SET desired_version=%s,
status=CASE WHEN active_version IS NULL THEN 'queued' ELSE status END,
error_code=NULL,updated_at=clock_timestamp() WHERE id=%s
""", (version, attachment_id))
job = enqueue(conn, attachment_id, version)
event(conn, job, 'reprocess_requested')
return carrier(job)
def replay(job_id):
with get_connection() as conn:
row = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE', (job_id,)).fetchone()
if not row or row['state'] != 'failed':
raise ValueError('Replay requires a failed job')
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE',
(row['attachment_id'],)).fetchone()
if attachment['desired_version'] != row['pipeline_version']:
raise ValueError('Replay requires the current pipeline version')
if attachment['staging_key'] is None and attachment['object_key'].startswith('staging/'):
raise ValueError('Staging retention expired')
conn.execute("""
UPDATE job_dead_letters SET replayed_at=clock_timestamp() WHERE job_id=%s AND attempt=%s
""", (job_id, row['attempt']))
updated = conn.execute("""
UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=0,
error_code=NULL WHERE id=%s RETURNING *
""", (job_id,)).fetchone()
conn.execute("""
UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp()
WHERE id=%s AND active_version IS NULL
""", (row['attachment_id'],))
schedule(conn, job_id)
event(conn, updated, 'operator_replay')
return carrier(updated)
def redeliver(job_id):
with get_connection() as conn:
row = conn.execute('SELECT * FROM image_jobs WHERE id=%s', (job_id,)).fetchone()
if not row:
raise ValueError('Job not found')
message = carrier(row)
with channel() as (_, ch):
ch.confirm_delivery()
send(ch, message)
return message
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
new = commands.add_parser('reprocess')
new.add_argument('attachment_id')
new.add_argument('version', type=int, choices=[1, 2])
commands.add_parser('replay').add_argument('job_id')
commands.add_parser('redeliver').add_argument('job_id')
args = parser.parse_args()
open_database()
try:
if args.command == 'reprocess':
result = reprocess(args.attachment_id, args.version)
elif args.command == 'replay':
result = replay(args.job_id)
else:
result = redeliver(args.job_id)
print(json.dumps(result))
except ValueError as exc:
parser.exit(2, f'{exc}\n')
finally:
close_database()
if __name__ == '__main__':
main()
+29
View File
@@ -0,0 +1,29 @@
"""Check recent progress by the service loop in this container."""
from pathlib import Path
import time
PROGRESS = Path('/tmp/pipeline-progress')
MAX_IDLE_SECONDS = 60
_last_write = float('-inf')
def mark_progress():
global _last_write
now = time.monotonic()
if now - _last_write >= 1:
PROGRESS.touch()
_last_write = now
def clear_progress():
global _last_write
PROGRESS.unlink(missing_ok=True)
_last_write = float('-inf')
if __name__ == '__main__':
try:
healthy = time.time() - PROGRESS.stat().st_mtime < MAX_IDLE_SECONDS
except FileNotFoundError:
healthy = False
raise SystemExit(0 if healthy else 1)
+31
View File
@@ -0,0 +1,31 @@
import logging
import signal
from threading import Event
import pika
from psycopg import OperationalError
from psycopg_pool import PoolTimeout
from botocore.exceptions import BotoCoreError, ClientError
from .pipeline_health import mark_progress, clear_progress
from .database import open_database, close_database
def serve(loop):
logging.basicConfig(level=logging.WARNING, format='%(asctime)s %(levelname)s %(message)s')
stopping = Event()
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: stopping.set())
clear_progress()
open_database()
try:
while not stopping.is_set():
try:
loop(stopping, mark_progress)
except (OperationalError, PoolTimeout, BotoCoreError, ClientError, pika.exceptions.AMQPError, OSError) as exc:
clear_progress()
logging.warning('dependency failure: %s', type(exc).__name__)
stopping.wait(1)
finally:
clear_progress()
close_database()
+50
View File
@@ -0,0 +1,50 @@
"""Drain committed outbox rows, deleting only after a positive broker confirm."""
import logging
import os
from .broker import channel, send
from .database import get_connection
from .faults import take
from .jobs import carrier, event
from .process import serve
def publish_one(ch):
with get_connection() as conn:
job = conn.execute("""
SELECT j.* FROM job_outbox o JOIN image_jobs j ON j.id=o.job_id
WHERE o.publish_after <= clock_timestamp()
ORDER BY o.publish_after LIMIT 1 FOR UPDATE OF o SKIP LOCKED
""").fetchone()
if job is None:
return False
if job['state'] != 'queued':
conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],))
return True
args = (job['attachment_id'], job['pipeline_version'])
crash_before = take(conn, *args, 'before_publish') == 'crash'
if not crash_before:
send(ch, carrier(job))
crash_after = take(conn, *args, 'after_confirm') == 'crash'
if not crash_after:
conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],))
event(conn, job, 'broker_confirmed')
if crash_before or crash_after:
point = 'before_publish' if crash_before else 'after_confirm'
logging.warning('fault %s: crash %s', point, job['id'])
os._exit(86)
return True
def loop(stopping, progress):
with channel() as (connection, ch):
ch.confirm_delivery()
while not stopping.is_set():
if not publish_one(ch):
connection.process_data_events(time_limit=0.2)
stopping.wait(0.1)
progress()
if __name__ == '__main__':
serve(loop)
+50
View File
@@ -0,0 +1,50 @@
"""Reclaim expired jobs and remove staging after safe publication or retention."""
from datetime import datetime, timedelta, timezone
import os
import time
from .database import get_connection
from .jobs import recover_one
from .process import serve
from .storage import BUCKET, s3
RETENTION = int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))
def cleanup():
cutoff = datetime.now(timezone.utc) - timedelta(seconds=RETENTION)
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='staging/'):
for obj in page.get('Contents', []):
key = obj['Key']
with get_connection() as conn:
locked = conn.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS ok',
(key,)).fetchone()['ok']
if not locked:
continue
row = conn.execute("""
SELECT id,status,active_version,updated_at FROM attachments WHERE staging_key=%s FOR UPDATE
""", (key,)).fetchone()
if row:
if not (row['active_version'] or (row['status'] == 'failed' and row['updated_at'] < cutoff)):
continue
elif obj['LastModified'] >= cutoff:
continue
s3.delete_object(Bucket=BUCKET, Key=key)
if row:
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s', (row['id'],))
def loop(stopping, progress):
next_cleanup = 0
while not stopping.is_set():
while not stopping.is_set() and recover_one():
progress()
if time.monotonic() >= next_cleanup:
cleanup()
next_cleanup = time.monotonic() + 15
progress()
stopping.wait(0.5)
if __name__ == '__main__':
serve(loop)
+49
View File
@@ -0,0 +1,49 @@
"""Bounded subprocess for untrusted image decoding and resizing."""
import resource
import sys
import warnings
from pathlib import Path
from PIL import Image, ImageOps
# Linux runtime. macOS does not implement RLIMIT_AS with the same semantics.
if sys.platform.startswith('linux'):
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.simplefilter('error', Image.DecompressionBombWarning)
def fit(image, bound):
width, height = image.size
largest = max(width, height)
if largest <= bound:
return image
return image.resize((max(1, width * bound // largest), max(1, height * bound // largest)),
Image.Resampling.LANCZOS)
def render(directory, version):
with Image.open(directory / 'original') as source:
image = ImageOps.exif_transpose(source)
if 'A' in image.getbands() or 'transparency' in image.info:
rgba = image.convert('RGBA')
del image
image = Image.new('RGB', rgba.size, 'white')
image.paste(rgba, mask=rgba.getchannel('A'))
del rgba
else:
image = image.convert('RGB')
image.info.clear()
jpeg_quality, webp_quality = {1: (85, 82), 2: (90, 88)}[version]
fit(image, 256).save(directory / 'thumbnail', format='JPEG', quality=jpeg_quality)
ImageOps.fit(image, (128, 128), Image.Resampling.LANCZOS).save(
directory / 'crop', format='JPEG', quality=jpeg_quality)
fit(image, 1280).save(directory / 'optimized', format='WEBP', quality=webp_quality, method=4)
if __name__ == '__main__':
try:
render(Path(sys.argv[1]), int(sys.argv[2]))
except (OSError, ValueError, Image.DecompressionBombError, Image.DecompressionBombWarning):
sys.exit(15)
+48
View File
@@ -0,0 +1,48 @@
import json
import os
from pathlib import Path
import redis
def get_cache_url():
secret_file = os.getenv("CACHE_URL_FILE")
if secret_file:
return Path(secret_file).read_text().strip()
return os.getenv("CACHE_URL", "redis://redis:6379/0")
client = redis.Redis.from_url(
get_cache_url(),
socket_connect_timeout=1,
socket_timeout=1,
decode_responses=True,
)
def get_cached_user(token_hash):
try:
value = client.get("session:" + token_hash)
return json.loads(value) if value else None
except (redis.RedisError, ValueError):
return None
def cache_user(token_hash, user, ttl):
if ttl <= 0:
return
try:
client.setex(
"session:" + token_hash,
ttl,
json.dumps(user),
)
except redis.RedisError:
pass
def invalidate_session(token_hash):
try:
client.delete("session:" + token_hash)
except redis.RedisError:
pass
+19
View File
@@ -0,0 +1,19 @@
"""Private S3 access; no automatic PUT retries, explicit bounded timeouts."""
import os
from pathlib import Path
import boto3
from botocore.config import Config
def secret(name):
return Path(os.environ[name + '_FILE']).read_text().strip()
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
s3 = boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'}),
)
+136
View File
@@ -0,0 +1,136 @@
from uuid import uuid4
from psycopg.errors import UniqueViolation
from .database import get_connection
def serialize_user(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def create_user(username, display_name, password_hash):
with get_connection() as conn:
with conn.cursor() as cur:
try:
cur.execute(
"""
INSERT INTO users (
id, username, display_name, password_hash
)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(uuid4(), username, display_name, password_hash),
)
return serialize_user(cur.fetchone())
except UniqueViolation:
conn.rollback()
return None
def find_user_by_username(username):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, username, display_name,
password_hash, created_at
FROM users
WHERE username = %s
""",
(username,),
)
return cur.fetchone()
def create_session(user_id, token_hash, expires_at):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO sessions (
id, user_id, token_hash, expires_at
)
VALUES (%s, %s, %s, %s)
""",
(uuid4(), user_id, token_hash, expires_at),
)
def revoke_session(token_hash):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE sessions
SET revoked_at = NOW()
WHERE token_hash = %s
AND revoked_at IS NULL
AND expires_at > NOW()
RETURNING id
""",
(token_hash,),
)
return cur.fetchone() is not None
def find_session_user_cached(token_hash):
from datetime import datetime, timezone
from .session_cache import cache_user, get_cached_user
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
s.user_id,
s.expires_at
FROM sessions s
WHERE s.token_hash = %s
AND s.revoked_at IS NULL
AND s.expires_at > NOW()
""",
(token_hash,),
)
session = cur.fetchone()
if session is None:
return None
cached = get_cached_user(token_hash)
if cached is not None and str(cached.get("id")) == str(session["user_id"]):
return cached
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, username, display_name, created_at
FROM users
WHERE id = %s
""",
(session["user_id"],),
)
row = cur.fetchone()
if row is None:
return None
user = serialize_user(row)
ttl = int(
(session["expires_at"] - datetime.now(timezone.utc)).total_seconds()
)
if ttl > 0:
cache_user(token_hash, user, ttl)
return user
+124
View File
@@ -0,0 +1,124 @@
"""One in-flight delivery per worker process. ACK follows the durable decision."""
from datetime import datetime
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import tempfile
from uuid import UUID
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image
from .broker import QUEUE, channel
from .faults import checkpoint
from .jobs import claim, fail, finish
from .process import serve
from .storage import BUCKET, s3
VARIANTS = {'thumbnail': 'image/jpeg', 'crop': 'image/jpeg', 'optimized': 'image/webp'}
class PermanentError(Exception):
pass
def original(attachment):
key = attachment['object_key']
response = s3.get_object(Bucket=BUCKET, Key=key)
body = response['Body']
try:
data = body.read(10 * 1024**2 + 1)
finally:
body.close()
if (len(data) != attachment['size_bytes']
or hashlib.sha256(data).hexdigest() != attachment['sha256']):
raise PermanentError('source_checksum_mismatch')
return data
def put(job, variant, data, mime, dimensions=None):
checksum = hashlib.sha256(data).hexdigest()
key = f'results/{job["attachment_id"]}/v{job["pipeline_version"]}/{variant}/{checksum}'
s3.put_object(Bucket=BUCKET, Key=key, Body=data, ContentType=mime)
entry = dict(key=key, sha256=checksum, size_bytes=len(data), content_type=mime)
if dimensions:
entry.update(width=dimensions[0], height=dimensions[1])
return entry
def process(job, attachment):
data = original(attachment)
with tempfile.TemporaryDirectory(prefix='pipeline-') as target:
directory = Path(target)
(directory / 'original').write_bytes(data)
proc = subprocess.run([sys.executable, '-m', 'src.render_image', target,
str(job['pipeline_version'])], timeout=20, check=False)
if proc.returncode:
raise PermanentError('invalid_image' if proc.returncode == 15 else 'processing_limit')
action = checkpoint(job, 'before_s3')
if action == 'permanent':
raise PermanentError('invalid_image')
if action == 'temporary':
raise OSError('injected transient error')
manifest = {'original': put(job, 'original', data, attachment['content_type'])}
for variant, mime in VARIANTS.items():
path = directory / variant
with Image.open(path) as image:
dimensions = image.size
manifest[variant] = put(job, variant, path.read_bytes(), mime, dimensions)
checkpoint(job, 'after_s3')
return manifest
def handle(ch, method, body):
try:
message = json.loads(body)
if (type(message['schema_version']) is not int or message['schema_version'] != 1
or message['type'] != QUEUE
or any(type(message[field]) is not int or message[field] < 1
for field in ('pipeline_version', 'attempt'))):
raise ValueError('Invalid carrier')
if datetime.fromisoformat(message['created_at']).tzinfo is None:
raise ValueError('Missing UTC offset')
for field in ('job_id', 'attachment_id'):
message[field] = UUID(message[field])
except (ValueError, TypeError, KeyError, AttributeError):
ch.basic_reject(method.delivery_tag, requeue=False)
return
claimed = claim(message)
if claimed:
job, attachment = claimed
try:
manifest = process(job, attachment)
except PermanentError as exc:
fail(job, str(exc), permanent=True)
except subprocess.TimeoutExpired:
fail(job, 'processing_limit', permanent=True)
except ClientError as exc:
code = exc.response['Error']['Code']
fail(job, 'source_missing' if code in ('NoSuchKey', '404') else 'storage_unavailable',
permanent=code in ('NoSuchKey', '404'))
except (BotoCoreError, OSError):
fail(job, 'temporary_failure')
else:
finish(job, manifest)
checkpoint(job, 'after_commit')
ch.basic_ack(method.delivery_tag)
def loop(stopping, progress):
with channel() as (connection, ch):
ch.basic_qos(prefetch_count=1)
while not stopping.is_set():
method, _, body = ch.basic_get(QUEUE, auto_ack=False)
if method:
handle(ch, method, body)
else:
connection.process_data_events(time_limit=0.2)
progress()
if __name__ == '__main__':
serve(loop)
+344
View File
@@ -0,0 +1,344 @@
#!/usr/bin/env python3
"""Destructive fault injection for a disposable, dedicated LAB5 Compose project."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
from image_probe import jpeg_size
COMPOSE = str(ROOT / 'scripts/compose.sh')
def compose(*args):
return subprocess.run([COMPOSE, *args], check=True, text=True, capture_output=True, timeout=120).stdout.strip()
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1',
'-U', 'messenger', '-d', 'messenger', '-c', query)
def job(attachment, version=1):
return json.loads(sql(f"""SELECT row_to_json(j) FROM image_jobs j
WHERE attachment_id='{attachment['id']}' AND pipeline_version={version}"""))
def events(job_id, name):
return int(sql(f"SELECT count(*) FROM job_events WHERE job_id='{job_id}' AND event='{name}'"))
def operator(command, identifier, *args):
return json.loads(compose('exec', '-T', 'recovery', 'python', '-m', 'src.operator', command,
identifier, *args))
def wait(predicate, description, timeout=105):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.2)
raise AssertionError(f'timeout: {description}')
def arm(attachment, point, action, version=1):
sql(f"""INSERT INTO pipeline_faults(attachment_id,pipeline_version,point,action)
VALUES ('{attachment['id']}',{version},'{point}','{action}')""")
def hit(attachment, point, release=False):
scope = f"attachment_id='{attachment['id']}' AND pipeline_version=1 AND point='{point}'"
wait(lambda: sql(f'SELECT hit_at IS NOT NULL FROM pipeline_faults WHERE {scope}') == 't', point)
if release:
sql(f'UPDATE pipeline_faults SET released=true WHERE {scope}')
def objects(attachment, version):
prefix = f"results/{attachment['id']}/v{version}/"
code = ("import json; from src.storage import s3,BUCKET; "
f"print(json.dumps(sorted(x['Key'] for x in s3.list_objects_v2(Bucket=BUCKET,Prefix={prefix!r})"
".get('Contents',[]))))")
return json.loads(compose('exec', '-T', 'recovery', 'python', '-c', code))
def report(name, started, attachment, **metrics):
print(json.dumps(dict(scenario=name, result='PASS', seconds=round(time.monotonic() - started, 3),
attachment_id=attachment['id'], **metrics), ensure_ascii=False), flush=True)
def service_health(service):
return [json.loads(line)['Health'] for line in
compose('ps', '--format', 'json', service).splitlines()]
def experiments(client):
alice, bob, eve, chat = client.bootstrap()
worker_count = compose('ps', '-q', 'worker').splitlines()
check(len(worker_count) >= 2, 'two workers are required')
for service in ('publisher', 'worker'):
enabled = compose('exec', '-T', service, 'printenv', 'PIPELINE_FAULTS')
check(enabled == '1', f'fault injection disabled on {service}')
def upload():
return client.attachment(alice, chat)
def meta(attachment, status=200):
return client.call('GET', '/api/v1/attachments/' + attachment['id'], status,
actor=bob, schema='Attachment')['json']
def verify(attachment, content, version=1):
wait(lambda: job(attachment, version)['state'] == 'ready', f'job v{version} ready')
ready = meta(attachment)
check(ready['status'] == 'ready', 'attachment is not ready')
row = job(attachment, version)
active = int(sql(f"SELECT active_version FROM attachments WHERE id='{attachment['id']}'"))
check(active == version, 'wrong active version')
check(events(row['id'], 'published') == 1, 'expected one published manifest')
check(set(row['manifest']) == {'original', 'thumbnail', 'crop', 'optimized'}, 'manifest incomplete')
check(set(objects(attachment, version)) == {v['key'] for v in row['manifest'].values()},
'unexpected keys or duplicated side effects')
path = '/api/v1/attachments/' + attachment['id'] + '/content?variant='
check({v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'}, 'wrong API variants')
for variant, entry in row['manifest'].items():
downloaded = client.call('GET', path + variant, actor=bob)
data = downloaded['bytes']
check(downloaded['headers'].get_content_type() == entry['content_type'], 'wrong MIME')
check(len(data) == entry['size_bytes'], 'wrong size')
check(hashlib.sha256(data).hexdigest() == entry['sha256'], 'wrong checksum')
if variant == 'original':
check(data == content, 'original changed')
elif variant == 'optimized':
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature')
check((entry['width'], entry['height']) == (320, 200), 'WebP must not be enlarged')
else:
expected = (256, 160) if variant == 'thumbnail' else (128, 128)
check(jpeg_size(data) == expected, 'wrong JPEG dimensions')
return row
started = time.monotonic()
key = 'concurrent-message-' + chat['id']
def post_message(_):
return client.message(alice, chat, 'concurrent', key=key)
with ThreadPoolExecutor(max_workers=6) as executor:
responses = list(executor.map(post_message, range(6)))
check(len({response['json']['id'] for response in responses}) == 1, 'concurrent writes created duplicates')
check(len({response['headers']['X-Instance-Id'] for response in responses}) >= 2,
'concurrent requests did not reach two API instances')
report('concurrent_message', started, {'id': None}, requests=6)
started = time.monotonic()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
client.call('POST', path, 404, actor=eve, raw=fixture, headers={'Content-Type': 'image/png'})
client.call('POST', path, 415, actor=alice, raw=fixture, headers={'Content-Type': 'image/jpeg'})
client.call('POST', path, 415, actor=alice, raw=b'not a png', headers={'Content-Type': 'image/png'})
client.call('POST', path, 413, actor=alice, raw=b'',
headers={'Content-Type': 'image/png', 'Content-Length': '10485761'})
report('upload_validation', started, {'id': None}, statuses=[404, 415, 415, 413])
started = time.monotonic()
compose('stop', '-t', '1', 'broker')
try:
attachment, content = upload()
row = job(attachment)
check(meta(attachment)['status'] == 'queued', 'upload did not remain queued')
check(sql(f"SELECT count(*) FROM job_outbox WHERE job_id='{row['id']}'") == '1',
'lost intent during real broker outage')
wait(lambda: service_health('publisher') == ['unhealthy']
and service_health('worker') == ['unhealthy'] * len(worker_count),
'AMQP outage reflected in publisher and worker health', timeout=45)
finally:
compose('start', 'broker')
verify(attachment, content)
wait(lambda: service_health('publisher') == ['healthy']
and service_health('worker') == ['healthy'] * len(worker_count),
'publisher and all workers became healthy again', timeout=45)
report('broker_unavailable', started, attachment, attempt=1, pipeline_health_recovered=True)
for point in ('before_publish', 'after_confirm', 'before_s3', 'after_s3', 'after_commit'):
started = time.monotonic()
compose('stop', '-t', '1', 'publisher')
if point in ('before_publish', 'after_confirm'):
compose('stop', '-t', '1', 'worker')
attachment, content = upload()
arm(attachment, point, 'crash')
compose('start', 'publisher')
hit(attachment, point)
if point in ('before_publish', 'after_confirm'):
job_id = job(attachment)['id']
wait(lambda: events(job_id, 'broker_confirmed'), 'outbox drained after publisher crash')
compose('start', 'worker')
if point in ('before_s3', 'after_s3'):
keys = objects(attachment, 1)
check(len(keys) == (0 if point == 'before_s3' else 4), 'wrong S3 effects at crash')
check(job(attachment)['state'] == 'processing', 'manifest was already committed at crash')
row = verify(attachment, content)
wanted = {'before_s3': 2, 'after_s3': 2}.get(point, 1)
check(row['attempt'] == wanted, f'wrong attempt after {point}')
check(f'fault {point}: crash {row["id"]}' in compose('logs', '--no-color', 'publisher', 'worker'),
'missing crash log')
if point in ('after_commit', 'after_confirm'):
wait(lambda: events(row['id'], 'delivery_ignored'), 'duplicate delivery ignored')
report(point, started, attachment, attempt=row['attempt'], published=1)
started = time.monotonic()
compose('stop', '-t', '1', 'publisher')
attachment, content = upload()
arm(attachment, 'before_s3', 'pause')
compose('start', 'publisher')
hit(attachment, 'before_s3')
initial = job(attachment)
operator('redeliver', initial['id'])
wait(lambda: events(initial['id'], 'delivery_ignored'), 'parallel duplicate rejected')
check(events(initial['id'], 'claimed') == 1, 'duplicate obtained live lease')
current = verify(attachment, content)
check(current['fence'] > initial['fence'] and current['attempt'] == 2, 'lease was not reclaimed')
hit(attachment, 'before_s3', release=True)
wait(lambda: events(initial['id'], 'fence_rejected'), 'old attempt fenced')
ignored = events(initial['id'], 'delivery_ignored')
operator('redeliver', initial['id'])
wait(lambda: events(initial['id'], 'delivery_ignored') > ignored, 'completed duplicate rejected')
check(verify(attachment, content)['manifest'] == current['manifest'], 'duplicate changed the manifest')
report('duplicate_and_fencing', started, attachment, fence=current['fence'], attempt=current['attempt'])
started = time.monotonic()
compose('stop', '-t', '1', 'publisher')
attachment, content = upload()
arm(attachment, 'after_s3', 'pause')
compose('start', 'publisher')
hit(attachment, 'after_s3')
operator('reprocess', attachment['id'], '2')
second = verify(attachment, content, 2)
hit(attachment, 'after_s3', release=True)
old = job(attachment)
wait(lambda: job(attachment)['state'] == 'superseded' or events(old['id'], 'fence_rejected'),
'v1 completed after v2 without publication')
ignored = events(old['id'], 'delivery_ignored')
operator('redeliver', old['id'])
wait(lambda: events(old['id'], 'delivery_ignored') > ignored, 'reordered v1 redelivery ignored')
check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'v1 changed v2')
check(events(old['id'], 'published') == 0, 'v1 published despite newer version')
report('reorder_in_flight', started, attachment, active_version=2)
started = time.monotonic()
compose('stop', '-t', '1', 'publisher')
attachment, content = upload()
old = job(attachment)
second = operator('reprocess', attachment['id'], '2')
operator('redeliver', second['job_id'])
verify(attachment, content, 2)
check(job(attachment)['state'] == 'queued', 'v1 must still be queued before late delivery')
operator('redeliver', old['id'])
wait(lambda: events(old['id'], 'superseded') == 1, 'late v1 delivery superseded')
check(events(old['id'], 'claimed') == 0, 'late v1 obtained a lease')
compose('start', 'publisher')
check(not objects(attachment, 1), 'late v1 produced S3 side effects')
verify(attachment, content, 2)
report('late_old_delivery', started, attachment, active_version=2)
started = time.monotonic()
attachment, content = upload()
first = verify(attachment, content)
compose('stop', '-t', '1', 'publisher')
try:
operator('reprocess', attachment['id'], '2')
check(meta(attachment)['status'] == 'ready', 'reprocessing hid the published version')
visible = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=thumbnail',
actor=bob)['bytes']
check(hashlib.sha256(visible).hexdigest() == first['manifest']['thumbnail']['sha256'],
'published version changed before v2 commit')
finally:
compose('start', 'publisher')
second = verify(attachment, content, 2)
for name in ('thumbnail', 'crop', 'optimized'):
check(first['manifest'][name]['sha256'] != second['manifest'][name]['sha256'], 'v2 did not change bytes')
operator('redeliver', first['id'])
wait(lambda: events(first['id'], 'delivery_ignored'), 'duplicate of published v1')
check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'old duplicate changed v2')
report('atomic_version_switch', started, attachment, active_version=2)
for action in ('temporary', 'permanent'):
started = time.monotonic()
compose('stop', '-t', '1', 'publisher')
attachment, content = upload()
arm(attachment, 'before_s3', action)
compose('start', 'publisher')
initial = job(attachment)
if action == 'permanent':
wait(lambda: job(attachment)['state'] == 'failed', 'permanent failure')
check(meta(attachment)['error_code'] == 'invalid_image', 'unsafe or wrong error code')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
client.call('GET', path, 409, actor=bob)
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': '', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': attachment['id']})
check(sql(f"SELECT error_code FROM job_dead_letters WHERE job_id='{initial['id']}'") == 'invalid_image',
'missing DLQ entry')
operator('replay', initial['id'])
check(sql(f"SELECT replayed_at IS NOT NULL FROM job_dead_letters WHERE job_id='{initial['id']}'") == 't',
'missing replay audit')
completed = verify(attachment, content)
check(completed['attempt'] == 2, 'expected two attempts')
if action == 'temporary':
check(events(initial['id'], 'retry_scheduled') == 1, 'missing transient retry')
report(action + '_error', started, attachment, attempt=2)
stable_attachment, stable_content = attachment, content
stable_manifest = completed['manifest']
started = time.monotonic()
compose('stop', '-t', '1', 'worker')
attachment, content = upload()
row = job(attachment)
wait(lambda: events(row['id'], 'broker_confirmed'), 'durable queue entry')
check(meta(attachment)['status'] == 'queued', 'worker processed despite stop')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob)
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
compose('restart', 'postgres', 'redis', 's3', 'broker', 'api', 'lb', 'publisher', 'recovery')
wait(lambda: health(client), 'HAProxy readiness')
check(not compose('ps', '-q', 'worker'), 'restart brought workers back too early')
check(meta(attachment)['status'] == 'queued', 'confirmed job changed before worker restart')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob)
compose('start', 'worker')
verify(attachment, content)
page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=bob,
schema='MessagePage')['json']
check(any(item['id'] == message['id'] and attachment['id'] in item['attachment_ids']
for item in page['items']), 'attachment link did not survive restart')
wait(lambda: sql(f"SELECT staging_key IS NULL FROM attachments WHERE id='{attachment['id']}'") == 't',
'recovery cleaned staging after publication')
wait(lambda: service_health('recovery') == ['healthy'], 'recovery health after restart')
check(verify(stable_attachment, stable_content)['manifest'] == stable_manifest,
'published attachment changed on restart')
check(job(attachment)['attempt'] == 1, 'broker restart lost the original delivery')
report('restart_with_queued_attachment', started, attachment, attempt=1,
message_link_preserved=True, staging_cleaned=True)
def health(client):
try:
client.call('GET', '/health/ready')
except (OSError, AssertionError):
return False
return True
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--disposable', action='store_true', required=True,
help='confirm this is an isolated test project')
args = parser.parse_args()
try:
experiments(Client(args.base_url))
except (AssertionError, OSError, ValueError, subprocess.SubprocessError) as exc:
# Subprocess output can include internal details. Keep the public summary short.
print('FAIL:', str(exc), file=sys.stderr)
raise SystemExit(1)
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Check image encoders, geometry, EXIF orientation and alpha on local fixtures."""
from pathlib import Path
import struct
import subprocess
import sys
import tempfile
import zlib
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / 'tests/fixtures/sample.png'
def render(directory, version):
subprocess.run([sys.executable, '-m', 'src.render_image', str(directory), str(version)],
cwd=ROOT, check=True)
outputs = {}
for variant, format_name in [('thumbnail', 'JPEG'), ('crop', 'JPEG'), ('optimized', 'WEBP')]:
with Image.open(directory / variant) as image:
image.load()
assert image.format == format_name
assert not image.getexif()
outputs[variant] = (image.size, (directory / variant).read_bytes())
return outputs
def validate(path, mime):
return subprocess.run([sys.executable, '-m', 'src.image_check', str(path), mime],
cwd=ROOT, timeout=10, check=False).returncode
def main():
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
original = directory / 'original'
data = FIXTURE.read_bytes()
original.write_bytes(data)
assert validate(original, 'image/png') == 0
assert validate(original, 'image/jpeg') == 15
original.write_bytes(b'corrupt')
assert validate(original, 'image/png') == 15
# Rewrite only IHDR, with a valid CRC, to exercise the pre-decode limit.
header = data[12:16] + struct.pack('>II', 10000, 10000) + data[24:29]
original.write_bytes(data[:12] + header + struct.pack('>I', zlib.crc32(header)) + data[33:])
assert validate(original, 'image/png') == 13
original.write_bytes(data)
print('PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding')
first = render(directory, 1)
assert first == render(directory, 1)
second = render(directory, 2)
assert first.keys() == second.keys()
for variant, shape in [('thumbnail', (256, 160)), ('crop', (128, 128)), ('optimized', (320, 200))]:
assert first[variant][0] == second[variant][0] == shape
assert first[variant][1] != second[variant][1]
assert original.read_bytes() == data
print('PASS: fixture geometry, original preserved, deterministic v1, different v2')
transparent = Image.new('RGBA', (7, 3), (255, 0, 0, 0))
transparent.save(original, format='PNG')
alpha = render(directory, 1)
assert alpha['thumbnail'][0] == (7, 3)
with Image.open(directory / 'thumbnail') as thumbnail:
assert thumbnail.getpixel((0, 0)) == (255, 255, 255)
rotated = Image.new('RGB', (17, 31), 'red')
exif = rotated.getexif()
exif[274] = 6
rotated.save(original, format='JPEG', exif=exif)
assert render(directory, 1)['thumbnail'][0] == (31, 17)
print('PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed')
if __name__ == '__main__':
main()