ЛР 5 - БПИ2301 - Дугина - на 5 (трек 2) #103
@@ -18,3 +18,5 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
DEFENSE.md
|
||||
|
||||
+6
-20
@@ -1,25 +1,11 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Generate the local .env and isolated secret files with scripts/init-local.py.
|
||||
COMPOSE_PROJECT_NAME=mtusi2026lab5
|
||||
PUBLIC_PORT=8080
|
||||
BROKER_CONSOLE_PORT=15685
|
||||
S3_CONSOLE_PORT=9005
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
PIPELINE_FAULTS=0
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
PIPELINE_FAULTS: '1'
|
||||
JOB_TIMEOUT_SECONDS: '20'
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab5-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Build and wait for readiness
|
||||
run: |
|
||||
python scripts/init-local.py
|
||||
./scripts/compose.sh up -d --build --wait --wait-timeout 150
|
||||
curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready
|
||||
- name: Mandatory smoke and recovery
|
||||
run: |
|
||||
set -o pipefail
|
||||
make test 2>&1 | tee evidence/ci-smoke.txt
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh 2>&1 | tee evidence/ci-persistence.txt
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh 2>&1 | tee evidence/ci-failover.txt
|
||||
./scripts/compose.sh start api
|
||||
curl -fsS --retry 30 --retry-delay 1 --retry-all-errors http://localhost:8080/health/ready
|
||||
- name: Track 2 experiments
|
||||
run: |
|
||||
set -o pipefail
|
||||
python student-tests/pipeline.py --disposable 2>&1 | tee evidence/ci-pipeline.jsonl
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: lab5-evidence
|
||||
path: |
|
||||
evidence/ci-*
|
||||
evidence/restart-*.txt
|
||||
- name: Stop test project
|
||||
if: always()
|
||||
run: ./scripts/compose.sh down
|
||||
@@ -30,3 +30,5 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
/DEFENSE.md
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
FROM python:3.11.16-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
|
||||
COPY --chown=appuser:appuser src/ ./src/
|
||||
|
||||
COPY --chown=appuser:appuser db/ ./db/
|
||||
|
||||
COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
COPY infra/app-start.sh /usr/local/bin/app-start
|
||||
ENTRYPOINT ["/bin/sh", "/usr/local/bin/app-start"]
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
|
||||
|
||||
CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2", "--limit-concurrency", "64"]
|
||||
@@ -1,37 +1,210 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа — Дугина Александра, БПИ2301.
|
||||
- Целевая оценка — 5, трек 2 «Доказанная идемпотентность pipeline». Проверки выполнены на реальных контейнерах Docker Desktop.
|
||||
- Использованные источники и помощь — я использовала контракты и тесты курса, собственную реализацию мессенджера из ЛР №4, FastAPI, psycopg, Pillow, boto3 и Pika.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Мне понадобятся Docker Engine или Docker Desktop с доступным daemon, Compose v2,
|
||||
Python 3.11+ на хосте для проверок и локальные свободные порты 8080, 15685, 9005.
|
||||
|
||||
```sh
|
||||
python3 scripts/init-local.py
|
||||
./scripts/compose.sh config --quiet
|
||||
./scripts/compose.sh up -d --build --wait --wait-timeout 150
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
./scripts/compose.sh start api
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 150
|
||||
```
|
||||
|
||||
`init-local.py` генерирует случайные пароли в игнорируемом каталоге `secrets/`
|
||||
и создаёт локальный `.env`. При повторном запуске я сохраняю существующие значения.
|
||||
Порты можно заменить в `.env`. При изменении `PUBLIC_PORT` я передаю
|
||||
`BASE_URL=http://localhost:НОВЫЙ_ПОРТ` в Make и `--base-url` в тестовые скрипты.
|
||||
|
||||
Swagger открывается на <http://localhost:8080/docs>. Веб-консоли RabbitMQ и MinIO
|
||||
доступны на <http://localhost:15685> и <http://localhost:9005>. Сетевые порты
|
||||
привязаны к loopback. Остановить стенд с сохранением данных можно через
|
||||
`./scripts/compose.sh down`. Для намеренного удаления **своих** учебных данных
|
||||
существует `./scripts/compose.sh down --volumes`. Перед этим я проверяю имя
|
||||
Compose-проекта и убеждаюсь, что его volumes больше не нужны.
|
||||
|
||||
| Роль | Compose и версия | Готовность и данные | Доступ |
|
||||
| --- | --- | --- | --- |
|
||||
| Балансировщик | `lb`, HAProxy 3.0.9-alpine | HTTP-проверки двух реплик, локальных данных нет | 8080 через loopback |
|
||||
| API и миграции | `api` ×2, `migrate`, Python 3.11.16-slim-bookworm | `/health/ready` проверяет БД и доступ к bucket, миграции одноразовые | Внутренняя сеть, secrets с DSN и доступом к S3 |
|
||||
| База | `postgres`, PostgreSQL 16.8-bookworm | TCP `pg_isready`, `postgres_data` | Внутренняя сеть, файл пароля |
|
||||
| Кэш профилей | `redis`, Redis 7.4.2-alpine | `redis-cli ping`, AOF в `redis_data` | Внутренняя сеть |
|
||||
| Объекты | `s3`, MinIO 2025.7.23 с digest из Compose | MinIO live, `s3_data` | Приватный bucket, ограниченная policy приложения, локальная консоль 9005 |
|
||||
| Брокер | `broker`, RabbitMQ 4.1.4-management | `rabbitmq-diagnostics check_running`, `broker_data` | Внутренний AMQP, консоль 15685, секрет аутентификации |
|
||||
| Фоновые процессы | `publisher`, `worker` ×2, `recovery`, образ приложения | Проверка продвижения рабочего цикла, reconnect, restart policy | Нет публичных портов |
|
||||
|
||||
Compose дожидается PostgreSQL и применяет SQL-миграции под advisory lock. Список
|
||||
применённых файлов хранится в `schema_migrations`. MinIO bootstrap создаёт
|
||||
приватный bucket, сервисную учётную запись и policy. Publisher и worker объявляют
|
||||
одну и ту же durable quorum queue при подключении. `.env.example` показывает
|
||||
переменные портов, TTL и pipeline, фактически используемые в Compose. Секреты
|
||||
подключаются файлами, а entrypoint приложения копирует нужные файлы перед
|
||||
запуском процессов от непривилегированного пользователя.
|
||||
|
||||
Все три фоновые роли обновляют метку прогресса в собственном tmpfs.
|
||||
Worker обновляет её после обработки доставки или успешного опроса очереди,
|
||||
publisher — после итерации outbox через канал с publisher confirms, recovery —
|
||||
после итерации проверки lease и плановой очистки. Метка появляется после
|
||||
первой успешной итерации, сбрасывается, если ошибка зависимости прервала цикл, и при завершении
|
||||
процесса. Она истекает через 60 секунд без прогресса. Compose опрашивает
|
||||
проверку раз в 3 секунды и выставляет unhealthy после трёх неуспехов.
|
||||
Миграции и S3 bootstrap остаются одноразовыми: Compose ждёт их успешного
|
||||
завершения.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Моя цель — отвечать на upload после надёжного приёма файла и обрабатывать его
|
||||
в отдельных процессах. PostgreSQL проверяет сессию при каждом
|
||||
запросе. Redis ускоряет получение профиля. При конкурентных вставках keyset
|
||||
пагинация видит записи после курсора с ключом `(created_at, id)`. Между
|
||||
страницами нет общей snapshot-транзакции.
|
||||
|
||||
1. API принимает PNG/JPEG потоком с лимитом 10 MiB и проверяет настоящий формат,
|
||||
декодируемость и лимит 40 Мп в отдельном процессе. Временный файл нужен для
|
||||
проверки и записи. Под advisory lock API записывает исходник в приватный S3
|
||||
prefix `staging/`. В одной PostgreSQL-транзакции сохраняются attachment,
|
||||
job и outbox. После COMMIT API возвращает снимок `202 queued`. В состоянии
|
||||
queued файл уже можно привязать к сообщению. Download вернёт 409.
|
||||
2. Publisher читает только зафиксированные outbox-строки через
|
||||
`FOR UPDATE SKIP LOCKED`, отправляет carrier по учебной JSON-схеме и удаляет
|
||||
строку после confirm. RabbitMQ сохраняет сообщения в durable quorum queue.
|
||||
При потере связи или аварии после confirm до удаления отправка может
|
||||
повториться. В AMQP передаются идентификаторы и номера, байты остаются в S3.
|
||||
3. Worker держит одну неподтверждённую доставку, перед работой атомарно
|
||||
захватывает job, увеличивает fencing token и выставляет lease. По умолчанию
|
||||
два worker-процесса, lease 60 секунд. Worker берёт серверный ключ из БД,
|
||||
сверяет байты и SHA-256, применяет EXIF orientation, композитит альфа-канал
|
||||
на белый. JPEG thumbnail вписывается в 256×256 без увеличения и с
|
||||
округлением вниз. JPEG crop центрирован и имеет размер 128×128.
|
||||
WebP вписывается в 1280×1280 без увеличения, method 4. Качество JPEG/WebP
|
||||
для v1 равно 85/82, для v2 — 90/88. Производные сохраняются без исходных
|
||||
EXIF/ICC. Исходник копируется побайтно. Linux-процесс декодирования worker
|
||||
ограничен 512 MiB виртуальной памяти, 15 секундами CPU и 20 секундами по
|
||||
часам родителя. Лимиты контейнера — 1 CPU и 768 MiB на worker.
|
||||
4. Ключ результата включает attachment, pipeline version, вариант и SHA-256.
|
||||
Повторная запись того же ключа содержит те же байты. В одной транзакции
|
||||
worker публикует manifest и переключает активную версию, если lease ещё
|
||||
действует, fencing token совпадает и версия остаётся желаемой. Затем
|
||||
отправляет ack. Чтение через API использует только активный manifest.
|
||||
При переходе с готовой v1 на v2 прежняя версия доступна до атомарной
|
||||
публикации v2.
|
||||
5. Временная ошибка или истёкший lease увеличивает номер попытки и возвращает
|
||||
job в outbox с exponential backoff, jitter 0,75–1,25 от базовой задержки и
|
||||
пределом базовой задержки 30 секунд. После пяти попыток по умолчанию либо
|
||||
постоянной ошибки worker/recovery сохраняет failed и запись в PostgreSQL DLQ.
|
||||
Я использую этот реестр как долговечную очередь ручного replay. Replay
|
||||
разрешён только для failed job текущей версии и сохраняет UUID и историю
|
||||
предыдущих попыток.
|
||||
6. Recovery удаляет staging готовых вложений после commit. Непривязанные и
|
||||
failed-исходники остаются по умолчанию сутки. Cleanup проверяет метаданные
|
||||
и берёт тот же lock, что uploader. Во время reprocessing источником
|
||||
служит опубликованный оригинал. Внутренний ключ бакета в API отсутствует.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Клиент] --> H[HAProxy]
|
||||
H --> A[Две API-реплики]
|
||||
A --> S[(MinIO)]
|
||||
A --> P[(PostgreSQL)]
|
||||
P --> O[Publisher]
|
||||
O --> B[(RabbitMQ)]
|
||||
B --> W[Worker ×2]
|
||||
W --> S
|
||||
W --> P
|
||||
R[Recovery] --> P
|
||||
R --> S
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Docker VM использовала Linux aarch64, 8 CPU и 7,65 GiB RAM.
|
||||
Хост Darwin arm64, 8 CPU, 16 GiB RAM, тестовый клиент Python 3.13.3.
|
||||
Строки CPU и памяти в `container-resources.txt` — один снимок после запуска.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Проверка и наблюдение | Доказательство |
|
||||
| --- | --- | --- |
|
||||
| Контракты и примеры | `make check` и полная OpenAPI/JSON Schema валидация прошли | [offline](evidence/local-checks.txt) |
|
||||
| Воспроизводимый запуск | `up -d --build --wait --wait-timeout 150` завершился с кодом 0. Две API-реплики, балансировщик, хранилища, брокер и все фоновые процессы healthy | [запуск](evidence/compose-up.txt), [после перезапуска](evidence/post-restart-health.txt) |
|
||||
| Уровень 3, публичный API | `make test` прошёл, включая права, SHA-256 оригинала и размер thumbnail | [smoke](evidence/smoke.txt) |
|
||||
| Уровень 3, сохранность | `make test-persistence` подтвердил сохранность данных и прежней сессии. Изменились timestamps запуска API, HAProxy, хранилищ, RabbitMQ и фоновых процессов | [тест](evidence/persistence.txt), [до](evidence/restart-before.txt), [после](evidence/restart-after.txt) |
|
||||
| Уровень 3, failover | `make test-failover` прошёл. Поочерёдно остановлены `api-2` и `api-1`. Первая успешная запись через 2,280 и 1,925 с соответственно. В каждом прогоне подтверждено 120 записей | [тест](evidence/failover.txt), [состояние контейнеров](evidence/failover-containers.txt), [повтор](evidence/failover-repeat.txt) |
|
||||
| Уровни 3–4, обработка изображений | Проверки с Pillow 11.2.1 внутри образа прошли, включая геометрию, детерминированность, EXIF, прозрачность и лимит пикселей | [render](evidence/container-render.txt) |
|
||||
| Уровни 3–5, отказоустойчивость, retry, DLQ и идемпотентность | Сценарии и результаты приведены ниже | [JSONL](evidence/track2.jsonl) |
|
||||
|
||||
Отдельный `make load` выполнил 7583 успешных POST сообщений за 30.037 с, concurrency 4. Ошибок 0, 252.458 успешных запросов/с. Задержки p50/p95/p99 успешных запросов составили 12.714/37.804/73.943 мс. Это один closed-loop замер HTTP-записей сообщений. [Метрики нагрузки](evidence/load.txt) · [снимок CPU и памяти](evidence/container-resources.txt)
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотеза.** В заданной модели отказов принятый upload дойдёт до готового
|
||||
результата после восстановления компонентов. Повторные доставки не создадут
|
||||
вторую публикацию того же manifest. Новая версия сохранится, если позднее
|
||||
возобновить старый worker или доставить старую job.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Эксперимент использует отдельный Compose-проект, две реплики worker,
|
||||
lease 20 секунд, PNG 320×200 и обычный HTTP API. Для каждого сценария
|
||||
создаётся новый объект. Время в JSONL измеряется монотонными часами от начала
|
||||
сценария до последнего assertion и включает команды Compose, рестарты и
|
||||
ожидание lease. Отдельный сценарий отправляет шесть одновременных POST
|
||||
сообщений с общим ключом через HAProxy.
|
||||
|
||||
```sh
|
||||
export COMPOSE_PROJECT_NAME=mtusi2026lab5-track2
|
||||
export PUBLIC_PORT=8085 BROKER_CONSOLE_PORT=15686 S3_CONSOLE_PORT=9006
|
||||
export PIPELINE_FAULTS=1 WORKER_CONCURRENCY=2 JOB_TIMEOUT_SECONDS=20
|
||||
./scripts/compose.sh up -d --build --wait --wait-timeout 150
|
||||
export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1
|
||||
set -o pipefail
|
||||
python3 student-tests/pipeline.py --disposable --base-url http://localhost:8085 | tee evidence/track2.jsonl
|
||||
```
|
||||
|
||||
Исполнитель читает PostgreSQL и MinIO и проверяет скачанные байты по HTTP.
|
||||
Для аварий он ждёт записи о достижении конкретной точки, сверяет журнал
|
||||
контейнеров и проверяет число попыток, публикаций, ключей S3, состояние и хеши.
|
||||
Сценарий брокера останавливает реальный контейнер. `crash` завершает процесс
|
||||
через `os._exit(86)`. `pause` удерживает старого исполнителя до явного
|
||||
освобождения в эксперименте.
|
||||
|
||||
| Воздействие | Наблюдение | Время сценария |
|
||||
| --- | --- | --- |
|
||||
| Конкурентные записи и проверка входа | Шесть запросов с одним ключом через две API-реплики дали один ID сообщения. Отклонены чужая загрузка, неверный MIME, повреждённое и слишком большое тело | 0,041 / 0,379 с |
|
||||
| Остановка RabbitMQ | Принятый upload остался в outbox. Publisher и оба worker стали unhealthy. После старта RabbitMQ они вернулись в healthy, а вложение стало ready с первой попытки | 19,129 с |
|
||||
| Авария издателя до publish и после confirm | Задание дошло до ready без пересоздания. Повтор после confirm проигнорирован. В каждом сценарии зафиксирована одна публикация manifest | 12,328 / 12,365 с |
|
||||
| Авария worker до записи S3 | В точке сбоя итоговых объектов не было. После истечения lease вторая попытка опубликовала manifest один раз | 28,879 с |
|
||||
| Авария после S3 до PostgreSQL | В S3 уже лежали четыре объекта, а job была processing. После второй попытки сохранился один набор из четырёх ключей и одна публикация manifest | 28,390 с |
|
||||
| Авария после COMMIT результата до ack | Дубль доставки проигнорирован, номер попытки остался 1, manifest опубликован один раз | 7,570 с |
|
||||
| Дубль при активном lease и возобновление старого worker | При активном lease зафиксирован один захват. После истечения lease вторая попытка получила fencing token 2. Поздняя фиксация старой попытки отклонена | 33,826 с |
|
||||
| v2 готова раньше приостановленной или запоздалой v1 | API продолжил выдавать v2. Устаревшая v1 не опубликовала manifest. Запоздалая v1 не получила lease и не создала объектов S3 | 14,563 / 16,117 с |
|
||||
| Повторная обработка готовой v1 | Пока v2 строилась, скачивалась v1. После COMMIT все скачанные варианты соответствовали v2. Дубль v1 не изменил опубликованный manifest | 16,302 с |
|
||||
| Временная и постоянная ошибки | Временная ошибка отложила повтор. Постоянная дала failed, безопасный error_code и запись DLQ. Явный replay завершил то же задание со второй попытки | 8,366 / 9,060 с |
|
||||
| Рестарт API, брокера и хранилищ при остановленных worker | Подтверждённое задание осталось queued, скачивание вернуло 409. После запуска worker вложение стало ready с первой попытки, ссылка в сообщении сохранилась, staging очищен | 23,390 с |
|
||||
|
||||
Все 15 сценариев завершились с PASS. После эксперимента Compose повторно подтвердил готовность всех долгоживущих сервисов. [Результаты JSONL](evidence/track2.jsonl) · [код завершения](evidence/track2-errors.txt) · [готовность после эксперимента](evidence/track2-final-health.txt)
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Я опираюсь на сохранность volumes PostgreSQL, RabbitMQ и MinIO на одном хосте.
|
||||
Временная потеря связи поддерживается протоколом, а потеря диска или всего
|
||||
хоста выходит за пределы эксперимента. Outbox и lease закрывают разные окна
|
||||
отказа. Publisher confirm подтверждает приём брокером, а consumer ack снимает
|
||||
ответственность за конкретную доставку после долговечного решения. Между ними
|
||||
процесс может остановиться, и RabbitMQ может повторить доставку. Операция S3
|
||||
может выполниться несколько раз. Идемпотентность проявляется в единственной
|
||||
публикации активного manifest и в стабильном содержимом адресуемых объектов.
|
||||
Гарантии exactly-once доставки здесь нет.
|
||||
|
||||
Старые версии результата и объекты незавершённой устаревшей версии могут
|
||||
оставаться в закрытом bucket. Для длительной эксплуатации потребуется отдельная
|
||||
политика их удаления и аудит потребления места. Lease фиксированный, без
|
||||
продления, с запасом относительно бюджета обработки. RabbitMQ в стенде
|
||||
одноузловой. Эксперимент измеряет восстановление одного стенда и использует
|
||||
один профиль входного изображения.
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
name: mtusi2026lab5
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -h 127.0.0.1 -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always]
|
||||
volumes: [redis_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi2026lab5-api:local
|
||||
command: [python, -m, src.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
restart: 'no'
|
||||
api:
|
||||
build: .
|
||||
image: mtusi2026lab5-api:local
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
replicas: 2
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()']
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
tmpfs:
|
||||
- /tmp:size=16m,mode=1777
|
||||
stop_grace_period: 6s
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
restart: unless-stopped
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-management
|
||||
entrypoint: [/bin/sh, /broker-start.sh]
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
secrets: [broker_password]
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
- ./infra/broker-start.sh:/broker-start.sh:ro
|
||||
hostname: broker
|
||||
ports: ['127.0.0.1:${BROKER_CONSOLE_PORT:-15685}:15672']
|
||||
healthcheck:
|
||||
test: [CMD, rabbitmq-diagnostics, -q, check_running]
|
||||
interval: 4s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
mem_limit: 768m
|
||||
cpus: 1.0
|
||||
restart: unless-stopped
|
||||
publisher: &pipeline
|
||||
image: mtusi2026lab5-api:local
|
||||
command: [python, -m, src.publisher]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
PIPELINE_FAULTS: ${PIPELINE_FAULTS:-0}
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: [CMD, python, -m, src.pipeline_health]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
tmpfs:
|
||||
- /tmp:size=16m,mode=1777
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
restart: unless-stopped
|
||||
worker:
|
||||
<<: *pipeline
|
||||
command: [python, -m, src.worker]
|
||||
deploy:
|
||||
replicas: ${WORKER_CONCURRENCY:-2}
|
||||
init: true
|
||||
tmpfs:
|
||||
- /tmp:size=64m,mode=1777
|
||||
cpus: 1.0
|
||||
mem_limit: 768m
|
||||
stop_grace_period: 30s
|
||||
recovery:
|
||||
<<: *pipeline
|
||||
command: [python, -m, src.recovery]
|
||||
lb:
|
||||
image: haproxy:3.0.9-alpine
|
||||
ports: ['127.0.0.1:${PUBLIC_PORT:-8080}:8080']
|
||||
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
s3:
|
||||
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
entrypoint: [/opt/bitnami/minio/bin/minio]
|
||||
user: '0:0'
|
||||
command: [server, /data, --console-address, ':9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
ports: ['127.0.0.1:${S3_CONSOLE_PORT:-9005}:9001']
|
||||
healthcheck:
|
||||
test: [CMD, curl, -f, http://localhost:9000/minio/health/live]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
mem_limit: 768m
|
||||
cpus: 1.0
|
||||
s3-bootstrap:
|
||||
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
entrypoint: [/bin/sh, /bootstrap.sh]
|
||||
volumes:
|
||||
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||
- ./infra/s3-policy.json:/policy.json:ro
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: 'no'
|
||||
volumes:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
broker_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
@@ -0,0 +1,28 @@
|
||||
# Операторские операции pipeline
|
||||
|
||||
Эти команды выполняются внутри доверенного Compose-проекта. HTTP-контракт
|
||||
мессенджера остаётся исходным. Идентификаторы берутся из PostgreSQL.
|
||||
|
||||
```sh
|
||||
./scripts/compose.sh exec -T recovery python -m src.operator reprocess ATTACHMENT_UUID 2
|
||||
./scripts/compose.sh exec -T recovery python -m src.operator replay JOB_UUID
|
||||
./scripts/compose.sh exec -T recovery python -m src.operator redeliver JOB_UUID
|
||||
```
|
||||
|
||||
- `reprocess` принимает версию выше желаемой. Реализованы 1 и 2. Старая готовая
|
||||
версия доступна, пока новая строится. Для вложения без готовой версии
|
||||
используются обычные состояния LAB5.
|
||||
- `replay` принимает только failed job текущей желаемой версии с сохранённым
|
||||
исходником. UUID job сохраняется, attempt растёт, бюджет попыток обновляется.
|
||||
История отказа остаётся в `job_dead_letters`.
|
||||
- `redeliver` служит для диагностики. Команда посылает текущий carrier через
|
||||
RabbitMQ и ждёт publisher confirm. Она сама не выдаёт lease и не меняет статус.
|
||||
|
||||
В `image_jobs` хранится состояние каждой версии. Журнал `job_events` сохраняет
|
||||
захват, номер fencing token, подтверждение, retry, DLQ, публикацию результата и
|
||||
отклонение устаревшего исполнителя. Выход CLI содержит поля carrier.
|
||||
|
||||
Внедрение сбоев разрешает только переменная `PIPELINE_FAULTS=1`. Управление
|
||||
производится непосредственно через PostgreSQL на выделенном тестовом стенде.
|
||||
Разовые команды в `pipeline_faults` привязаны к UUID вложения и версии. Выключайте
|
||||
внедрение сбоев в обычном запуске.
|
||||
@@ -0,0 +1,51 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name VARCHAR(100) NOT NULL
|
||||
CHECK (length(btrim(display_name)) > 0),
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL
|
||||
CHECK (length(btrim(title)) > 0),
|
||||
created_by UUID NOT NULL REFERENCES users(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
text TEXT NOT NULL
|
||||
CHECK (length(text) <= 4000 AND length(btrim(text)) > 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_members_user_id
|
||||
ON chat_members (user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
|
||||
ON sessions (expires_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE message_requests (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
key VARCHAR(128) NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
message_id UUID REFERENCES messages(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (chat_id, sender_id, key)
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
filename TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages ADD CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0));
|
||||
@@ -0,0 +1,77 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
ALTER TABLE attachments ADD CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN desired_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN active_version INTEGER,
|
||||
ADD COLUMN staging_key TEXT UNIQUE,
|
||||
ADD COLUMN error_code TEXT;
|
||||
|
||||
CREATE TABLE image_jobs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL CHECK (pipeline_version > 0),
|
||||
state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt > 0),
|
||||
failures INTEGER NOT NULL DEFAULT 0,
|
||||
fence BIGINT NOT NULL DEFAULT 0,
|
||||
lease_until TIMESTAMPTZ,
|
||||
error_code TEXT,
|
||||
manifest JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CHECK ((state = 'processing') = (lease_until IS NOT NULL)),
|
||||
CHECK ((state = 'ready') = (manifest IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX image_jobs_expired ON image_jobs(lease_until) WHERE state = 'processing';
|
||||
|
||||
CREATE TABLE job_outbox (
|
||||
job_id UUID PRIMARY KEY REFERENCES image_jobs(id),
|
||||
publish_after TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_outbox_due ON job_outbox(publish_after);
|
||||
|
||||
CREATE TABLE job_events (
|
||||
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES image_jobs(id),
|
||||
attempt INTEGER NOT NULL,
|
||||
event TEXT NOT NULL,
|
||||
detail TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_lookup ON job_events(job_id, id);
|
||||
|
||||
-- Durable operator-visible DLQ. Replays retain the historical failure rows.
|
||||
CREATE TABLE job_dead_letters (
|
||||
job_id UUID NOT NULL REFERENCES image_jobs(id),
|
||||
attempt INTEGER NOT NULL,
|
||||
error_code TEXT NOT NULL,
|
||||
failed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
replayed_at TIMESTAMPTZ,
|
||||
PRIMARY KEY (job_id, attempt)
|
||||
);
|
||||
|
||||
-- Used only by an explicitly enabled synthetic fault-injection stand.
|
||||
CREATE TABLE pipeline_faults (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
point TEXT NOT NULL,
|
||||
action TEXT NOT NULL CHECK (action IN ('crash','pause','temporary','permanent')),
|
||||
hit_at TIMESTAMPTZ,
|
||||
released BOOLEAN NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (attachment_id,pipeline_version,point)
|
||||
);
|
||||
|
||||
-- Upgrade persisted lab4 originals, which remain the source until v1 commits.
|
||||
UPDATE attachments SET status='queued', updated_at=now();
|
||||
WITH inserted AS (
|
||||
INSERT INTO image_jobs(attachment_id,pipeline_version)
|
||||
SELECT id,1 FROM attachments RETURNING id
|
||||
)
|
||||
INSERT INTO job_outbox(job_id) SELECT id FROM inserted;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CHECK (desired_version > 0 AND (active_version IS NULL OR active_version <= desired_version)),
|
||||
ADD CHECK ((status = 'ready') = (active_version IS NOT NULL)),
|
||||
ADD CHECK ((status = 'failed') = (error_code IS NOT NULL)),
|
||||
ADD FOREIGN KEY (id, active_version) REFERENCES image_jobs(attachment_id, pipeline_version);
|
||||
@@ -0,0 +1,231 @@
|
||||
Compose now can delegate build to bake for better performances
|
||||
Just set COMPOSE_BAKE=true
|
||||
#0 building with "desktop-linux" instance using docker driver
|
||||
|
||||
#1 [migrate internal] load build definition from Dockerfile
|
||||
#1 transferring dockerfile: 1.04kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [migrate internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [migrate internal] load .dockerignore
|
||||
#3 transferring context: 208B done
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [migrate internal] load build context
|
||||
#4 transferring context: 3.49kB done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [migrate 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
|
||||
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [migrate 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
#6 CACHED
|
||||
|
||||
#7 [migrate 3/9] COPY requirements.txt .
|
||||
#7 CACHED
|
||||
|
||||
#8 [migrate 2/9] WORKDIR /app
|
||||
#8 CACHED
|
||||
|
||||
#9 [migrate 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
#9 CACHED
|
||||
|
||||
#10 [migrate 6/9] COPY --chown=appuser:appuser src/ ./src/
|
||||
#10 DONE 0.0s
|
||||
|
||||
#11 [migrate 7/9] COPY --chown=appuser:appuser db/ ./db/
|
||||
#11 DONE 0.0s
|
||||
|
||||
#12 [migrate 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
#12 DONE 0.0s
|
||||
|
||||
#13 [migrate 9/9] COPY infra/app-start.sh /usr/local/bin/app-start
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 [migrate] exporting to image
|
||||
#14 exporting layers 0.1s done
|
||||
#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44
|
||||
#14 exporting manifest sha256:9a13f24443edccaf781d342712646f49ac012621efd9e13b992d8868171e7e44 done
|
||||
#14 exporting config sha256:6e878a3e1844f17aa93a4277e04be0bd98ca42f3ba53997976cb1499c4ab5a22 done
|
||||
#14 exporting attestation manifest sha256:8998f445e472a60a93fd3480f4000dede2d0fa76d5fcb690911d90e747483592 0.0s done
|
||||
#14 exporting manifest list sha256:404b7aaa2cb3fda8ba8e0ea873f7171be30797c57a39261d9b78ec48f3e477c5 done
|
||||
#14 naming to docker.io/library/mtusi2026lab5-api:local done
|
||||
#14 unpacking to docker.io/library/mtusi2026lab5-api:local 0.0s done
|
||||
#14 DONE 0.2s
|
||||
|
||||
#15 [migrate] resolving provenance for metadata file
|
||||
#15 DONE 0.0s
|
||||
|
||||
#16 [api internal] load build definition from Dockerfile
|
||||
#16 transferring dockerfile: 1.04kB done
|
||||
#16 DONE 0.0s
|
||||
|
||||
#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
|
||||
#2 DONE 0.0s
|
||||
|
||||
#17 [api internal] load .dockerignore
|
||||
#17 transferring context: 208B done
|
||||
#17 DONE 0.0s
|
||||
|
||||
#18 [api internal] load build context
|
||||
#18 transferring context: 2.45kB done
|
||||
#18 DONE 0.0s
|
||||
|
||||
#5 [api 1/9] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
|
||||
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#19 [api 3/9] COPY requirements.txt .
|
||||
#19 CACHED
|
||||
|
||||
#20 [api 4/9] RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
#20 CACHED
|
||||
|
||||
#21 [api 7/9] COPY --chown=appuser:appuser db/ ./db/
|
||||
#21 CACHED
|
||||
|
||||
#8 [api 2/9] WORKDIR /app
|
||||
#8 CACHED
|
||||
|
||||
#22 [api 8/9] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
#22 CACHED
|
||||
|
||||
#23 [api 5/9] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
#23 CACHED
|
||||
|
||||
#24 [api 6/9] COPY --chown=appuser:appuser src/ ./src/
|
||||
#24 CACHED
|
||||
|
||||
#25 [api 9/9] COPY infra/app-start.sh /usr/local/bin/app-start
|
||||
#25 CACHED
|
||||
|
||||
#26 [api] exporting to image
|
||||
#26 exporting layers done
|
||||
#26 exporting manifest sha256:afc88e63bc12b5238aa444e8bdb808d7f36d417557b747036fa9fb8a16d604b3 done
|
||||
#26 exporting config sha256:c50287970425ce9631e003b98819e176d36d3d36d2b33f9de55894b567a27cc3 done
|
||||
#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179
|
||||
#26 exporting attestation manifest sha256:f4fe9e66dbc37d13c9b2f057987ada38e907ba0e4ff81600382b4b36b62fe179 0.0s done
|
||||
#26 exporting manifest list sha256:a8591065abe521d14b78bd1884238576e83d46e560f228be8faa90ec3e9a677f done
|
||||
#26 naming to docker.io/library/mtusi2026lab5-api:local done
|
||||
#26 unpacking to docker.io/library/mtusi2026lab5-api:local done
|
||||
#26 DONE 0.1s
|
||||
|
||||
#27 [api] resolving provenance for metadata file
|
||||
#27 DONE 0.0s
|
||||
api Built
|
||||
migrate Built
|
||||
Container mtusi2026lab5-s3-1 Running
|
||||
Container mtusi2026lab5-postgres-1 Running
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Created
|
||||
Container mtusi2026lab5-broker-1 Running
|
||||
Container mtusi2026lab5-redis-1 Running
|
||||
Container mtusi2026lab5-recovery-1 Stopping
|
||||
Container mtusi2026lab5-api-2 Stopping
|
||||
Container mtusi2026lab5-api-1 Stopping
|
||||
Container mtusi2026lab5-worker-1 Stopping
|
||||
Container mtusi2026lab5-worker-2 Stopping
|
||||
Container mtusi2026lab5-publisher-1 Stopping
|
||||
Container mtusi2026lab5-publisher-1 Stopped
|
||||
Container mtusi2026lab5-worker-1 Stopped
|
||||
Container mtusi2026lab5-worker-2 Stopped
|
||||
Container mtusi2026lab5-api-2 Stopped
|
||||
Container mtusi2026lab5-api-1 Stopped
|
||||
Container mtusi2026lab5-recovery-1 Stopped
|
||||
Container mtusi2026lab5-migrate-1 Recreate
|
||||
Container mtusi2026lab5-migrate-1 Recreated
|
||||
Container mtusi2026lab5-publisher-1 Recreate
|
||||
Container mtusi2026lab5-worker-1 Recreate
|
||||
Container mtusi2026lab5-worker-2 Recreate
|
||||
Container mtusi2026lab5-recovery-1 Recreate
|
||||
Container mtusi2026lab5-worker-1 Recreated
|
||||
Container mtusi2026lab5-worker-2 Recreated
|
||||
Container mtusi2026lab5-recovery-1 Recreated
|
||||
Container mtusi2026lab5-publisher-1 Recreated
|
||||
Container mtusi2026lab5-lb-1 Stopping
|
||||
Container mtusi2026lab5-lb-1 Stopped
|
||||
Container mtusi2026lab5-api-1 Recreate
|
||||
Container mtusi2026lab5-api-1 Recreated
|
||||
Container mtusi2026lab5-lb-1 Stopping
|
||||
Container mtusi2026lab5-lb-1 Stopped
|
||||
Container mtusi2026lab5-api-2 Recreate
|
||||
Container mtusi2026lab5-api-2 Recreated
|
||||
Container mtusi2026lab5-lb-1 Created
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab5-migrate-1 Starting
|
||||
Container mtusi2026lab5-migrate-1 Started
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-recovery-1 Starting
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-api-2 Starting
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-worker-2 Starting
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-publisher-1 Starting
|
||||
Container mtusi2026lab5-publisher-1 Started
|
||||
Container mtusi2026lab5-api-2 Started
|
||||
Container mtusi2026lab5-api-1 Starting
|
||||
Container mtusi2026lab5-recovery-1 Started
|
||||
Container mtusi2026lab5-worker-2 Started
|
||||
Container mtusi2026lab5-worker-1 Starting
|
||||
Container mtusi2026lab5-api-1 Started
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-worker-1 Started
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
Container mtusi2026lab5-lb-1 Starting
|
||||
Container mtusi2026lab5-lb-1 Started
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-worker-2 Waiting
|
||||
Container mtusi2026lab5-worker-1 Waiting
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-publisher-1 Waiting
|
||||
Container mtusi2026lab5-lb-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-recovery-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-recovery-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-publisher-1 Healthy
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
Container mtusi2026lab5-worker-2 Healthy
|
||||
Container mtusi2026lab5-worker-1 Healthy
|
||||
Container mtusi2026lab5-lb-1 Healthy
|
||||
@@ -0,0 +1,3 @@
|
||||
PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding
|
||||
PASS: fixture geometry, original preserved, deterministic v1, different v2
|
||||
PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed
|
||||
@@ -0,0 +1,11 @@
|
||||
mtusi2026lab5-api-1 12.98% 65.53MiB / 768MiB
|
||||
mtusi2026lab5-api-2 14.53% 65.53MiB / 768MiB
|
||||
mtusi2026lab5-broker-1 10.98% 92.7MiB / 768MiB
|
||||
mtusi2026lab5-lb-1 0.37% 11.71MiB / 128MiB
|
||||
mtusi2026lab5-postgres-1 2.31% 35.48MiB / 512MiB
|
||||
mtusi2026lab5-publisher-1 0.22% 26MiB / 256MiB
|
||||
mtusi2026lab5-recovery-1 0.09% 52.61MiB / 256MiB
|
||||
mtusi2026lab5-redis-1 0.33% 9.324MiB / 128MiB
|
||||
mtusi2026lab5-s3-1 3.14% 418.5MiB / 768MiB
|
||||
mtusi2026lab5-worker-1 0.16% 53.38MiB / 768MiB
|
||||
mtusi2026lab5-worker-2 0.18% 53.38MiB / 768MiB
|
||||
@@ -0,0 +1,4 @@
|
||||
/mtusi2026lab5-api-1 running healthy
|
||||
/mtusi2026lab5-api-2 running healthy
|
||||
/mtusi2026lab5-api-1 running healthy
|
||||
/mtusi2026lab5-api-2 exited unhealthy
|
||||
@@ -0,0 +1,9 @@
|
||||
Repeat 1
|
||||
/mtusi2026lab5-api-1 running healthy
|
||||
/mtusi2026lab5-api-2 running healthy
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 2, "first_recovery_seconds": 1.925}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
/mtusi2026lab5-api-1 exited unhealthy
|
||||
/mtusi2026lab5-api-2 running healthy
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 3, "first_recovery_seconds": 2.28}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,18 @@
|
||||
python3 tests/load.py --base-url "http://localhost:8080"
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30,
|
||||
"elapsed_seconds": 30.037,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7583,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 252.458,
|
||||
"successful_latency_ms": {
|
||||
"p50": 12.714,
|
||||
"p95": 37.804,
|
||||
"p99": 73.943
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
Fri Nov 20 15:25:49 UTC 2026
|
||||
276dbda
|
||||
Darwin arm64
|
||||
8
|
||||
17179869184
|
||||
Python 3.13.3
|
||||
Docker Compose version v2.33.1-desktop.1
|
||||
Engine=28.0.1 OS=linux arch=aarch64 CPUs=8 memory_bytes=8218316800
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding
|
||||
PASS: fixture geometry, original preserved, deterministic v1, different v2
|
||||
PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,75 @@
|
||||
Container mtusi2026lab5-s3-1 Running
|
||||
Container mtusi2026lab5-broker-1 Running
|
||||
Container mtusi2026lab5-postgres-1 Running
|
||||
Container mtusi2026lab5-redis-1 Running
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Created
|
||||
Container mtusi2026lab5-migrate-1 Created
|
||||
Container mtusi2026lab5-publisher-1 Running
|
||||
Container mtusi2026lab5-recovery-1 Running
|
||||
Container mtusi2026lab5-worker-1 Running
|
||||
Container mtusi2026lab5-api-1 Running
|
||||
Container mtusi2026lab5-worker-2 Running
|
||||
Container mtusi2026lab5-api-2 Running
|
||||
Container mtusi2026lab5-lb-1 Running
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-migrate-1 Starting
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab5-migrate-1 Started
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
Container mtusi2026lab5-recovery-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-broker-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-publisher-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-worker-1 Waiting
|
||||
Container mtusi2026lab5-worker-2 Waiting
|
||||
Container mtusi2026lab5-lb-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-broker-1 Healthy
|
||||
Container mtusi2026lab5-recovery-1 Healthy
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-publisher-1 Healthy
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-lb-1 Healthy
|
||||
Container mtusi2026lab5-worker-1 Healthy
|
||||
Container mtusi2026lab5-worker-2 Healthy
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
@@ -0,0 +1,11 @@
|
||||
/mtusi2026lab5-api-1 2026-11-20T15:26:46.863943835Z running
|
||||
/mtusi2026lab5-api-2 2026-11-20T15:26:46.921602543Z running
|
||||
/mtusi2026lab5-broker-1 2026-11-20T15:26:47.614707835Z running
|
||||
/mtusi2026lab5-lb-1 2026-11-20T15:26:46.466838501Z running
|
||||
/mtusi2026lab5-postgres-1 2026-11-20T15:26:46.30309871Z running
|
||||
/mtusi2026lab5-publisher-1 2026-11-20T15:26:46.328859918Z running
|
||||
/mtusi2026lab5-recovery-1 2026-11-20T15:26:46.960652918Z running
|
||||
/mtusi2026lab5-redis-1 2026-11-20T15:26:46.267219626Z running
|
||||
/mtusi2026lab5-s3-1 2026-11-20T15:26:46.428910043Z running
|
||||
/mtusi2026lab5-worker-1 2026-11-20T15:26:46.488719835Z running
|
||||
/mtusi2026lab5-worker-2 2026-11-20T15:26:46.475586876Z running
|
||||
@@ -0,0 +1,11 @@
|
||||
/mtusi2026lab5-api-1 2026-11-20T15:26:01.213628633Z running
|
||||
/mtusi2026lab5-api-2 2026-11-20T15:26:01.040028133Z running
|
||||
/mtusi2026lab5-broker-1 2026-11-20T15:23:46.29031721Z running
|
||||
/mtusi2026lab5-lb-1 2026-11-20T15:26:06.958527761Z running
|
||||
/mtusi2026lab5-postgres-1 2026-11-20T15:23:44.823417251Z running
|
||||
/mtusi2026lab5-publisher-1 2026-11-20T15:26:01.037922008Z running
|
||||
/mtusi2026lab5-recovery-1 2026-11-20T15:26:01.0402713Z running
|
||||
/mtusi2026lab5-redis-1 2026-11-20T15:23:44.789881876Z running
|
||||
/mtusi2026lab5-s3-1 2026-11-20T15:23:45.012262667Z running
|
||||
/mtusi2026lab5-worker-1 2026-11-20T15:26:01.26324605Z running
|
||||
/mtusi2026lab5-worker-2 2026-11-20T15:26:01.0369378Z running
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1 @@
|
||||
pipeline.py exit=0
|
||||
@@ -0,0 +1,75 @@
|
||||
Container mtusi2026lab5-track2-postgres-1 Running
|
||||
Container mtusi2026lab5-track2-s3-1 Running
|
||||
Container mtusi2026lab5-track2-broker-1 Running
|
||||
Container mtusi2026lab5-track2-redis-1 Running
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Created
|
||||
Container mtusi2026lab5-track2-migrate-1 Created
|
||||
Container mtusi2026lab5-track2-recovery-1 Running
|
||||
Container mtusi2026lab5-track2-publisher-1 Running
|
||||
Container mtusi2026lab5-track2-worker-1 Running
|
||||
Container mtusi2026lab5-track2-worker-2 Running
|
||||
Container mtusi2026lab5-track2-api-1 Running
|
||||
Container mtusi2026lab5-track2-api-2 Running
|
||||
Container mtusi2026lab5-track2-lb-1 Running
|
||||
Container mtusi2026lab5-track2-s3-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Healthy
|
||||
Container mtusi2026lab5-track2-migrate-1 Starting
|
||||
Container mtusi2026lab5-track2-s3-1 Healthy
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab5-track2-migrate-1 Started
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-redis-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-redis-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-api-2 Waiting
|
||||
Container mtusi2026lab5-track2-api-1 Waiting
|
||||
Container mtusi2026lab5-track2-api-2 Healthy
|
||||
Container mtusi2026lab5-track2-api-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-api-2 Waiting
|
||||
Container mtusi2026lab5-track2-api-1 Waiting
|
||||
Container mtusi2026lab5-track2-recovery-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Waiting
|
||||
Container mtusi2026lab5-track2-redis-1 Waiting
|
||||
Container mtusi2026lab5-track2-worker-1 Waiting
|
||||
Container mtusi2026lab5-track2-worker-2 Waiting
|
||||
Container mtusi2026lab5-track2-lb-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-1 Waiting
|
||||
Container mtusi2026lab5-track2-publisher-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Healthy
|
||||
Container mtusi2026lab5-track2-publisher-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-redis-1 Healthy
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-lb-1 Healthy
|
||||
Container mtusi2026lab5-track2-recovery-1 Healthy
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-1 Healthy
|
||||
Container mtusi2026lab5-track2-api-2 Healthy
|
||||
Container mtusi2026lab5-track2-api-1 Healthy
|
||||
Container mtusi2026lab5-track2-worker-1 Healthy
|
||||
Container mtusi2026lab5-track2-worker-2 Healthy
|
||||
@@ -0,0 +1,112 @@
|
||||
Container mtusi2026lab5-track2-postgres-1 Running
|
||||
Container mtusi2026lab5-track2-broker-1 Running
|
||||
Container mtusi2026lab5-track2-s3-1 Running
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Created
|
||||
Container mtusi2026lab5-track2-redis-1 Running
|
||||
Container mtusi2026lab5-track2-recovery-1 Stopping
|
||||
Container mtusi2026lab5-track2-publisher-1 Stopping
|
||||
Container mtusi2026lab5-track2-api-2 Stopping
|
||||
Container mtusi2026lab5-track2-api-1 Stopping
|
||||
Container mtusi2026lab5-track2-worker-1 Stopping
|
||||
Container mtusi2026lab5-track2-worker-2 Stopping
|
||||
Container mtusi2026lab5-track2-publisher-1 Stopped
|
||||
Container mtusi2026lab5-track2-worker-2 Stopped
|
||||
Container mtusi2026lab5-track2-worker-1 Stopped
|
||||
Container mtusi2026lab5-track2-api-2 Stopped
|
||||
Container mtusi2026lab5-track2-recovery-1 Stopped
|
||||
Container mtusi2026lab5-track2-api-1 Stopped
|
||||
Container mtusi2026lab5-track2-migrate-1 Recreate
|
||||
Container mtusi2026lab5-track2-migrate-1 Recreated
|
||||
Container mtusi2026lab5-track2-recovery-1 Recreate
|
||||
Container mtusi2026lab5-track2-publisher-1 Recreate
|
||||
Container mtusi2026lab5-track2-worker-1 Recreate
|
||||
Container mtusi2026lab5-track2-worker-2 Recreate
|
||||
Container mtusi2026lab5-track2-publisher-1 Recreated
|
||||
Container mtusi2026lab5-track2-recovery-1 Recreated
|
||||
Container mtusi2026lab5-track2-worker-2 Recreated
|
||||
Container mtusi2026lab5-track2-worker-1 Recreated
|
||||
Container mtusi2026lab5-track2-lb-1 Stopping
|
||||
Container mtusi2026lab5-track2-lb-1 Stopped
|
||||
Container mtusi2026lab5-track2-api-1 Recreate
|
||||
Container mtusi2026lab5-track2-api-1 Recreated
|
||||
Container mtusi2026lab5-track2-lb-1 Stopping
|
||||
Container mtusi2026lab5-track2-lb-1 Stopped
|
||||
Container mtusi2026lab5-track2-api-2 Recreate
|
||||
Container mtusi2026lab5-track2-api-2 Recreated
|
||||
Container mtusi2026lab5-track2-lb-1 Created
|
||||
Container mtusi2026lab5-track2-s3-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Healthy
|
||||
Container mtusi2026lab5-track2-migrate-1 Starting
|
||||
Container mtusi2026lab5-track2-s3-1 Healthy
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab5-track2-migrate-1 Started
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-redis-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-redis-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-api-2 Starting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-publisher-1 Starting
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-recovery-1 Starting
|
||||
Container mtusi2026lab5-track2-worker-1 Starting
|
||||
Container mtusi2026lab5-track2-recovery-1 Started
|
||||
Container mtusi2026lab5-track2-worker-1 Started
|
||||
Container mtusi2026lab5-track2-worker-2 Starting
|
||||
Container mtusi2026lab5-track2-publisher-1 Started
|
||||
Container mtusi2026lab5-track2-api-2 Started
|
||||
Container mtusi2026lab5-track2-api-1 Starting
|
||||
Container mtusi2026lab5-track2-api-1 Started
|
||||
Container mtusi2026lab5-track2-api-2 Waiting
|
||||
Container mtusi2026lab5-track2-api-1 Waiting
|
||||
Container mtusi2026lab5-track2-worker-2 Started
|
||||
Container mtusi2026lab5-track2-api-2 Healthy
|
||||
Container mtusi2026lab5-track2-api-1 Healthy
|
||||
Container mtusi2026lab5-track2-lb-1 Starting
|
||||
Container mtusi2026lab5-track2-lb-1 Started
|
||||
Container mtusi2026lab5-track2-lb-1 Waiting
|
||||
Container mtusi2026lab5-track2-api-2 Waiting
|
||||
Container mtusi2026lab5-track2-api-1 Waiting
|
||||
Container mtusi2026lab5-track2-recovery-1 Waiting
|
||||
Container mtusi2026lab5-track2-redis-1 Waiting
|
||||
Container mtusi2026lab5-track2-broker-1 Waiting
|
||||
Container mtusi2026lab5-track2-publisher-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-track2-s3-1 Waiting
|
||||
Container mtusi2026lab5-track2-migrate-1 Waiting
|
||||
Container mtusi2026lab5-track2-worker-1 Waiting
|
||||
Container mtusi2026lab5-track2-worker-2 Waiting
|
||||
Container mtusi2026lab5-track2-postgres-1 Waiting
|
||||
Container mtusi2026lab5-track2-recovery-1 Healthy
|
||||
Container mtusi2026lab5-track2-broker-1 Healthy
|
||||
Container mtusi2026lab5-track2-publisher-1 Healthy
|
||||
Container mtusi2026lab5-track2-postgres-1 Healthy
|
||||
Container mtusi2026lab5-track2-s3-1 Healthy
|
||||
Container mtusi2026lab5-track2-migrate-1 Exited
|
||||
Container mtusi2026lab5-track2-redis-1 Healthy
|
||||
Container mtusi2026lab5-track2-api-2 Healthy
|
||||
Container mtusi2026lab5-track2-api-1 Healthy
|
||||
Container mtusi2026lab5-track2-worker-1 Healthy
|
||||
Container mtusi2026lab5-track2-worker-2 Healthy
|
||||
Container mtusi2026lab5-track2-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-track2-lb-1 Healthy
|
||||
@@ -0,0 +1,15 @@
|
||||
{"scenario": "concurrent_message", "result": "PASS", "seconds": 0.041, "attachment_id": null, "requests": 6}
|
||||
{"scenario": "upload_validation", "result": "PASS", "seconds": 0.379, "attachment_id": null, "statuses": [404, 415, 415, 413]}
|
||||
{"scenario": "broker_unavailable", "result": "PASS", "seconds": 19.129, "attachment_id": "5b35bebd-3c16-431c-9167-1fae38db338d", "attempt": 1, "pipeline_health_recovered": true}
|
||||
{"scenario": "before_publish", "result": "PASS", "seconds": 12.328, "attachment_id": "ac1de244-d64b-43f2-98c9-d4662503eb35", "attempt": 1, "published": 1}
|
||||
{"scenario": "after_confirm", "result": "PASS", "seconds": 12.365, "attachment_id": "75602f19-039e-4bbf-bfb9-25c677d39ba9", "attempt": 1, "published": 1}
|
||||
{"scenario": "before_s3", "result": "PASS", "seconds": 28.879, "attachment_id": "e8fa8c69-04d7-4f51-b644-17b538a4f6c7", "attempt": 2, "published": 1}
|
||||
{"scenario": "after_s3", "result": "PASS", "seconds": 28.39, "attachment_id": "7c7439af-6b20-4941-afbb-41c8de02bf3c", "attempt": 2, "published": 1}
|
||||
{"scenario": "after_commit", "result": "PASS", "seconds": 7.57, "attachment_id": "753ad377-cac2-4630-8207-efeffce7522c", "attempt": 1, "published": 1}
|
||||
{"scenario": "duplicate_and_fencing", "result": "PASS", "seconds": 33.826, "attachment_id": "c0e36849-d59d-43ff-9f77-2af03cdd9bcf", "fence": 2, "attempt": 2}
|
||||
{"scenario": "reorder_in_flight", "result": "PASS", "seconds": 14.563, "attachment_id": "7717096b-a8a2-45d1-ad9c-d178ad545d4d", "active_version": 2}
|
||||
{"scenario": "late_old_delivery", "result": "PASS", "seconds": 16.117, "attachment_id": "0ee3fe7e-bd09-403d-8f6e-a7881ed2a1f0", "active_version": 2}
|
||||
{"scenario": "atomic_version_switch", "result": "PASS", "seconds": 16.302, "attachment_id": "3a58381d-2c97-4728-b3f3-4b72457f3e35", "active_version": 2}
|
||||
{"scenario": "temporary_error", "result": "PASS", "seconds": 8.366, "attachment_id": "7e9d1700-8dee-4622-9946-dc6bd2527a3b", "attempt": 2}
|
||||
{"scenario": "permanent_error", "result": "PASS", "seconds": 9.06, "attachment_id": "7853eedb-f16a-421d-870e-f72496843dc3", "attempt": 2}
|
||||
{"scenario": "restart_with_queued_attachment", "result": "PASS", "seconds": 23.39, "attachment_id": "e1baa3d5-bed2-48f7-ac29-5713b91b56d2", "attempt": 1, "message_link_preserved": true, "staging_cleaned": true}
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Compose file-based secrets inherit host ownership. Copy only the secrets
|
||||
# mounted for this role before dropping privileges.
|
||||
install -d -m 0700 -o appuser -g appuser /run/app-secrets
|
||||
for key in DATABASE_URL_FILE S3_ACCESS_KEY_ID_FILE S3_SECRET_ACCESS_KEY_FILE BROKER_URL_FILE; do
|
||||
source_file=$(printenv "$key") || continue
|
||||
target="/run/app-secrets/$key"
|
||||
install -m 0400 -o appuser -g appuser "$source_file" "$target"
|
||||
export "$key=$target"
|
||||
done
|
||||
exec setpriv --reuid=appuser --regid=appuser --init-groups -- "$@"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)"
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 60s
|
||||
timeout server 60s
|
||||
timeout check 1s
|
||||
# Never replay a potentially committed POST automatically.
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 1-10 api:8080 check
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy create local attachments-api /policy.json
|
||||
mc admin policy attach local attachments-api --user "$(cat /run/secrets/s3_access_key_id)"
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments/originals/*",
|
||||
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||
"arn:aws:s3:::messenger-attachments/results/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
],
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": [
|
||||
"originals/*", "staging/*", "results/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
pydantic==2.11.4
|
||||
psycopg[binary]==3.2.6
|
||||
psycopg-pool==3.2.6
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.38.12
|
||||
Pillow==11.2.1
|
||||
|
||||
pika==1.3.2
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then
|
||||
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
|
||||
fi
|
||||
exec docker compose --env-file .env "$@"
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Generate local credentials once; never overwrite an existing secret."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / 'secrets'
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
values = {'broker_password': secrets.token_hex(24), 'postgres_password': secrets.token_hex(24), 's3_root_user': 'lab5root', 's3_root_password': secrets.token_hex(24), 's3_access_key_id': 'lab5api', 's3_secret_access_key': secrets.token_hex(24)}
|
||||
for name, value in values.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + '\n')
|
||||
path.chmod(0o600)
|
||||
path = directory / 'database_url'
|
||||
if not path.exists():
|
||||
password = (directory / 'postgres_password').read_text().strip()
|
||||
path.write_text(f'postgresql://messenger:{password}@postgres:5432/messenger\n')
|
||||
path.chmod(0o600)
|
||||
path = directory / 'broker_url'
|
||||
if not path.exists():
|
||||
password = (directory / 'broker_password').read_text().strip()
|
||||
path.write_text(f'amqp://messenger:{password}@broker:5672/%2F\n')
|
||||
path.chmod(0o600)
|
||||
path = root / '.env'
|
||||
if not path.exists():
|
||||
path.write_text('COMPOSE_PROJECT_NAME=mtusi2026lab5\nPUBLIC_PORT=8080\n')
|
||||
print('Local configuration ready (secret values suppressed)')
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
containers=$(./scripts/compose.sh ps -aq postgres redis s3 broker api lb publisher worker recovery)
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-before.txt
|
||||
./scripts/compose.sh restart postgres redis s3 broker api lb publisher worker recovery
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-after.txt
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}"
|
||||
for container in $(./scripts/compose.sh ps -q api); do
|
||||
instance=$(docker exec "$container" python -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])')
|
||||
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop -t 2 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
exit 1
|
||||
@@ -0,0 +1,149 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .database import get_connection
|
||||
from .jobs import enqueue
|
||||
from .storage import s3, BUCKET
|
||||
|
||||
MAX_BYTES = 10 * 1024**2
|
||||
upload_slots = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
def serialize(row):
|
||||
result = {key: str(row[key]) for key in ('id', 'chat_id', 'owner_id')}
|
||||
result.update({key: row[key] for key in ('filename', 'content_type', 'size_bytes', 'sha256', 'status')})
|
||||
result.update({key: row[key].isoformat().replace('+00:00', 'Z') for key in ('created_at', 'updated_at')})
|
||||
manifest = row.get('manifest') or {}
|
||||
variants = [
|
||||
dict(name=name, **{field: obj[field] for field in
|
||||
('content_type', 'size_bytes', 'sha256', 'width', 'height')})
|
||||
for name, obj in sorted(manifest.items()) if name != 'original'
|
||||
]
|
||||
return dict(result, variants=variants, error_code=row['error_code'])
|
||||
|
||||
|
||||
def persist(path, chat, owner, filename, mime, size, checksum):
|
||||
attachment_id = uuid4()
|
||||
key = 'staging/' + str(attachment_id)
|
||||
# The same lock is taken by cleanup before its final DB check/deletion.
|
||||
# A process crash releases the transaction; an uncommitted object is later cleaned.
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (key,))
|
||||
with open(path, 'rb') as source:
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=source, ContentLength=size, ContentType=mime)
|
||||
row = conn.execute('''INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, object_key, size_bytes, content_type, sha256, staging_key, status)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *''',
|
||||
(attachment_id, UUID(chat), UUID(owner), filename, key, size, mime, checksum, key)).fetchone()
|
||||
enqueue(conn, attachment_id, 1)
|
||||
# Exiting transaction above commits before returning 202.
|
||||
return serialize(row)
|
||||
|
||||
|
||||
def accessible(attachment_id, user_id):
|
||||
try:
|
||||
aid = UUID(attachment_id)
|
||||
except ValueError:
|
||||
raise HTTPException(400, 'Неверный UUID') from None
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('''SELECT a.*, j.manifest FROM attachments a
|
||||
LEFT JOIN image_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.active_version
|
||||
JOIN chat_members cm ON cm.chat_id=a.chat_id
|
||||
WHERE a.id=%s AND cm.user_id=%s''', (aid, UUID(user_id))).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(404, 'Вложение не найдено')
|
||||
return row
|
||||
|
||||
|
||||
def register(app, current_user, get_chat):
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload(chat_id: str, request: Request):
|
||||
user = await run_in_threadpool(current_user, request)
|
||||
chat = await run_in_threadpool(get_chat, chat_id, request)
|
||||
filename = request.query_params.get('filename', '')
|
||||
if (not 1 <= len(filename) <= 255 or not filename.strip()
|
||||
or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename)):
|
||||
raise HTTPException(400, 'Недопустимое имя файла')
|
||||
mime = request.headers.get('content-type', '').split(';')[0].strip().lower()
|
||||
if mime not in ('image/png', 'image/jpeg'):
|
||||
raise HTTPException(415, 'Ожидается PNG или JPEG')
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal() or len(length) > 12:
|
||||
raise HTTPException(400, 'Неверный размер')
|
||||
if int(length) > MAX_BYTES:
|
||||
raise HTTPException(413, 'Файл больше 10 MiB')
|
||||
try:
|
||||
await asyncio.wait_for(upload_slots.acquire(), timeout=30)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(503, 'Загрузчик занят') from None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(prefix='lab5-') as target:
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(30):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise HTTPException(413, 'Файл больше 10 MiB')
|
||||
target.write(chunk)
|
||||
digest.update(chunk)
|
||||
except ClientDisconnect:
|
||||
raise HTTPException(400, 'Загрузка прервана') from None
|
||||
except TimeoutError:
|
||||
raise HTTPException(400, 'Истекло время загрузки') from None
|
||||
target.flush()
|
||||
def validate():
|
||||
try:
|
||||
return subprocess.run([sys.executable, '-m', 'src.image_check', target.name, mime],
|
||||
timeout=10, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
return 15
|
||||
code = await run_in_threadpool(validate)
|
||||
if code != 0:
|
||||
raise HTTPException(413 if code == 13 else 415, 'Недопустимое изображение')
|
||||
return await run_in_threadpool(persist, target.name, chat['id'], user['id'], filename, mime, size, digest.hexdigest())
|
||||
finally:
|
||||
upload_slots.release()
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def metadata(attachment_id: str, request: Request):
|
||||
return serialize(accessible(attachment_id, current_user(request)['id']))
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def content(attachment_id: str, request: Request):
|
||||
row = accessible(attachment_id, current_user(request)['id'])
|
||||
if row['status'] != 'ready':
|
||||
raise HTTPException(409, 'Вложение не готово')
|
||||
variant = row['manifest'].get(request.query_params.get('variant', 'original'))
|
||||
if variant is None:
|
||||
raise HTTPException(404, 'Вариант не найден')
|
||||
obj = s3.get_object(Bucket=BUCKET, Key=variant['key'])
|
||||
def chunks():
|
||||
try:
|
||||
for chunk in obj['Body'].iter_chunks(64 * 1024):
|
||||
yield chunk
|
||||
finally:
|
||||
obj['Body'].close()
|
||||
return StreamingResponse(chunks(), media_type=variant['content_type'], headers={
|
||||
'Content-Length': str(variant['size_bytes']), 'X-Content-Type-Options': 'nosniff',
|
||||
'Content-Disposition': 'attachment', 'Cache-Control': 'private, no-store'})
|
||||
|
||||
@app.exception_handler(BotoCoreError)
|
||||
@app.exception_handler(ClientError)
|
||||
async def storage_error(request, exc):
|
||||
from .main import error_response
|
||||
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
|
||||
password_hasher = PasswordHasher(
|
||||
time_cost=3,
|
||||
memory_cost=65536,
|
||||
parallelism=4,
|
||||
)
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return password_hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return password_hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def parse_basic(authorization: str | None) -> tuple[str, str] | None:
|
||||
if not authorization:
|
||||
return None
|
||||
|
||||
scheme, separator, credentials = authorization.partition(" ")
|
||||
|
||||
if not separator or scheme.lower() != "basic":
|
||||
return None
|
||||
|
||||
try:
|
||||
raw = base64.b64decode(credentials, validate=True)
|
||||
decoded = raw.decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
return None
|
||||
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
return None
|
||||
|
||||
if not 12 <= len(password) <= 128:
|
||||
return None
|
||||
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(authorization: str | None) -> str | None:
|
||||
if not authorization:
|
||||
return None
|
||||
|
||||
scheme, separator, token = authorization.partition(" ")
|
||||
|
||||
if not separator or scheme.lower() != "bearer":
|
||||
return None
|
||||
|
||||
if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512:
|
||||
return None
|
||||
|
||||
return token
|
||||
|
||||
|
||||
def create_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
|
||||
def session_expiration() -> datetime:
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
if ttl < 1:
|
||||
raise ValueError("SESSION_TTL_SECONDS must be positive")
|
||||
|
||||
return datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
|
||||
|
||||
def format_datetime(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Persistent RabbitMQ transport with explicit publisher confirms."""
|
||||
import json
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pika
|
||||
|
||||
QUEUE = 'attachment.process.v1'
|
||||
|
||||
|
||||
@contextmanager
|
||||
def channel():
|
||||
params = pika.URLParameters(Path(os.environ['BROKER_URL_FILE']).read_text().strip())
|
||||
params.connection_attempts = 1
|
||||
params.socket_timeout = 4
|
||||
params.stack_timeout = 8
|
||||
params.blocked_connection_timeout = 10
|
||||
params.heartbeat = 120
|
||||
connection = pika.BlockingConnection(params)
|
||||
try:
|
||||
ch = connection.channel()
|
||||
ch.queue_declare(QUEUE, durable=True, arguments={'x-queue-type': 'quorum', 'x-delivery-limit': -1})
|
||||
yield connection, ch
|
||||
finally:
|
||||
if connection.is_open:
|
||||
connection.close()
|
||||
|
||||
|
||||
def send(ch, message):
|
||||
ch.basic_publish('', QUEUE, json.dumps(message).encode(), mandatory=True,
|
||||
properties=pika.BasicProperties(content_type='application/json', delivery_mode=2,
|
||||
message_id=message['job_id']))
|
||||
@@ -0,0 +1,254 @@
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
|
||||
def serialize_chat(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(member_id) for member_id in row["member_ids"]],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def serialize_message(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"attachment_ids": [str(a) for a in row["attachment_ids"]],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def create_chat(creator_id, title, member_ids):
|
||||
chat_id = uuid4()
|
||||
members = [UUID(member_id) for member_id in member_ids]
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM users
|
||||
WHERE id = ANY(%s::uuid[])
|
||||
""",
|
||||
(members,),
|
||||
)
|
||||
existing = {row["id"] for row in cur.fetchall()}
|
||||
|
||||
if len(existing) != len(members):
|
||||
return None
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO chats (id, title, created_by)
|
||||
VALUES (%s, %s, %s)
|
||||
RETURNING id, title, created_at
|
||||
""",
|
||||
(chat_id, title, UUID(creator_id)),
|
||||
)
|
||||
chat = cur.fetchone()
|
||||
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO chat_members (chat_id, user_id)
|
||||
VALUES (%s, %s)
|
||||
""",
|
||||
[(chat_id, member_id) for member_id in members],
|
||||
)
|
||||
|
||||
return {
|
||||
"id": str(chat["id"]),
|
||||
"title": chat["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": chat["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def get_chat(chat_id, user_id):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
c.id,
|
||||
c.title,
|
||||
c.created_at,
|
||||
ARRAY(
|
||||
SELECT cm2.user_id
|
||||
FROM chat_members cm2
|
||||
WHERE cm2.chat_id = c.id
|
||||
ORDER BY cm2.user_id
|
||||
) AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE c.id = %s
|
||||
AND cm.user_id = %s
|
||||
""",
|
||||
(UUID(chat_id), UUID(user_id)),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return serialize_chat(row) if row else None
|
||||
|
||||
|
||||
def list_chats(user_id, limit, last_chat_id=None):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
last_key = None
|
||||
|
||||
if last_chat_id is not None:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT c.created_at, c.id
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE c.id = %s
|
||||
AND cm.user_id = %s
|
||||
""",
|
||||
(UUID(last_chat_id), UUID(user_id)),
|
||||
)
|
||||
last_key = cur.fetchone()
|
||||
|
||||
if last_key is None:
|
||||
return None
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
c.id,
|
||||
c.title,
|
||||
c.created_at,
|
||||
ARRAY(
|
||||
SELECT cm2.user_id
|
||||
FROM chat_members cm2
|
||||
WHERE cm2.chat_id = c.id
|
||||
ORDER BY cm2.user_id
|
||||
) AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE cm.user_id = %s
|
||||
AND (
|
||||
%s::timestamptz IS NULL
|
||||
OR (c.created_at, c.id) >
|
||||
(%s::timestamptz, %s::uuid)
|
||||
)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(
|
||||
UUID(user_id),
|
||||
last_key["created_at"] if last_key else None,
|
||||
last_key["created_at"] if last_key else None,
|
||||
last_key["id"] if last_key else None,
|
||||
limit + 1,
|
||||
),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [serialize_chat(row) for row in rows]
|
||||
|
||||
|
||||
def create_message(chat_id, sender_id, text, key, attachment_ids):
|
||||
# INSERT ON CONFLICT waits for a concurrent transaction on the same key.
|
||||
# READ COMMITTED's next statement sees its committed result. Both the
|
||||
# message and deduplication record commit together, or both roll back.
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
attachments = [UUID(a) for a in attachment_ids]
|
||||
scope = (UUID(chat_id), UUID(sender_id), key)
|
||||
conn.execute("""
|
||||
DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s
|
||||
AND created_at <= clock_timestamp() - interval '24 hours'
|
||||
""", scope)
|
||||
row = conn.execute(
|
||||
"""INSERT INTO message_requests(chat_id, sender_id, key, text)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (chat_id, sender_id, key) DO NOTHING
|
||||
RETURNING key""", (*scope, text)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
previous = conn.execute(
|
||||
"""SELECT m.* FROM message_requests r
|
||||
JOIN messages m ON m.id = r.message_id
|
||||
WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""",
|
||||
scope,
|
||||
).fetchone()
|
||||
if previous["text"] != text or previous["attachment_ids"] != attachments:
|
||||
return "conflict"
|
||||
return serialize_message(previous)
|
||||
rows = conn.execute(
|
||||
'SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) '
|
||||
'ORDER BY id FOR SHARE', (UUID(chat_id), attachments)
|
||||
).fetchall()
|
||||
if len(rows) != len(attachments) or any(row['status'] == 'failed' for row in rows):
|
||||
# No unfinished idempotency record may survive validation failure.
|
||||
conn.execute('DELETE FROM message_requests WHERE chat_id=%s AND sender_id=%s AND key=%s', scope)
|
||||
return None if len(rows) != len(attachments) else "attachment_failed"
|
||||
message = conn.execute(
|
||||
"""INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
|
||||
(uuid4(), UUID(chat_id), UUID(sender_id), text, attachments),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"""UPDATE message_requests SET message_id=%s
|
||||
WHERE chat_id=%s AND sender_id=%s AND key=%s""",
|
||||
(message["id"], *scope),
|
||||
)
|
||||
return serialize_message(message)
|
||||
|
||||
|
||||
def list_messages(chat_id, limit, last_message_id=None):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
last_key = None
|
||||
|
||||
if last_message_id is not None:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT created_at, id
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
AND id = %s
|
||||
""",
|
||||
(UUID(chat_id), UUID(last_message_id)),
|
||||
)
|
||||
last_key = cur.fetchone()
|
||||
|
||||
if last_key is None:
|
||||
return None
|
||||
|
||||
# Разные формы SQL сохраняют индексный range scan и для generic plan.
|
||||
# В SQL подставляется только константа; значения остаются параметрами.
|
||||
cursor_condition = ""
|
||||
params = [UUID(chat_id)]
|
||||
if last_key is not None:
|
||||
cursor_condition = "AND (created_at, id) > (%s, %s)"
|
||||
params.extend([last_key["created_at"], last_key["id"]])
|
||||
params.append(limit + 1)
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT id, chat_id, sender_id, text, attachment_ids, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
{cursor_condition}
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
params,
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [serialize_message(row) for row in rows]
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
|
||||
def get_database_url() -> str:
|
||||
secret_file = os.getenv("DATABASE_URL_FILE")
|
||||
|
||||
if secret_file:
|
||||
return Path(secret_file).read_text().strip()
|
||||
|
||||
database_url = os.getenv("DATABASE_URL")
|
||||
|
||||
if database_url:
|
||||
return database_url
|
||||
|
||||
raise RuntimeError("Database connection is not configured")
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo=get_database_url(),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=5,
|
||||
open=False,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": "-c statement_timeout=5000",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def open_database():
|
||||
pool.open(wait=True, timeout=15)
|
||||
|
||||
|
||||
def close_database():
|
||||
pool.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_connection():
|
||||
with pool.connection() as connection:
|
||||
yield connection
|
||||
@@ -0,0 +1,42 @@
|
||||
"""One-shot fault controls, enabled only by the operator on a disposable stand."""
|
||||
import os
|
||||
import time
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
ENABLED = os.getenv('PIPELINE_FAULTS') == '1'
|
||||
|
||||
|
||||
def take(conn, attachment_id, version, point):
|
||||
if not ENABLED:
|
||||
return None
|
||||
row = conn.execute("""
|
||||
UPDATE pipeline_faults SET hit_at=clock_timestamp()
|
||||
WHERE attachment_id=%s AND pipeline_version=%s AND point=%s AND hit_at IS NULL
|
||||
RETURNING action
|
||||
""", (attachment_id, version, point)).fetchone()
|
||||
return row['action'] if row else None
|
||||
|
||||
|
||||
def checkpoint(job, point):
|
||||
if not ENABLED:
|
||||
return None
|
||||
key = (job['attachment_id'], job['pipeline_version'], point)
|
||||
with get_connection() as conn:
|
||||
action = take(conn, *key)
|
||||
if action == 'crash':
|
||||
print(f'fault {point}: crash {job["id"]}', flush=True)
|
||||
os._exit(86)
|
||||
if action == 'pause':
|
||||
print(f'fault {point}: pause {job["id"]}', flush=True)
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
with get_connection() as conn:
|
||||
row = conn.execute("""
|
||||
SELECT released FROM pipeline_faults
|
||||
WHERE attachment_id=%s AND pipeline_version=%s AND point=%s
|
||||
""", key).fetchone()
|
||||
if row is None or row['released']:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
return action
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Isolated image decoder: bounded address space, CPU and parent wall timeout."""
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
if sys.platform.startswith('linux'):
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (8, 8))
|
||||
from PIL import Image
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
try:
|
||||
with Image.open(sys.argv[1]) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
sys.exit(13)
|
||||
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
|
||||
if actual != sys.argv[2] or getattr(image, 'n_frames', 1) != 1:
|
||||
sys.exit(15)
|
||||
image.verify()
|
||||
with Image.open(sys.argv[1]) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
sys.exit(13)
|
||||
except Exception:
|
||||
sys.exit(15)
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
"""PostgreSQL owns execution state and the publication pointer."""
|
||||
import os
|
||||
import random
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
||||
LEASE_SECONDS = int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))
|
||||
|
||||
|
||||
def enqueue(conn, attachment_id, version):
|
||||
job = conn.execute('''
|
||||
INSERT INTO image_jobs(attachment_id,pipeline_version) VALUES (%s,%s) RETURNING *
|
||||
''', (attachment_id, version)).fetchone()
|
||||
schedule(conn, job['id'])
|
||||
return job
|
||||
|
||||
|
||||
def schedule(conn, job_id, delay=0):
|
||||
conn.execute('''
|
||||
INSERT INTO job_outbox(job_id,publish_after)
|
||||
VALUES (%s, clock_timestamp() + %s * interval '1 second')
|
||||
ON CONFLICT (job_id) DO UPDATE SET publish_after=excluded.publish_after
|
||||
''', (job_id, delay))
|
||||
|
||||
|
||||
def event(conn, job, name, detail=None):
|
||||
conn.execute('INSERT INTO job_events(job_id,attempt,event,detail) VALUES (%s,%s,%s,%s)',
|
||||
(job['id'], job['attempt'], name, detail))
|
||||
|
||||
|
||||
def carrier(job):
|
||||
return dict(schema_version=1, type='attachment.process.v1', job_id=str(job['id']),
|
||||
attachment_id=str(job['attachment_id']), pipeline_version=job['pipeline_version'],
|
||||
attempt=job['attempt'], created_at=job['created_at'].isoformat().replace('+00:00', 'Z'))
|
||||
|
||||
|
||||
def claim(message):
|
||||
with get_connection() as conn:
|
||||
job = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE',
|
||||
(message['job_id'],)).fetchone()
|
||||
if (job is None or job['attachment_id'] != message['attachment_id']
|
||||
or job['pipeline_version'] != message['pipeline_version']
|
||||
or job['attempt'] != message['attempt']):
|
||||
return None
|
||||
if job['state'] != 'queued':
|
||||
event(conn, job, 'delivery_ignored', job['state'])
|
||||
return None
|
||||
attachment = lock_attachment(conn, job)
|
||||
if attachment['desired_version'] != job['pipeline_version']:
|
||||
supersede(conn, job)
|
||||
return None
|
||||
job = conn.execute('''
|
||||
UPDATE image_jobs SET state='processing', fence=fence+1,
|
||||
lease_until=clock_timestamp() + %s * interval '1 second'
|
||||
WHERE id=%s RETURNING *
|
||||
''', (LEASE_SECONDS, job['id'])).fetchone()
|
||||
update_attachment(conn, job, 'processing')
|
||||
event(conn, job, 'claimed', str(job['fence']))
|
||||
return job, attachment
|
||||
|
||||
|
||||
def lock_attachment(conn, job):
|
||||
return conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE',
|
||||
(job['attachment_id'],)).fetchone()
|
||||
|
||||
|
||||
def update_attachment(conn, job, state, error=None):
|
||||
# While building a new version, continue serving the previously published version.
|
||||
conn.execute('''
|
||||
UPDATE attachments SET status=%s, error_code=%s, updated_at=clock_timestamp()
|
||||
WHERE id=%s AND desired_version=%s AND active_version IS NULL
|
||||
''', (state, error, job['attachment_id'], job['pipeline_version']))
|
||||
|
||||
|
||||
def supersede(conn, job):
|
||||
conn.execute("UPDATE image_jobs SET state='superseded', lease_until=NULL WHERE id=%s", (job['id'],))
|
||||
event(conn, job, 'superseded')
|
||||
|
||||
|
||||
def owned(conn, job):
|
||||
current = conn.execute('''
|
||||
SELECT *, lease_until > clock_timestamp() AS live FROM image_jobs WHERE id=%s FOR NO KEY UPDATE
|
||||
''', (job['id'],)).fetchone()
|
||||
return current['state'] == 'processing' and current['fence'] == job['fence'] and current['live']
|
||||
|
||||
|
||||
def finish(job, manifest):
|
||||
with get_connection() as conn:
|
||||
if not owned(conn, job):
|
||||
event(conn, job, 'fence_rejected', str(job['fence']))
|
||||
return
|
||||
attachment = lock_attachment(conn, job)
|
||||
if not owned(conn, job):
|
||||
event(conn, job, 'fence_rejected', str(job['fence']))
|
||||
return
|
||||
if attachment['desired_version'] != job['pipeline_version']:
|
||||
supersede(conn, job)
|
||||
return
|
||||
conn.execute("""
|
||||
UPDATE image_jobs SET state='ready', manifest=%s, lease_until=NULL,error_code=NULL WHERE id=%s
|
||||
""", (Jsonb(manifest), job['id']))
|
||||
conn.execute("""
|
||||
UPDATE attachments SET status='ready', active_version=%s, error_code=NULL,
|
||||
object_key=%s, updated_at=clock_timestamp() WHERE id=%s
|
||||
""", (job['pipeline_version'], manifest['original']['key'], attachment['id']))
|
||||
event(conn, job, 'published')
|
||||
|
||||
|
||||
def fail_locked(conn, job, reason, permanent=False):
|
||||
attachment = lock_attachment(conn, job)
|
||||
if attachment['desired_version'] != job['pipeline_version']:
|
||||
supersede(conn, job)
|
||||
return
|
||||
failures = job['failures'] + 1
|
||||
if permanent or failures >= MAX_ATTEMPTS:
|
||||
code = reason if permanent else 'retry_exhausted'
|
||||
conn.execute("""
|
||||
UPDATE image_jobs SET state='failed', error_code=%s, lease_until=NULL, failures=%s WHERE id=%s
|
||||
""", (code, failures, job['id']))
|
||||
conn.execute('INSERT INTO job_dead_letters(job_id,attempt,error_code) VALUES (%s,%s,%s)',
|
||||
(job['id'], job['attempt'], code))
|
||||
update_attachment(conn, job, 'failed', code)
|
||||
event(conn, job, 'dead_letter', reason)
|
||||
return
|
||||
delay = min(30, 2 ** (failures - 1)) * random.uniform(0.75, 1.25)
|
||||
conn.execute("""
|
||||
UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=%s,
|
||||
lease_until=NULL,error_code=%s WHERE id=%s
|
||||
""", (failures, reason, job['id']))
|
||||
schedule(conn, job['id'], delay)
|
||||
update_attachment(conn, job, 'queued')
|
||||
event(conn, job, 'retry_scheduled', reason)
|
||||
|
||||
|
||||
def fail(job, reason, permanent=False):
|
||||
with get_connection() as conn:
|
||||
if owned(conn, job):
|
||||
fail_locked(conn, job, reason, permanent)
|
||||
else:
|
||||
event(conn, job, 'fence_rejected', str(job['fence']))
|
||||
|
||||
|
||||
def recover_one():
|
||||
with get_connection() as conn:
|
||||
job = conn.execute("""
|
||||
SELECT * FROM image_jobs WHERE state='processing' AND lease_until <= clock_timestamp()
|
||||
ORDER BY lease_until LIMIT 1 FOR NO KEY UPDATE SKIP LOCKED
|
||||
""").fetchone()
|
||||
if job:
|
||||
fail_locked(conn, job, 'lease_expired')
|
||||
return job is not None
|
||||
+519
@@ -0,0 +1,519 @@
|
||||
"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from pathlib import Path
|
||||
import logging
|
||||
import re
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
|
||||
|
||||
from psycopg import OperationalError, Error as DatabaseError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .auth import (
|
||||
create_token,
|
||||
format_datetime,
|
||||
hash_password,
|
||||
hash_token,
|
||||
parse_basic,
|
||||
parse_bearer,
|
||||
session_expiration,
|
||||
verify_password,
|
||||
)
|
||||
from .database import open_database, close_database, get_connection
|
||||
from .user_repository import (
|
||||
create_user as insert_user,
|
||||
create_session,
|
||||
find_session_user_cached,
|
||||
find_user_by_username,
|
||||
revoke_session,
|
||||
serialize_user,
|
||||
)
|
||||
|
||||
from .chat_repository import (
|
||||
create_chat as insert_chat,
|
||||
get_chat as find_chat,
|
||||
list_chats as find_chats,
|
||||
create_message as insert_message,
|
||||
list_messages as find_messages,
|
||||
)
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger", version="1.5.0")
|
||||
# Serve the mandatory contract in Swagger, including auth and request schemas.
|
||||
OPENAPI_SCHEMA = json.loads(
|
||||
(Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text()
|
||||
)
|
||||
app.openapi = lambda: OPENAPI_SCHEMA
|
||||
|
||||
INSTANCE_ID = str(uuid4())
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
503: "unavailable",
|
||||
}
|
||||
|
||||
|
||||
def error_response(request, status_code, code, message):
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# Логируем путь, метод и request_id, но НЕ тело сообщения.
|
||||
await run_in_threadpool(
|
||||
logger.error,
|
||||
"Unhandled error: %s %s request_id=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
request.state.request_id,
|
||||
)
|
||||
response = error_response(
|
||||
request, 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(request: Request, exc: StarletteHTTPException):
|
||||
response = error_response(
|
||||
request,
|
||||
exc.status_code,
|
||||
ERROR_CODES.get(exc.status_code, "invalid_request"),
|
||||
str(exc.detail),
|
||||
)
|
||||
if exc.headers:
|
||||
response.headers.update(exc.headers)
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(DatabaseError)
|
||||
@app.exception_handler(PoolTimeout)
|
||||
async def database_error_handler(request: Request, exc):
|
||||
return error_response(request, 503, "unavailable", "База данных временно недоступна")
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные данные запроса")
|
||||
|
||||
|
||||
async def read_json(request: Request):
|
||||
"""415 — только для неправильного Content-Type; 400 — для неверного JSON."""
|
||||
content_type = (
|
||||
request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
)
|
||||
if content_type != "application/json":
|
||||
raise HTTPException(415, "Неподдерживаемый Content-Type")
|
||||
|
||||
try:
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(raw) + len(chunk) > 65536:
|
||||
raise HTTPException(413, "JSON слишком большой")
|
||||
raw.extend(chunk)
|
||||
data = json.loads(raw)
|
||||
except (ValueError, UnicodeError):
|
||||
raise HTTPException(400, "Неверный JSON") from None
|
||||
|
||||
# [], строка, число, true, false, null — это корректный JSON,
|
||||
# но НЕ объект, обязательный по контракту.
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(400, "Ожидается JSON-объект")
|
||||
return data
|
||||
|
||||
|
||||
def get_current_user(request: Request):
|
||||
token = parse_bearer(request.headers.get("Authorization"))
|
||||
if token is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Требуется действующий Bearer-токен",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
try:
|
||||
user = find_session_user_cached(hash_token(token))
|
||||
except (OperationalError, PoolTimeout):
|
||||
raise HTTPException(
|
||||
503,
|
||||
"База данных временно недоступна",
|
||||
) from None
|
||||
if user is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Недействительная или истёкшая сессия",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def get_limit(request: Request):
|
||||
"""Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500."""
|
||||
limit_raw = request.query_params.get("limit", "50")
|
||||
if not limit_raw.isascii() or not limit_raw.isdecimal():
|
||||
raise HTTPException(400, "Неверный limit")
|
||||
|
||||
significant = limit_raw.lstrip("0")
|
||||
if not significant or len(significant) > 3:
|
||||
raise HTTPException(400, "Неверный limit")
|
||||
|
||||
limit = int(significant) # Теперь строка содержит максимум 3 цифры.
|
||||
if not 1 <= limit <= 100:
|
||||
raise HTTPException(400, "limit должен быть от 1 до 100")
|
||||
return limit
|
||||
|
||||
|
||||
def encode_cursor(first_id, last_id):
|
||||
raw = f"{first_id}|{last_id}".encode("ascii")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor):
|
||||
try:
|
||||
if not cursor or len(cursor) > 256:
|
||||
raise ValueError("empty or too long")
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
raw = base64.b64decode(
|
||||
padded, altchars=b"-_", validate=True
|
||||
).decode("ascii")
|
||||
first_id, last_id = raw.split("|")
|
||||
return str(UUID(first_id)), str(UUID(last_id))
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise HTTPException(400, "Неверный cursor") from None
|
||||
|
||||
|
||||
# ---- Health ----
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
try:
|
||||
with get_connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
from .storage import s3, BUCKET
|
||||
s3.list_objects_v2(Bucket=BUCKET, Prefix="originals/", MaxKeys=1)
|
||||
except (OperationalError, PoolTimeout):
|
||||
raise HTTPException(503, "База данных недоступна") from None
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# ---- Users ----
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
data = await read_json(request)
|
||||
|
||||
if set(data) != {"username", "display_name", "password"}:
|
||||
raise HTTPException(400, "Неверные поля пользователя")
|
||||
|
||||
username = data["username"]
|
||||
display_name = data["display_name"]
|
||||
password = data["password"]
|
||||
|
||||
if not isinstance(username, str) or not re.fullmatch(
|
||||
r"[a-z0-9_]{3,32}", username
|
||||
):
|
||||
raise HTTPException(400, "Недопустимый username")
|
||||
|
||||
if (
|
||||
not isinstance(display_name, str)
|
||||
or not 1 <= len(display_name) <= 100
|
||||
or not display_name.strip()
|
||||
):
|
||||
raise HTTPException(400, "Недопустимое имя пользователя")
|
||||
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
raise HTTPException(400, "Пароль должен содержать 12–128 символов")
|
||||
|
||||
password_hash = await run_in_threadpool(hash_password, password)
|
||||
user = await run_in_threadpool(insert_user, username, display_name, password_hash)
|
||||
if user is None:
|
||||
raise HTTPException(409, "Username уже занят")
|
||||
|
||||
return user
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def get_me(request: Request):
|
||||
return get_current_user(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def login(request: Request):
|
||||
credentials = parse_basic(request.headers.get("Authorization"))
|
||||
|
||||
if credentials is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Неверные учётные данные",
|
||||
headers={
|
||||
"WWW-Authenticate":
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
},
|
||||
)
|
||||
|
||||
username, password = credentials
|
||||
user_row = find_user_by_username(username)
|
||||
|
||||
if user_row is None or not verify_password(
|
||||
user_row["password_hash"], password
|
||||
):
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Неверные учётные данные",
|
||||
headers={
|
||||
"WWW-Authenticate":
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
},
|
||||
)
|
||||
|
||||
token = create_token()
|
||||
expires_at = session_expiration()
|
||||
|
||||
create_session(user_row["id"], hash_token(token), expires_at)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": format_datetime(expires_at),
|
||||
"user": serialize_user(user_row),
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request):
|
||||
get_current_user(request)
|
||||
token = parse_bearer(request.headers.get("Authorization"))
|
||||
|
||||
from .session_cache import invalidate_session
|
||||
|
||||
if not revoke_session(hash_token(token)):
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Недействительная сессия",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
invalidate_session(hash_token(token))
|
||||
|
||||
from fastapi import Response
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
|
||||
# ---- Chats ----
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await run_in_threadpool(get_current_user, request)
|
||||
data = await read_json(request)
|
||||
|
||||
if set(data) != {"title", "member_ids"}:
|
||||
raise HTTPException(400, "Неверные поля чата")
|
||||
|
||||
title = data["title"]
|
||||
member_ids = data["member_ids"]
|
||||
|
||||
if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip():
|
||||
raise HTTPException(400, "Недопустимое название чата")
|
||||
|
||||
if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100:
|
||||
raise HTTPException(400, "Неверный список участников")
|
||||
|
||||
normalized_ids = []
|
||||
|
||||
for member_id in member_ids:
|
||||
if not isinstance(member_id, str):
|
||||
raise HTTPException(400, "Неверный ID участника")
|
||||
|
||||
try:
|
||||
normalized_ids.append(str(UUID(member_id)))
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверный UUID участника") from None
|
||||
|
||||
if len(normalized_ids) != len(set(normalized_ids)):
|
||||
raise HTTPException(400, "Участники повторяются")
|
||||
|
||||
if user["id"] not in normalized_ids:
|
||||
raise HTTPException(400, "Создатель должен быть участником")
|
||||
|
||||
chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids)
|
||||
|
||||
if chat is None:
|
||||
raise HTTPException(400, "Неизвестный участник")
|
||||
|
||||
return chat
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request):
|
||||
user = get_current_user(request)
|
||||
limit = get_limit(request)
|
||||
cursor = request.query_params.get("cursor")
|
||||
|
||||
last_chat_id = None
|
||||
|
||||
if cursor is not None:
|
||||
cursor_user_id, last_chat_id = decode_cursor(cursor)
|
||||
|
||||
if cursor_user_id != user["id"]:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
rows = find_chats(user["id"], limit, last_chat_id)
|
||||
|
||||
if rows is None:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = (
|
||||
encode_cursor(user["id"], page[-1]["id"])
|
||||
if has_more and page
|
||||
else None
|
||||
)
|
||||
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: str, request: Request):
|
||||
user = get_current_user(request)
|
||||
|
||||
try:
|
||||
chat_id = str(UUID(chat_id))
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверный UUID чата") from None
|
||||
|
||||
chat = find_chat(chat_id, user["id"])
|
||||
|
||||
if chat is None:
|
||||
raise HTTPException(404, "Чат не найден")
|
||||
|
||||
return chat
|
||||
|
||||
|
||||
# ---- Messages ----
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def send_message(chat_id: str, request: Request):
|
||||
user = await run_in_threadpool(get_current_user, request)
|
||||
chat = await run_in_threadpool(get_chat, chat_id, request)
|
||||
data = await read_json(request)
|
||||
|
||||
if "text" not in data or set(data) - {"text", "attachment_ids"}:
|
||||
raise HTTPException(400, "Неверные поля сообщения")
|
||||
|
||||
text = data["text"]
|
||||
|
||||
attachment_ids = data.get("attachment_ids", [])
|
||||
if not isinstance(attachment_ids, list) or len(attachment_ids) > 10:
|
||||
raise HTTPException(400, "Неверный список вложений")
|
||||
try:
|
||||
if any(not isinstance(a, str) for a in attachment_ids):
|
||||
raise ValueError()
|
||||
attachment_ids = [str(UUID(a)) for a in attachment_ids]
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise ValueError()
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверные ID вложений") from None
|
||||
if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids):
|
||||
raise HTTPException(400, "Недопустимый текст сообщения")
|
||||
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise HTTPException(400, "Неверный Idempotency-Key")
|
||||
message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key, attachment_ids)
|
||||
if message == "attachment_failed":
|
||||
raise HTTPException(409, "Обработка вложения завершилась ошибкой")
|
||||
if message == "conflict":
|
||||
raise HTTPException(409, "Ключ использован для другого сообщения")
|
||||
|
||||
if message is None:
|
||||
raise HTTPException(404, "Чат не найден")
|
||||
|
||||
return message
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(chat_id: str, request: Request):
|
||||
chat = get_chat(chat_id, request)
|
||||
limit = get_limit(request)
|
||||
cursor = request.query_params.get("cursor")
|
||||
|
||||
last_message_id = None
|
||||
|
||||
if cursor is not None:
|
||||
cursor_chat_id, last_message_id = decode_cursor(cursor)
|
||||
|
||||
if cursor_chat_id != chat["id"]:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
rows = find_messages(chat["id"], limit, last_message_id)
|
||||
|
||||
if rows is None:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = (
|
||||
encode_cursor(chat["id"], page[-1]["id"])
|
||||
if has_more and page
|
||||
else None
|
||||
)
|
||||
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def start_database_pool():
|
||||
await run_in_threadpool(open_database)
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def stop_database_pool():
|
||||
await run_in_threadpool(close_database)
|
||||
|
||||
from .attachments import register
|
||||
register(app, get_current_user, get_chat)
|
||||
@@ -0,0 +1,16 @@
|
||||
"""One-shot migrations, serialized and recorded in PostgreSQL."""
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from .database import get_database_url
|
||||
|
||||
with psycopg.connect(get_database_url()) as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(310026)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())")
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / "db/migrations").glob("*.sql")):
|
||||
if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
|
||||
conn.execute(path.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,))
|
||||
print("Applied", path.name, flush=True)
|
||||
else:
|
||||
print("Already applied", path.name, flush=True)
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Local administrative commands. Run from the trusted Compose project."""
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from .broker import channel, send
|
||||
from .database import close_database, get_connection, open_database
|
||||
from .jobs import carrier, enqueue, event, schedule
|
||||
|
||||
|
||||
def reprocess(attachment_id, version):
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE', (attachment_id,)).fetchone()
|
||||
if not row or version <= row['desired_version']:
|
||||
raise ValueError('Version must increase for an existing attachment')
|
||||
if row['active_version'] is None and row['staging_key'] is None and row['object_key'].startswith('staging/'):
|
||||
raise ValueError('Original expired')
|
||||
conn.execute("""
|
||||
UPDATE attachments SET desired_version=%s,
|
||||
status=CASE WHEN active_version IS NULL THEN 'queued' ELSE status END,
|
||||
error_code=NULL,updated_at=clock_timestamp() WHERE id=%s
|
||||
""", (version, attachment_id))
|
||||
job = enqueue(conn, attachment_id, version)
|
||||
event(conn, job, 'reprocess_requested')
|
||||
return carrier(job)
|
||||
|
||||
|
||||
def replay(job_id):
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('SELECT * FROM image_jobs WHERE id=%s FOR NO KEY UPDATE', (job_id,)).fetchone()
|
||||
if not row or row['state'] != 'failed':
|
||||
raise ValueError('Replay requires a failed job')
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR NO KEY UPDATE',
|
||||
(row['attachment_id'],)).fetchone()
|
||||
if attachment['desired_version'] != row['pipeline_version']:
|
||||
raise ValueError('Replay requires the current pipeline version')
|
||||
if attachment['staging_key'] is None and attachment['object_key'].startswith('staging/'):
|
||||
raise ValueError('Staging retention expired')
|
||||
conn.execute("""
|
||||
UPDATE job_dead_letters SET replayed_at=clock_timestamp() WHERE job_id=%s AND attempt=%s
|
||||
""", (job_id, row['attempt']))
|
||||
updated = conn.execute("""
|
||||
UPDATE image_jobs SET state='queued', attempt=attempt+1, failures=0,
|
||||
error_code=NULL WHERE id=%s RETURNING *
|
||||
""", (job_id,)).fetchone()
|
||||
conn.execute("""
|
||||
UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp()
|
||||
WHERE id=%s AND active_version IS NULL
|
||||
""", (row['attachment_id'],))
|
||||
schedule(conn, job_id)
|
||||
event(conn, updated, 'operator_replay')
|
||||
return carrier(updated)
|
||||
|
||||
|
||||
def redeliver(job_id):
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('SELECT * FROM image_jobs WHERE id=%s', (job_id,)).fetchone()
|
||||
if not row:
|
||||
raise ValueError('Job not found')
|
||||
message = carrier(row)
|
||||
with channel() as (_, ch):
|
||||
ch.confirm_delivery()
|
||||
send(ch, message)
|
||||
return message
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest='command', required=True)
|
||||
new = commands.add_parser('reprocess')
|
||||
new.add_argument('attachment_id')
|
||||
new.add_argument('version', type=int, choices=[1, 2])
|
||||
commands.add_parser('replay').add_argument('job_id')
|
||||
commands.add_parser('redeliver').add_argument('job_id')
|
||||
args = parser.parse_args()
|
||||
open_database()
|
||||
try:
|
||||
if args.command == 'reprocess':
|
||||
result = reprocess(args.attachment_id, args.version)
|
||||
elif args.command == 'replay':
|
||||
result = replay(args.job_id)
|
||||
else:
|
||||
result = redeliver(args.job_id)
|
||||
print(json.dumps(result))
|
||||
except ValueError as exc:
|
||||
parser.exit(2, f'{exc}\n')
|
||||
finally:
|
||||
close_database()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Check recent progress by the service loop in this container."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
PROGRESS = Path('/tmp/pipeline-progress')
|
||||
MAX_IDLE_SECONDS = 60
|
||||
_last_write = float('-inf')
|
||||
|
||||
|
||||
def mark_progress():
|
||||
global _last_write
|
||||
now = time.monotonic()
|
||||
if now - _last_write >= 1:
|
||||
PROGRESS.touch()
|
||||
_last_write = now
|
||||
|
||||
|
||||
def clear_progress():
|
||||
global _last_write
|
||||
PROGRESS.unlink(missing_ok=True)
|
||||
_last_write = float('-inf')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
healthy = time.time() - PROGRESS.stat().st_mtime < MAX_IDLE_SECONDS
|
||||
except FileNotFoundError:
|
||||
healthy = False
|
||||
raise SystemExit(0 if healthy else 1)
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
import signal
|
||||
from threading import Event
|
||||
|
||||
import pika
|
||||
from psycopg import OperationalError
|
||||
from psycopg_pool import PoolTimeout
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .pipeline_health import mark_progress, clear_progress
|
||||
from .database import open_database, close_database
|
||||
|
||||
|
||||
def serve(loop):
|
||||
logging.basicConfig(level=logging.WARNING, format='%(asctime)s %(levelname)s %(message)s')
|
||||
stopping = Event()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: stopping.set())
|
||||
clear_progress()
|
||||
open_database()
|
||||
try:
|
||||
while not stopping.is_set():
|
||||
try:
|
||||
loop(stopping, mark_progress)
|
||||
except (OperationalError, PoolTimeout, BotoCoreError, ClientError, pika.exceptions.AMQPError, OSError) as exc:
|
||||
clear_progress()
|
||||
logging.warning('dependency failure: %s', type(exc).__name__)
|
||||
stopping.wait(1)
|
||||
finally:
|
||||
clear_progress()
|
||||
close_database()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Drain committed outbox rows, deleting only after a positive broker confirm."""
|
||||
import logging
|
||||
import os
|
||||
|
||||
from .broker import channel, send
|
||||
from .database import get_connection
|
||||
from .faults import take
|
||||
from .jobs import carrier, event
|
||||
from .process import serve
|
||||
|
||||
|
||||
def publish_one(ch):
|
||||
with get_connection() as conn:
|
||||
job = conn.execute("""
|
||||
SELECT j.* FROM job_outbox o JOIN image_jobs j ON j.id=o.job_id
|
||||
WHERE o.publish_after <= clock_timestamp()
|
||||
ORDER BY o.publish_after LIMIT 1 FOR UPDATE OF o SKIP LOCKED
|
||||
""").fetchone()
|
||||
if job is None:
|
||||
return False
|
||||
if job['state'] != 'queued':
|
||||
conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],))
|
||||
return True
|
||||
args = (job['attachment_id'], job['pipeline_version'])
|
||||
crash_before = take(conn, *args, 'before_publish') == 'crash'
|
||||
if not crash_before:
|
||||
send(ch, carrier(job))
|
||||
crash_after = take(conn, *args, 'after_confirm') == 'crash'
|
||||
if not crash_after:
|
||||
conn.execute('DELETE FROM job_outbox WHERE job_id=%s', (job['id'],))
|
||||
event(conn, job, 'broker_confirmed')
|
||||
if crash_before or crash_after:
|
||||
point = 'before_publish' if crash_before else 'after_confirm'
|
||||
logging.warning('fault %s: crash %s', point, job['id'])
|
||||
os._exit(86)
|
||||
return True
|
||||
|
||||
|
||||
def loop(stopping, progress):
|
||||
with channel() as (connection, ch):
|
||||
ch.confirm_delivery()
|
||||
while not stopping.is_set():
|
||||
if not publish_one(ch):
|
||||
connection.process_data_events(time_limit=0.2)
|
||||
stopping.wait(0.1)
|
||||
progress()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
serve(loop)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Reclaim expired jobs and remove staging after safe publication or retention."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import os
|
||||
import time
|
||||
|
||||
from .database import get_connection
|
||||
from .jobs import recover_one
|
||||
from .process import serve
|
||||
from .storage import BUCKET, s3
|
||||
|
||||
RETENTION = int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))
|
||||
|
||||
|
||||
def cleanup():
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=RETENTION)
|
||||
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='staging/'):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
with get_connection() as conn:
|
||||
locked = conn.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS ok',
|
||||
(key,)).fetchone()['ok']
|
||||
if not locked:
|
||||
continue
|
||||
row = conn.execute("""
|
||||
SELECT id,status,active_version,updated_at FROM attachments WHERE staging_key=%s FOR UPDATE
|
||||
""", (key,)).fetchone()
|
||||
if row:
|
||||
if not (row['active_version'] or (row['status'] == 'failed' and row['updated_at'] < cutoff)):
|
||||
continue
|
||||
elif obj['LastModified'] >= cutoff:
|
||||
continue
|
||||
s3.delete_object(Bucket=BUCKET, Key=key)
|
||||
if row:
|
||||
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s', (row['id'],))
|
||||
|
||||
|
||||
def loop(stopping, progress):
|
||||
next_cleanup = 0
|
||||
while not stopping.is_set():
|
||||
while not stopping.is_set() and recover_one():
|
||||
progress()
|
||||
if time.monotonic() >= next_cleanup:
|
||||
cleanup()
|
||||
next_cleanup = time.monotonic() + 15
|
||||
progress()
|
||||
stopping.wait(0.5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
serve(loop)
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Bounded subprocess for untrusted image decoding and resizing."""
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
# Linux runtime. macOS does not implement RLIMIT_AS with the same semantics.
|
||||
if sys.platform.startswith('linux'):
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def fit(image, bound):
|
||||
width, height = image.size
|
||||
largest = max(width, height)
|
||||
if largest <= bound:
|
||||
return image
|
||||
return image.resize((max(1, width * bound // largest), max(1, height * bound // largest)),
|
||||
Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def render(directory, version):
|
||||
with Image.open(directory / 'original') as source:
|
||||
image = ImageOps.exif_transpose(source)
|
||||
if 'A' in image.getbands() or 'transparency' in image.info:
|
||||
rgba = image.convert('RGBA')
|
||||
del image
|
||||
image = Image.new('RGB', rgba.size, 'white')
|
||||
image.paste(rgba, mask=rgba.getchannel('A'))
|
||||
del rgba
|
||||
else:
|
||||
image = image.convert('RGB')
|
||||
image.info.clear()
|
||||
jpeg_quality, webp_quality = {1: (85, 82), 2: (90, 88)}[version]
|
||||
fit(image, 256).save(directory / 'thumbnail', format='JPEG', quality=jpeg_quality)
|
||||
ImageOps.fit(image, (128, 128), Image.Resampling.LANCZOS).save(
|
||||
directory / 'crop', format='JPEG', quality=jpeg_quality)
|
||||
fit(image, 1280).save(directory / 'optimized', format='WEBP', quality=webp_quality, method=4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
render(Path(sys.argv[1]), int(sys.argv[2]))
|
||||
except (OSError, ValueError, Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
sys.exit(15)
|
||||
@@ -0,0 +1,48 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
|
||||
|
||||
def get_cache_url():
|
||||
secret_file = os.getenv("CACHE_URL_FILE")
|
||||
if secret_file:
|
||||
return Path(secret_file).read_text().strip()
|
||||
return os.getenv("CACHE_URL", "redis://redis:6379/0")
|
||||
|
||||
|
||||
client = redis.Redis.from_url(
|
||||
get_cache_url(),
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=1,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
def get_cached_user(token_hash):
|
||||
try:
|
||||
value = client.get("session:" + token_hash)
|
||||
return json.loads(value) if value else None
|
||||
except (redis.RedisError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def cache_user(token_hash, user, ttl):
|
||||
if ttl <= 0:
|
||||
return
|
||||
try:
|
||||
client.setex(
|
||||
"session:" + token_hash,
|
||||
ttl,
|
||||
json.dumps(user),
|
||||
)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
|
||||
|
||||
def invalidate_session(token_hash):
|
||||
try:
|
||||
client.delete("session:" + token_hash)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Private S3 access; no automatic PUT retries, explicit bounded timeouts."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.environ[name + '_FILE']).read_text().strip()
|
||||
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
s3 = boto3.client(
|
||||
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
|
||||
s3={'addressing_style': 'path'}),
|
||||
)
|
||||
@@ -0,0 +1,136 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from psycopg.errors import UniqueViolation
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
|
||||
def serialize_user(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def create_user(username, display_name, password_hash):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO users (
|
||||
id, username, display_name, password_hash
|
||||
)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
""",
|
||||
(uuid4(), username, display_name, password_hash),
|
||||
)
|
||||
return serialize_user(cur.fetchone())
|
||||
except UniqueViolation:
|
||||
conn.rollback()
|
||||
return None
|
||||
|
||||
|
||||
def find_user_by_username(username):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, username, display_name,
|
||||
password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = %s
|
||||
""",
|
||||
(username,),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def create_session(user_id, token_hash, expires_at):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO sessions (
|
||||
id, user_id, token_hash, expires_at
|
||||
)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
(uuid4(), user_id, token_hash, expires_at),
|
||||
)
|
||||
|
||||
|
||||
def revoke_session(token_hash):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET revoked_at = NOW()
|
||||
WHERE token_hash = %s
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > NOW()
|
||||
RETURNING id
|
||||
""",
|
||||
(token_hash,),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def find_session_user_cached(token_hash):
|
||||
from datetime import datetime, timezone
|
||||
from .session_cache import cache_user, get_cached_user
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
s.user_id,
|
||||
s.expires_at
|
||||
FROM sessions s
|
||||
WHERE s.token_hash = %s
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > NOW()
|
||||
""",
|
||||
(token_hash,),
|
||||
)
|
||||
session = cur.fetchone()
|
||||
|
||||
if session is None:
|
||||
return None
|
||||
|
||||
cached = get_cached_user(token_hash)
|
||||
|
||||
if cached is not None and str(cached.get("id")) == str(session["user_id"]):
|
||||
return cached
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, username, display_name, created_at
|
||||
FROM users
|
||||
WHERE id = %s
|
||||
""",
|
||||
(session["user_id"],),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
user = serialize_user(row)
|
||||
ttl = int(
|
||||
(session["expires_at"] - datetime.now(timezone.utc)).total_seconds()
|
||||
)
|
||||
|
||||
if ttl > 0:
|
||||
cache_user(token_hash, user, ttl)
|
||||
|
||||
return user
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
"""One in-flight delivery per worker process. ACK follows the durable decision."""
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from uuid import UUID
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image
|
||||
|
||||
from .broker import QUEUE, channel
|
||||
from .faults import checkpoint
|
||||
from .jobs import claim, fail, finish
|
||||
from .process import serve
|
||||
from .storage import BUCKET, s3
|
||||
|
||||
VARIANTS = {'thumbnail': 'image/jpeg', 'crop': 'image/jpeg', 'optimized': 'image/webp'}
|
||||
|
||||
|
||||
class PermanentError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def original(attachment):
|
||||
key = attachment['object_key']
|
||||
response = s3.get_object(Bucket=BUCKET, Key=key)
|
||||
body = response['Body']
|
||||
try:
|
||||
data = body.read(10 * 1024**2 + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if (len(data) != attachment['size_bytes']
|
||||
or hashlib.sha256(data).hexdigest() != attachment['sha256']):
|
||||
raise PermanentError('source_checksum_mismatch')
|
||||
return data
|
||||
|
||||
|
||||
def put(job, variant, data, mime, dimensions=None):
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
key = f'results/{job["attachment_id"]}/v{job["pipeline_version"]}/{variant}/{checksum}'
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=data, ContentType=mime)
|
||||
entry = dict(key=key, sha256=checksum, size_bytes=len(data), content_type=mime)
|
||||
if dimensions:
|
||||
entry.update(width=dimensions[0], height=dimensions[1])
|
||||
return entry
|
||||
|
||||
|
||||
def process(job, attachment):
|
||||
data = original(attachment)
|
||||
with tempfile.TemporaryDirectory(prefix='pipeline-') as target:
|
||||
directory = Path(target)
|
||||
(directory / 'original').write_bytes(data)
|
||||
proc = subprocess.run([sys.executable, '-m', 'src.render_image', target,
|
||||
str(job['pipeline_version'])], timeout=20, check=False)
|
||||
if proc.returncode:
|
||||
raise PermanentError('invalid_image' if proc.returncode == 15 else 'processing_limit')
|
||||
action = checkpoint(job, 'before_s3')
|
||||
if action == 'permanent':
|
||||
raise PermanentError('invalid_image')
|
||||
if action == 'temporary':
|
||||
raise OSError('injected transient error')
|
||||
manifest = {'original': put(job, 'original', data, attachment['content_type'])}
|
||||
for variant, mime in VARIANTS.items():
|
||||
path = directory / variant
|
||||
with Image.open(path) as image:
|
||||
dimensions = image.size
|
||||
manifest[variant] = put(job, variant, path.read_bytes(), mime, dimensions)
|
||||
checkpoint(job, 'after_s3')
|
||||
return manifest
|
||||
|
||||
|
||||
def handle(ch, method, body):
|
||||
try:
|
||||
message = json.loads(body)
|
||||
if (type(message['schema_version']) is not int or message['schema_version'] != 1
|
||||
or message['type'] != QUEUE
|
||||
or any(type(message[field]) is not int or message[field] < 1
|
||||
for field in ('pipeline_version', 'attempt'))):
|
||||
raise ValueError('Invalid carrier')
|
||||
if datetime.fromisoformat(message['created_at']).tzinfo is None:
|
||||
raise ValueError('Missing UTC offset')
|
||||
for field in ('job_id', 'attachment_id'):
|
||||
message[field] = UUID(message[field])
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
ch.basic_reject(method.delivery_tag, requeue=False)
|
||||
return
|
||||
claimed = claim(message)
|
||||
if claimed:
|
||||
job, attachment = claimed
|
||||
try:
|
||||
manifest = process(job, attachment)
|
||||
except PermanentError as exc:
|
||||
fail(job, str(exc), permanent=True)
|
||||
except subprocess.TimeoutExpired:
|
||||
fail(job, 'processing_limit', permanent=True)
|
||||
except ClientError as exc:
|
||||
code = exc.response['Error']['Code']
|
||||
fail(job, 'source_missing' if code in ('NoSuchKey', '404') else 'storage_unavailable',
|
||||
permanent=code in ('NoSuchKey', '404'))
|
||||
except (BotoCoreError, OSError):
|
||||
fail(job, 'temporary_failure')
|
||||
else:
|
||||
finish(job, manifest)
|
||||
checkpoint(job, 'after_commit')
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
|
||||
|
||||
def loop(stopping, progress):
|
||||
with channel() as (connection, ch):
|
||||
ch.basic_qos(prefetch_count=1)
|
||||
while not stopping.is_set():
|
||||
method, _, body = ch.basic_get(QUEUE, auto_ack=False)
|
||||
if method:
|
||||
handle(ch, method, body)
|
||||
else:
|
||||
connection.process_data_events(time_limit=0.2)
|
||||
progress()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
serve(loop)
|
||||
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Destructive fault injection for a disposable, dedicated LAB5 Compose project."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
COMPOSE = str(ROOT / 'scripts/compose.sh')
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run([COMPOSE, *args], check=True, text=True, capture_output=True, timeout=120).stdout.strip()
|
||||
|
||||
|
||||
def sql(query):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-XAt', '-v', 'ON_ERROR_STOP=1',
|
||||
'-U', 'messenger', '-d', 'messenger', '-c', query)
|
||||
|
||||
|
||||
def job(attachment, version=1):
|
||||
return json.loads(sql(f"""SELECT row_to_json(j) FROM image_jobs j
|
||||
WHERE attachment_id='{attachment['id']}' AND pipeline_version={version}"""))
|
||||
|
||||
|
||||
def events(job_id, name):
|
||||
return int(sql(f"SELECT count(*) FROM job_events WHERE job_id='{job_id}' AND event='{name}'"))
|
||||
|
||||
|
||||
def operator(command, identifier, *args):
|
||||
return json.loads(compose('exec', '-T', 'recovery', 'python', '-m', 'src.operator', command,
|
||||
identifier, *args))
|
||||
|
||||
|
||||
def wait(predicate, description, timeout=105):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f'timeout: {description}')
|
||||
|
||||
|
||||
def arm(attachment, point, action, version=1):
|
||||
sql(f"""INSERT INTO pipeline_faults(attachment_id,pipeline_version,point,action)
|
||||
VALUES ('{attachment['id']}',{version},'{point}','{action}')""")
|
||||
|
||||
|
||||
def hit(attachment, point, release=False):
|
||||
scope = f"attachment_id='{attachment['id']}' AND pipeline_version=1 AND point='{point}'"
|
||||
wait(lambda: sql(f'SELECT hit_at IS NOT NULL FROM pipeline_faults WHERE {scope}') == 't', point)
|
||||
if release:
|
||||
sql(f'UPDATE pipeline_faults SET released=true WHERE {scope}')
|
||||
|
||||
|
||||
def objects(attachment, version):
|
||||
prefix = f"results/{attachment['id']}/v{version}/"
|
||||
code = ("import json; from src.storage import s3,BUCKET; "
|
||||
f"print(json.dumps(sorted(x['Key'] for x in s3.list_objects_v2(Bucket=BUCKET,Prefix={prefix!r})"
|
||||
".get('Contents',[]))))")
|
||||
return json.loads(compose('exec', '-T', 'recovery', 'python', '-c', code))
|
||||
|
||||
|
||||
def report(name, started, attachment, **metrics):
|
||||
print(json.dumps(dict(scenario=name, result='PASS', seconds=round(time.monotonic() - started, 3),
|
||||
attachment_id=attachment['id'], **metrics), ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
def service_health(service):
|
||||
return [json.loads(line)['Health'] for line in
|
||||
compose('ps', '--format', 'json', service).splitlines()]
|
||||
|
||||
|
||||
def experiments(client):
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
worker_count = compose('ps', '-q', 'worker').splitlines()
|
||||
check(len(worker_count) >= 2, 'two workers are required')
|
||||
for service in ('publisher', 'worker'):
|
||||
enabled = compose('exec', '-T', service, 'printenv', 'PIPELINE_FAULTS')
|
||||
check(enabled == '1', f'fault injection disabled on {service}')
|
||||
|
||||
def upload():
|
||||
return client.attachment(alice, chat)
|
||||
|
||||
def meta(attachment, status=200):
|
||||
return client.call('GET', '/api/v1/attachments/' + attachment['id'], status,
|
||||
actor=bob, schema='Attachment')['json']
|
||||
|
||||
def verify(attachment, content, version=1):
|
||||
wait(lambda: job(attachment, version)['state'] == 'ready', f'job v{version} ready')
|
||||
ready = meta(attachment)
|
||||
check(ready['status'] == 'ready', 'attachment is not ready')
|
||||
row = job(attachment, version)
|
||||
active = int(sql(f"SELECT active_version FROM attachments WHERE id='{attachment['id']}'"))
|
||||
check(active == version, 'wrong active version')
|
||||
check(events(row['id'], 'published') == 1, 'expected one published manifest')
|
||||
check(set(row['manifest']) == {'original', 'thumbnail', 'crop', 'optimized'}, 'manifest incomplete')
|
||||
check(set(objects(attachment, version)) == {v['key'] for v in row['manifest'].values()},
|
||||
'unexpected keys or duplicated side effects')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content?variant='
|
||||
check({v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'}, 'wrong API variants')
|
||||
for variant, entry in row['manifest'].items():
|
||||
downloaded = client.call('GET', path + variant, actor=bob)
|
||||
data = downloaded['bytes']
|
||||
check(downloaded['headers'].get_content_type() == entry['content_type'], 'wrong MIME')
|
||||
check(len(data) == entry['size_bytes'], 'wrong size')
|
||||
check(hashlib.sha256(data).hexdigest() == entry['sha256'], 'wrong checksum')
|
||||
if variant == 'original':
|
||||
check(data == content, 'original changed')
|
||||
elif variant == 'optimized':
|
||||
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'invalid WebP signature')
|
||||
check((entry['width'], entry['height']) == (320, 200), 'WebP must not be enlarged')
|
||||
else:
|
||||
expected = (256, 160) if variant == 'thumbnail' else (128, 128)
|
||||
check(jpeg_size(data) == expected, 'wrong JPEG dimensions')
|
||||
return row
|
||||
|
||||
started = time.monotonic()
|
||||
key = 'concurrent-message-' + chat['id']
|
||||
def post_message(_):
|
||||
return client.message(alice, chat, 'concurrent', key=key)
|
||||
with ThreadPoolExecutor(max_workers=6) as executor:
|
||||
responses = list(executor.map(post_message, range(6)))
|
||||
check(len({response['json']['id'] for response in responses}) == 1, 'concurrent writes created duplicates')
|
||||
check(len({response['headers']['X-Instance-Id'] for response in responses}) >= 2,
|
||||
'concurrent requests did not reach two API instances')
|
||||
report('concurrent_message', started, {'id': None}, requests=6)
|
||||
|
||||
started = time.monotonic()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
|
||||
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
client.call('POST', path, 404, actor=eve, raw=fixture, headers={'Content-Type': 'image/png'})
|
||||
client.call('POST', path, 415, actor=alice, raw=fixture, headers={'Content-Type': 'image/jpeg'})
|
||||
client.call('POST', path, 415, actor=alice, raw=b'not a png', headers={'Content-Type': 'image/png'})
|
||||
client.call('POST', path, 413, actor=alice, raw=b'',
|
||||
headers={'Content-Type': 'image/png', 'Content-Length': '10485761'})
|
||||
report('upload_validation', started, {'id': None}, statuses=[404, 415, 415, 413])
|
||||
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'broker')
|
||||
try:
|
||||
attachment, content = upload()
|
||||
row = job(attachment)
|
||||
check(meta(attachment)['status'] == 'queued', 'upload did not remain queued')
|
||||
check(sql(f"SELECT count(*) FROM job_outbox WHERE job_id='{row['id']}'") == '1',
|
||||
'lost intent during real broker outage')
|
||||
wait(lambda: service_health('publisher') == ['unhealthy']
|
||||
and service_health('worker') == ['unhealthy'] * len(worker_count),
|
||||
'AMQP outage reflected in publisher and worker health', timeout=45)
|
||||
finally:
|
||||
compose('start', 'broker')
|
||||
verify(attachment, content)
|
||||
wait(lambda: service_health('publisher') == ['healthy']
|
||||
and service_health('worker') == ['healthy'] * len(worker_count),
|
||||
'publisher and all workers became healthy again', timeout=45)
|
||||
report('broker_unavailable', started, attachment, attempt=1, pipeline_health_recovered=True)
|
||||
|
||||
for point in ('before_publish', 'after_confirm', 'before_s3', 'after_s3', 'after_commit'):
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
if point in ('before_publish', 'after_confirm'):
|
||||
compose('stop', '-t', '1', 'worker')
|
||||
attachment, content = upload()
|
||||
arm(attachment, point, 'crash')
|
||||
compose('start', 'publisher')
|
||||
hit(attachment, point)
|
||||
if point in ('before_publish', 'after_confirm'):
|
||||
job_id = job(attachment)['id']
|
||||
wait(lambda: events(job_id, 'broker_confirmed'), 'outbox drained after publisher crash')
|
||||
compose('start', 'worker')
|
||||
if point in ('before_s3', 'after_s3'):
|
||||
keys = objects(attachment, 1)
|
||||
check(len(keys) == (0 if point == 'before_s3' else 4), 'wrong S3 effects at crash')
|
||||
check(job(attachment)['state'] == 'processing', 'manifest was already committed at crash')
|
||||
row = verify(attachment, content)
|
||||
wanted = {'before_s3': 2, 'after_s3': 2}.get(point, 1)
|
||||
check(row['attempt'] == wanted, f'wrong attempt after {point}')
|
||||
check(f'fault {point}: crash {row["id"]}' in compose('logs', '--no-color', 'publisher', 'worker'),
|
||||
'missing crash log')
|
||||
if point in ('after_commit', 'after_confirm'):
|
||||
wait(lambda: events(row['id'], 'delivery_ignored'), 'duplicate delivery ignored')
|
||||
report(point, started, attachment, attempt=row['attempt'], published=1)
|
||||
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
attachment, content = upload()
|
||||
arm(attachment, 'before_s3', 'pause')
|
||||
compose('start', 'publisher')
|
||||
hit(attachment, 'before_s3')
|
||||
initial = job(attachment)
|
||||
operator('redeliver', initial['id'])
|
||||
wait(lambda: events(initial['id'], 'delivery_ignored'), 'parallel duplicate rejected')
|
||||
check(events(initial['id'], 'claimed') == 1, 'duplicate obtained live lease')
|
||||
current = verify(attachment, content)
|
||||
check(current['fence'] > initial['fence'] and current['attempt'] == 2, 'lease was not reclaimed')
|
||||
hit(attachment, 'before_s3', release=True)
|
||||
wait(lambda: events(initial['id'], 'fence_rejected'), 'old attempt fenced')
|
||||
ignored = events(initial['id'], 'delivery_ignored')
|
||||
operator('redeliver', initial['id'])
|
||||
wait(lambda: events(initial['id'], 'delivery_ignored') > ignored, 'completed duplicate rejected')
|
||||
check(verify(attachment, content)['manifest'] == current['manifest'], 'duplicate changed the manifest')
|
||||
report('duplicate_and_fencing', started, attachment, fence=current['fence'], attempt=current['attempt'])
|
||||
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
attachment, content = upload()
|
||||
arm(attachment, 'after_s3', 'pause')
|
||||
compose('start', 'publisher')
|
||||
hit(attachment, 'after_s3')
|
||||
operator('reprocess', attachment['id'], '2')
|
||||
second = verify(attachment, content, 2)
|
||||
hit(attachment, 'after_s3', release=True)
|
||||
old = job(attachment)
|
||||
wait(lambda: job(attachment)['state'] == 'superseded' or events(old['id'], 'fence_rejected'),
|
||||
'v1 completed after v2 without publication')
|
||||
ignored = events(old['id'], 'delivery_ignored')
|
||||
operator('redeliver', old['id'])
|
||||
wait(lambda: events(old['id'], 'delivery_ignored') > ignored, 'reordered v1 redelivery ignored')
|
||||
check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'v1 changed v2')
|
||||
check(events(old['id'], 'published') == 0, 'v1 published despite newer version')
|
||||
report('reorder_in_flight', started, attachment, active_version=2)
|
||||
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
attachment, content = upload()
|
||||
old = job(attachment)
|
||||
second = operator('reprocess', attachment['id'], '2')
|
||||
operator('redeliver', second['job_id'])
|
||||
verify(attachment, content, 2)
|
||||
check(job(attachment)['state'] == 'queued', 'v1 must still be queued before late delivery')
|
||||
operator('redeliver', old['id'])
|
||||
wait(lambda: events(old['id'], 'superseded') == 1, 'late v1 delivery superseded')
|
||||
check(events(old['id'], 'claimed') == 0, 'late v1 obtained a lease')
|
||||
compose('start', 'publisher')
|
||||
check(not objects(attachment, 1), 'late v1 produced S3 side effects')
|
||||
verify(attachment, content, 2)
|
||||
report('late_old_delivery', started, attachment, active_version=2)
|
||||
|
||||
started = time.monotonic()
|
||||
attachment, content = upload()
|
||||
first = verify(attachment, content)
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
try:
|
||||
operator('reprocess', attachment['id'], '2')
|
||||
check(meta(attachment)['status'] == 'ready', 'reprocessing hid the published version')
|
||||
visible = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content?variant=thumbnail',
|
||||
actor=bob)['bytes']
|
||||
check(hashlib.sha256(visible).hexdigest() == first['manifest']['thumbnail']['sha256'],
|
||||
'published version changed before v2 commit')
|
||||
finally:
|
||||
compose('start', 'publisher')
|
||||
second = verify(attachment, content, 2)
|
||||
for name in ('thumbnail', 'crop', 'optimized'):
|
||||
check(first['manifest'][name]['sha256'] != second['manifest'][name]['sha256'], 'v2 did not change bytes')
|
||||
operator('redeliver', first['id'])
|
||||
wait(lambda: events(first['id'], 'delivery_ignored'), 'duplicate of published v1')
|
||||
check(verify(attachment, content, 2)['manifest'] == second['manifest'], 'old duplicate changed v2')
|
||||
report('atomic_version_switch', started, attachment, active_version=2)
|
||||
|
||||
for action in ('temporary', 'permanent'):
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'publisher')
|
||||
attachment, content = upload()
|
||||
arm(attachment, 'before_s3', action)
|
||||
compose('start', 'publisher')
|
||||
initial = job(attachment)
|
||||
if action == 'permanent':
|
||||
wait(lambda: job(attachment)['state'] == 'failed', 'permanent failure')
|
||||
check(meta(attachment)['error_code'] == 'invalid_image', 'unsafe or wrong error code')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content'
|
||||
client.call('GET', path, 409, actor=bob)
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': attachment['id']})
|
||||
check(sql(f"SELECT error_code FROM job_dead_letters WHERE job_id='{initial['id']}'") == 'invalid_image',
|
||||
'missing DLQ entry')
|
||||
operator('replay', initial['id'])
|
||||
check(sql(f"SELECT replayed_at IS NOT NULL FROM job_dead_letters WHERE job_id='{initial['id']}'") == 't',
|
||||
'missing replay audit')
|
||||
completed = verify(attachment, content)
|
||||
check(completed['attempt'] == 2, 'expected two attempts')
|
||||
if action == 'temporary':
|
||||
check(events(initial['id'], 'retry_scheduled') == 1, 'missing transient retry')
|
||||
report(action + '_error', started, attachment, attempt=2)
|
||||
|
||||
stable_attachment, stable_content = attachment, content
|
||||
stable_manifest = completed['manifest']
|
||||
started = time.monotonic()
|
||||
compose('stop', '-t', '1', 'worker')
|
||||
attachment, content = upload()
|
||||
row = job(attachment)
|
||||
wait(lambda: events(row['id'], 'broker_confirmed'), 'durable queue entry')
|
||||
check(meta(attachment)['status'] == 'queued', 'worker processed despite stop')
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob)
|
||||
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
compose('restart', 'postgres', 'redis', 's3', 'broker', 'api', 'lb', 'publisher', 'recovery')
|
||||
wait(lambda: health(client), 'HAProxy readiness')
|
||||
check(not compose('ps', '-q', 'worker'), 'restart brought workers back too early')
|
||||
check(meta(attachment)['status'] == 'queued', 'confirmed job changed before worker restart')
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=bob)
|
||||
compose('start', 'worker')
|
||||
verify(attachment, content)
|
||||
page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=bob,
|
||||
schema='MessagePage')['json']
|
||||
check(any(item['id'] == message['id'] and attachment['id'] in item['attachment_ids']
|
||||
for item in page['items']), 'attachment link did not survive restart')
|
||||
wait(lambda: sql(f"SELECT staging_key IS NULL FROM attachments WHERE id='{attachment['id']}'") == 't',
|
||||
'recovery cleaned staging after publication')
|
||||
wait(lambda: service_health('recovery') == ['healthy'], 'recovery health after restart')
|
||||
check(verify(stable_attachment, stable_content)['manifest'] == stable_manifest,
|
||||
'published attachment changed on restart')
|
||||
check(job(attachment)['attempt'] == 1, 'broker restart lost the original delivery')
|
||||
report('restart_with_queued_attachment', started, attachment, attempt=1,
|
||||
message_link_preserved=True, staging_cleaned=True)
|
||||
|
||||
|
||||
def health(client):
|
||||
try:
|
||||
client.call('GET', '/health/ready')
|
||||
except (OSError, AssertionError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--disposable', action='store_true', required=True,
|
||||
help='confirm this is an isolated test project')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
experiments(Client(args.base_url))
|
||||
except (AssertionError, OSError, ValueError, subprocess.SubprocessError) as exc:
|
||||
# Subprocess output can include internal details. Keep the public summary short.
|
||||
print('FAIL:', str(exc), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check image encoders, geometry, EXIF orientation and alpha on local fixtures."""
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE = ROOT / 'tests/fixtures/sample.png'
|
||||
|
||||
|
||||
def render(directory, version):
|
||||
subprocess.run([sys.executable, '-m', 'src.render_image', str(directory), str(version)],
|
||||
cwd=ROOT, check=True)
|
||||
outputs = {}
|
||||
for variant, format_name in [('thumbnail', 'JPEG'), ('crop', 'JPEG'), ('optimized', 'WEBP')]:
|
||||
with Image.open(directory / variant) as image:
|
||||
image.load()
|
||||
assert image.format == format_name
|
||||
assert not image.getexif()
|
||||
outputs[variant] = (image.size, (directory / variant).read_bytes())
|
||||
return outputs
|
||||
|
||||
|
||||
def validate(path, mime):
|
||||
return subprocess.run([sys.executable, '-m', 'src.image_check', str(path), mime],
|
||||
cwd=ROOT, timeout=10, check=False).returncode
|
||||
|
||||
|
||||
def main():
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
original = directory / 'original'
|
||||
data = FIXTURE.read_bytes()
|
||||
original.write_bytes(data)
|
||||
assert validate(original, 'image/png') == 0
|
||||
assert validate(original, 'image/jpeg') == 15
|
||||
original.write_bytes(b'corrupt')
|
||||
assert validate(original, 'image/png') == 15
|
||||
# Rewrite only IHDR, with a valid CRC, to exercise the pre-decode limit.
|
||||
header = data[12:16] + struct.pack('>II', 10000, 10000) + data[24:29]
|
||||
original.write_bytes(data[:12] + header + struct.pack('>I', zlib.crc32(header)) + data[33:])
|
||||
assert validate(original, 'image/png') == 13
|
||||
original.write_bytes(data)
|
||||
print('PASS: PNG validation, MIME mismatch, corrupt input and pixel cap before decoding')
|
||||
first = render(directory, 1)
|
||||
assert first == render(directory, 1)
|
||||
second = render(directory, 2)
|
||||
assert first.keys() == second.keys()
|
||||
for variant, shape in [('thumbnail', (256, 160)), ('crop', (128, 128)), ('optimized', (320, 200))]:
|
||||
assert first[variant][0] == second[variant][0] == shape
|
||||
assert first[variant][1] != second[variant][1]
|
||||
assert original.read_bytes() == data
|
||||
print('PASS: fixture geometry, original preserved, deterministic v1, different v2')
|
||||
transparent = Image.new('RGBA', (7, 3), (255, 0, 0, 0))
|
||||
transparent.save(original, format='PNG')
|
||||
alpha = render(directory, 1)
|
||||
assert alpha['thumbnail'][0] == (7, 3)
|
||||
with Image.open(directory / 'thumbnail') as thumbnail:
|
||||
assert thumbnail.getpixel((0, 0)) == (255, 255, 255)
|
||||
rotated = Image.new('RGB', (17, 31), 'red')
|
||||
exif = rotated.getexif()
|
||||
exif[274] = 6
|
||||
rotated.save(original, format='JPEG', exif=exif)
|
||||
assert render(directory, 1)['thumbnail'][0] == (31, 17)
|
||||
print('PASS: alpha on white, no upscaling, EXIF orientation applied and metadata removed')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user