Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec9809bd0e | ||
|
|
1d7d71870b | ||
|
|
ec277e7e64 |
+22
@@ -0,0 +1,22 @@
|
||||
FROM python:3.11.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
DATABASE_URL=postgresql://messenger:messenger@db:5432/messenger \
|
||||
SESSION_TTL_SECONDS=3600
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
ARG APP_REVISION=lab5-v1
|
||||
ENV APP_REVISION=$APP_REVISION
|
||||
COPY app.py attachments.py pipeline.py worker.py publisher.py transform.py janitor.py manage.py migrate.py /app/
|
||||
COPY migrations /app/migrations
|
||||
COPY scripts /app/scripts
|
||||
COPY tests /app/tests
|
||||
COPY contracts /app/contracts
|
||||
COPY lab.json /app/lab.json
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://localhost:8085
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -28,3 +28,29 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: up down research test-lifecycle test-backpressure test-extra
|
||||
up:
|
||||
./scripts/bootstrap.sh
|
||||
./scripts/compose.sh up -d --build
|
||||
$(PYTHON) scripts/wait_ready.py --base-url "http://localhost:8085"
|
||||
|
||||
down:
|
||||
./scripts/compose.sh down
|
||||
|
||||
research:
|
||||
./scripts/research.sh
|
||||
|
||||
# These manage real containers; run sequentially, never concurrently.
|
||||
test-lifecycle:
|
||||
$(PYTHON) tests/lifecycle.py
|
||||
|
||||
test-backpressure:
|
||||
$(PYTHON) tests/backpressure.py
|
||||
|
||||
test-extra:
|
||||
./scripts/compose.sh run --rm --no-deps ops python tests/uploads_negative.py
|
||||
./scripts/compose.sh run --rm --no-deps ops python tests/image_variants.py
|
||||
./scripts/compose.sh run --rm --no-deps ops python tests/private_storage.py
|
||||
./scripts/compose.sh run --rm --no-deps -e BASE_URL=http://lb:8080 ops python tests/cache_consistency.py
|
||||
$(PYTHON) tests/idempotency_concurrent.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор: healby; группа не указана. Целевая оценка: 5. Трек 2: идемпотентность versioned pipeline.
|
||||
|
||||
Сохранён мессенджер из lab4; upload теперь возвращает 202 после устойчивого staging и транзакции metadata/job/outbox. Добавлены отдельные Valkey Streams broker, publisher, worker и cleanup; thumbnail/crop/WebP, retry/backoff/jitter, failed registry/replay, lease и защита версий.
|
||||
|
||||
Запуск: `make up`, `make check`, `make test`. Полные команды и реальные доказательства: [REPORT.md](REPORT.md), исследование с таблицей/графиком: [RESEARCH.md](RESEARCH.md).
|
||||
|
||||
Проверены public smoke, OpenAPI, отрицательные сценарии, приватность S3, варианты, дубли/аварии/версии, restart API/broker, реальный lease, backpressure, persistence всех сервисов и failover API. Ограничения: один host, без HA дисков/backup; ускоренные часы в части fault-матрицы обозначены отдельно.
|
||||
|
||||
Источники: официальный Valkey/Pillow/PostgreSQL и исходные контракты курса; Codex использован для реализации, тестов и документации.
|
||||
|
||||
- [x] Весь обязательный минимум сохранён с объявленным переходом upload 201 → 202.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены, реальный вывод приложен.
|
||||
- [x] Сценарии отказов/безопасности воспроизведены.
|
||||
- [x] Локальные секреты исключены из Git, токены не включены в отчёт.
|
||||
- [x] README задания сохранён, REPORT.md описывает именно это решение.
|
||||
@@ -1,37 +1,224 @@
|
||||
# Отчёт — лабораторная 5
|
||||
# Лабораторная 5. Очереди и асинхронная обработка изображений
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: Савин Александр БВТ2303;
|
||||
- Целевая оценка: **5**, выполнены уровни 3 и 4; **трек 2 — доказанная идемпотентность pipeline**.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Git, Python 3.9+ для публичных HTTP-проверок, Python 3.11+ для локального окружения исследования, Docker Engine с Compose v2. На macOS использована Colima, Linux arm64, 4 vCPU и около 8 GiB RAM. Docker-версии и фактические ресурсы: [runtime.json](evidence/runtime.json), [containers.json](evidence/containers.json). Синтетический HTTP-стенд доступен только на `127.0.0.1:8085`.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
git clone https://gitea.privcloud.tech/healby/mtusi2026lab5.git
|
||||
cd mtusi2026lab5
|
||||
./scripts/bootstrap.sh
|
||||
./scripts/compose.sh config --quiet
|
||||
make up
|
||||
make check
|
||||
make test
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
# Полная проверка OpenAPI и генератор графиков:
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-research.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
# Проверки предыдущих возможностей и вариантов:
|
||||
make test-extra
|
||||
# Автоматические crash/retry/version/cleanup-сценарии:
|
||||
make research
|
||||
# Эти сценарии управляют сервисами; выполнять последовательно:
|
||||
python3 tests/lifecycle.py > evidence/lifecycle.json
|
||||
python3 tests/backpressure.py > evidence/backpressure.json
|
||||
python3 scripts/container_snapshot.py > evidence/persistence-before.json
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh > evidence/persistence.txt
|
||||
python3 scripts/container_snapshot.py > evidence/persistence-after.json
|
||||
python3 scripts/verify_persistence.py > evidence/persistence-verified.json
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
./scripts/compose.sh start api
|
||||
python3 scripts/wait_ready.py
|
||||
python3 scripts/container_snapshot.py > evidence/containers.json
|
||||
.venv/bin/python scripts/summarize_research.py
|
||||
python3 scripts/summarize_operations.py
|
||||
```
|
||||
|
||||
`bootstrap.sh` создаёт `.env` с тремя случайными секретами и портом 8085, не перезаписывая существующий файл. `.env` исключён из Git и Docker build context. Миграции PostgreSQL и создание приватного bucket/ограниченной учётной записи S3 выполняют одноразовые сервисы `migrate` и `s3-init`. Повторный запуск безопасен. Зафиксированы PostgreSQL 16.6, Valkey 8.0.2, HAProxy 3.0.5, Python 3.11.11, MinIO/mc по digest; Python-библиотеки — в [requirements.txt](requirements.txt).
|
||||
|
||||
Безопасная остановка сохраняет volumes: `make down`. Намеренное удаление **только учебных данных пятой работы**: `./scripts/compose.sh down -v`. После удаления volumes старые пользователи, сессии, очередь и файлы восстановить нельзя. Проекты лабораторных 2–4 имеют другие Compose names, порты и volumes.
|
||||
|
||||
## Архитектура и переход с lab4
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Клиент] --> LB[HAProxy]
|
||||
LB --> A[API ×2]
|
||||
A --> DB[(PostgreSQL: metadata, jobs, outbox)]
|
||||
A --> S[(MinIO: staging)]
|
||||
DB --> P[Publisher]
|
||||
P --> B[(Valkey Streams + AOF)]
|
||||
B --> W[Worker]
|
||||
W --> D[Pillow subprocess]
|
||||
W --> S3[(MinIO: results/id/version)]
|
||||
W --> DB
|
||||
A --> K[(Valkey: кэш сессий)]
|
||||
```
|
||||
|
||||
Из предыдущих работ сохранены Argon2id-пароли, устойчивые сессии и отзыв, авторизация участников чата, Unicode, keyset-пагинация, атомарная идемпотентность сообщений на 24 часа, две API-реплики и балансировка, проверка файлов, приватный S3 и авторизованная потоковая выдача оригинала. Источник истины для состояния — PostgreSQL; кэш не может подменить сессию/пользователя. Байты хранятся в MinIO.
|
||||
|
||||
Объявленный переход: upload `201 ready` заменён на `202 queued`; метаданные доступны до готовности, content отвечает 409, а queued/processing можно прикрепить к сообщению. Failed прикреплять нельзя. Новая миграция переводит существующие ready/pending-вложения lab4 в очередь, используя старый оригинал как источник; новая установка получает пустые таблицы. Автоматический reconciler из исследовательского трека lab4 заменён подходящей для асинхронной работы очисткой staging; прежний алгоритм нельзя применять к текущим queued jobs.
|
||||
|
||||
### Границы устойчивости
|
||||
|
||||
1. API аутентифицирует клиента, проверяет чат/имя/MIME/bytes/пиксели/целостность. Приём идёт небольшими блоками в ограниченный временный spool; это ещё не durable staging.
|
||||
2. Под общей advisory-блокировкой приёма проверяется лимит активных jobs. Полный оригинал записывается в приватный `staging/<UUID>` MinIO на persistent volume.
|
||||
3. В одной транзакции PostgreSQL создаются attachment, job и outbox. После COMMIT возвращается **снимок queued**. Падение после S3 до COMMIT может оставить только непринятый orphan; он убирается после retention. Такой запрос не получит подтверждённого 202.
|
||||
4. Publisher читает outbox, выполняет XADD и только затем сохраняет published_at. Ответ XADD при `appendfsync=always` — граница подтверждения брокером. Если процесс упал после XADD до COMMIT outbox, публикация повторится. Если упал после COMMIT API до publish, сохранённый outbox будет найден при запуске publisher.
|
||||
5. Worker использует consumer group и `XREADGROUP COUNT 1`. Краткая транзакция фиксирует processing, attempt и lease. Следующая транзакция блокирует job и attachment, проверяет поколение/попытку и держит блокировки до записи всех S3-эффектов и COMMIT ready. Живой worker не теряет право на эффект только из-за истечения времени lease.
|
||||
6. После устойчивого результата worker выполняет XACK, затем XDEL обработанной доставки. При временной ошибке ack допустим только после сохранения нового намерения retry в БД. Необработанная доставка остаётся pending; `XAUTOCLAIM` позволяет её восстановить после остановки consumer.
|
||||
|
||||
Staging уже находится в том же MinIO, что и результаты: приём файла включает S3 PUT. Асинхронная граница отделяет resize/encode и публикацию конечного `results/` от HTTP, а не устраняет передачу байтов клиентом. Readiness требует PostgreSQL и S3; брокер, worker и необязательный кэш не включены, поскольку 202 можно безопасно принять через staging/outbox и при их остановке.
|
||||
|
||||
### Варианты, состояния и лимиты
|
||||
|
||||
| Вариант | Правила |
|
||||
| --- | --- |
|
||||
| original | Побайтно исходный PNG/JPEG; SHA-256 и размер из upload |
|
||||
| thumbnail | JPEG, вписан в 256×256, без увеличения, размеры округляются вниз, минимум 1 px |
|
||||
| crop | JPEG, центральный квадрат 128×128, маленькие допускается увеличивать |
|
||||
| optimized | WebP, вписан в 1280×1280, без увеличения |
|
||||
|
||||
EXIF orientation применяется до геометрии; прозрачность компонуется на белый фон. Метаданные EXIF не переносятся в варианты; оригинал сохраняет исходные байты. Для v1 JPEG quality=85, WebP quality=80; для v2 и следующих JPEG=90, WebP=85. Optimized не обязан быть меньше каждого небольшого оригинала.
|
||||
|
||||
В API доступны queued → processing → ready/failed и processing → queued при временной ошибке. Error_code безопасный, без внутренних адресов и traceback. Готовность фиксируется после оригинала и всех трёх вариантов. Content проверяет авторизацию и статус для каждого запроса; redirect или публичная ссылка S3 не используются.
|
||||
|
||||
HTTP ограничен 10 MiB, 40 млн пикселей, 30 секундами передачи и 5 секундами простоя; одновременно принимаются два файла на API. Обработка выполняется в отдельном subprocess: 20 секунд на decode/encode, RLIMIT_AS 640 MiB, весь контейнер worker — 768 MiB. Процесс обработки получает минимальное окружение без credentials. Чрезмерно затратное, но формально допустимое по пикселям изображение может получить конечный resource_limit; 40 млн пикселей — защитный максимум, не обещание обработки при любом содержимом и выделенной памяти.
|
||||
|
||||
Обычный worker последовательный, prefetch/count=1; число отдельных workers можно увеличить через `--scale worker=N`. Максимум 100 queued/processing jobs контролируется транзакционно на всех API. При заполнении — 429 до записи нового staging. Это ограничение текущей работы, не общий лимит хранения всей истории файлов.
|
||||
|
||||
### Retry, failed, replay и cleanup
|
||||
|
||||
Временные S3-сбои и timeout повторяются до 5 управляемых попыток: задержка `uniform(0.5,1.5) × min(30, 2^(attempt−1))` секунд. Сам broker redelivery не создаёт новую логическую job. Неисправный/отсутствующий staging, invalid_image или resource_limit дают failed; исчерпание временных попыток — retry_exhausted. Registry failed — таблица jobs, а не отдельная очередь с потерянным контекстом. Операторский replay сбрасывает бюджет попыток текущей failed-job, пишет audit и не допускается для ready или отсутствующего staging.
|
||||
|
||||
```sh
|
||||
./scripts/compose.sh run --rm --no-deps ops python manage.py failed
|
||||
./scripts/compose.sh run --rm --no-deps ops python manage.py replay JOB_UUID
|
||||
./scripts/compose.sh run --rm --no-deps ops python manage.py upgrade ATTACHMENT_UUID
|
||||
# Очистка: безопасный dry-run по умолчанию:
|
||||
./scripts/compose.sh run --rm --no-deps ops python janitor.py --once
|
||||
./scripts/compose.sh run --rm --no-deps ops python janitor.py --once --apply
|
||||
```
|
||||
|
||||
Lease по умолчанию 30 секунд и не меньше timeout+5. Зависшая processing после смерти worker захватывается заново; старый процесс дополнительно ограничен блокировками и версией. Повторы получают те же ключи `(attachment_id, pipeline_version, variant)`. Старый job не может откатить ready или опубликовать результат поверх новой версии. PUT после аварии может повториться — это идемпотентность эффекта, а не exactly-once доставка.
|
||||
|
||||
Janitor запускается раз в минуту и удаляет staging старше суток только после проверки состояния под блокировками. Он сохраняет queued/processing независимо от возраста, удаляет orphan после неудавшегося upload и удерживает failed-источники сутки для replay. Dry-run ничего не меняет, повторный apply идемпотентен. Готовые объекты и старые опубликованные версии автоматически не удаляются. Источник новой версии может ссылаться на готовый original предыдущей версии; такой объект не попадает в staging cleanup.
|
||||
|
||||
### Где настраивается
|
||||
|
||||
Реальные локальные параметры показаны в [infra/local.env.example](infra/local.env.example); `bootstrap.sh` генерирует секреты отдельно. В исходном `.env.example` заданы логические интерфейсы задания, а не готовая конфигурация этого стенда.
|
||||
|
||||
| Переменная/настройка | Реальная реализация |
|
||||
| --- | --- |
|
||||
| APP_PORT, API_REPLICAS | `.env` → Compose: порт 8085, две API |
|
||||
| DATABASE_URL_FILE, CACHE_URL_FILE | Используются прямые DATABASE_URL/CACHE_URL в окружении контейнеров; значения собираются Compose из локальных секретов |
|
||||
| S3_*_FILE | Прямые S3_ACCESS_KEY_ID/S3_SECRET_ACCESS_KEY в контейнерах; только s3-init получает административные credentials |
|
||||
| BROKER_URL_FILE | Прямой BROKER_URL=`redis://broker:6379/0`, внутренняя сеть, порт наружу не опубликован |
|
||||
| WORKER_CONCURRENCY | Внутри worker фиксировано 1, горизонтальное число задаётся `--scale worker=N` |
|
||||
| JOB_MAX_ATTEMPTS | `.env`, по умолчанию 5 |
|
||||
| JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS | `.env`, 20 и 30 секунд; минимальный lease=timeout+5 |
|
||||
| MAX_PENDING_JOBS | `.env`, 100 на весь проект |
|
||||
| STAGING_RETENTION_SECONDS | `.env`, 86400, janitor не допускает меньше 60 секунд |
|
||||
| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | Фиксированные контрактом 10 MiB/40 млн в `attachments.py`, пиксельный лимит также в `transform.py` |
|
||||
| SESSION_TTL_SECONDS | 3600 в Compose, проверяется PostgreSQL, TTL не продлевается рестартом |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Структура и контракт | `make check`, `make validate PYTHON=.venv/bin/python` | PASS | [check](evidence/check.txt), [validate](evidence/validate.txt) |
|
||||
| Публичный минимум 1–5 | `make test` | PASS: 202, сообщение, ready, SHA оригинала, настоящий JPEG thumbnail | [smoke](evidence/smoke.txt) |
|
||||
| Сессии/кэш из lab2 | cache_consistency.py | Подделанный/устаревший кэш отвергнут, logout сохраняет отзыв, 8/8 идентификаций верны | [cache](evidence/cache-consistency.txt) |
|
||||
| Две API и message idempotency | idempotency_concurrent.py | 16 запросов через две реплики → одно сообщение | [idempotency](evidence/idempotency.json) |
|
||||
| Приватный S3, least privilege | private_storage.py | Анонимные list/get 403; admin/посторонний prefix запрещены; console недоступна | [private storage](evidence/private-storage.json) |
|
||||
| Ограничения и авторизация файлов | uploads_negative.py | 21 отрицательный сценарий, включая обрыв HTTP без сохранения частичного файла | [negative](evidence/uploads-negative.json) |
|
||||
| Worker не нужен для приёма | lifecycle.py | 202 queued, content 409, сообщение принято при остановленном worker | [lifecycle](evidence/lifecycle.json) |
|
||||
| Устойчивость API/outbox | lifecycle.py | SIGKILL обеих API после 202 до publish; прежнее вложение завершается | [lifecycle](evidence/lifecycle.json) |
|
||||
| Persistence брокера | lifecycle.py | Очередь сохранена после restart broker на прежнем volume | [lifecycle](evidence/lifecycle.json) |
|
||||
| Thumbnail/crop/WebP | image_variants.py | EXIF, белый alpha, no-upscale, crop по центру, 1280×640, размеры/hash реальны | [variants](evidence/image-variants.json) |
|
||||
| Разрыв COMMIT/publish и повтор publish | pipeline_faults.py, lifecycle.py | Outbox доставлен после восстановления; повтор не размножает эффект | [faults](evidence/pipeline-faults.json) |
|
||||
| Падение после S3 до ack | drive_job.py + pipeline_faults.py | Exit 86, незавершённая broker delivery восстановлена, четыре объекта | [faults](evidence/pipeline-faults.json) |
|
||||
| Lease после смерти worker | lifecycle.py | Реальное ожидание lease, та же job становится ready на второй попытке | [lifecycle](evidence/lifecycle.json) |
|
||||
| Backoff/jitter/failed/replay | pipeline_faults.py | Пять попыток, retry_exhausted, безопасный replay; ready replay отвергнут | [faults](evidence/pipeline-faults.json) |
|
||||
| Время/память subprocess | cleanup_and_limits.py | Реальный timeout на тестовом бюджете 50 мс и MemoryError при 32 MiB классифицированы | [limits](evidence/cleanup-and-limits.json) |
|
||||
| Staging retention/cleanup | cleanup_and_limits.py | Dry-run/apply/repeat; активный staging сохранён, orphan удалён | [cleanup](evidence/cleanup-and-limits.json) |
|
||||
| Backpressure и отказ брокера | backpressure.py | 100 приняты без broker, следующий 429, сообщения доступны, все 100 ready после восстановления | [backpressure](evidence/backpressure.json) |
|
||||
| Перезапуск всех сервисов | `make test-persistence` + snapshots | Сессия/данные/файл сохранены; StartedAt изменён у всех 10 работающих контейнеров, volumes прежние | [persistence](evidence/persistence.txt), [проверка hook](evidence/persistence-verified.json) |
|
||||
| Отказ одной API | `make test-failover` | Запись восстановлена в допустимый бюджет, подтверждённые сообщения/файл сохранены | [failover](evidence/failover.txt) |
|
||||
| Уровень 5, трек 2 | `make research` | Три повтора crash/duplicates/reorder, новая версия защищена, инварианты выполнены | [RESEARCH.md](RESEARCH.md) |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Реальный вывод основных команд:
|
||||
|
||||
## Эксперименты и отказы
|
||||
```text
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
`make check` не запускает приложение; сквозные и инфраструктурные доказательства приведены отдельно. Снимки контейнеров намеренно не содержат окружение и credentials.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
## Исследование уровня 5
|
||||
|
||||
Гипотеза: outbox, version fencing и устойчивые ключи S3 сохраняют логический результат при дублировании/изменении порядка/аварии worker. Методика, аппаратные ограничения, размеры файлов и границы выводов изложены в [RESEARCH.md](RESEARCH.md). Сценарии включают реальный consumer-group redelivery и отдельные процессы, завершённые `os._exit(86)`.
|
||||
|
||||
<!-- BEGIN RESEARCH RESULTS -->
|
||||
|
||||
| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Падение до PUT | 3 | 1550.718 | 1321.347–1786.888 | 232.845 | 4 | 0 |
|
||||
| Падение после PUT | 3 | 1549.527 | 1334.593–1872.025 | 284.404 | 4 | 0 |
|
||||
| Падение после COMMIT до ack | 3 | 1383.915 | 1275.246–1590.414 | 178.915 | 4 | 0 |
|
||||
| 8 конкурентных дублей | 3 | 143.795 | 121.12–172.849 | 26.448 | 4 | 0 |
|
||||
| Старая версия после новой | 3 | 223.21 | 109.191–440.476 | 188.235 | 4 | 0 |
|
||||
|
||||

|
||||
|
||||
<!-- END RESEARCH RESULTS -->
|
||||
|
||||
Время таблицы — длительность тестового сценария с запуском процессов; в матрице lease ускорен SQL-командой. Это не время ожидания реального lease и не production latency. Отдельный lifecycle использует реальные часы. Дополнительные результаты автоматически сведены ниже.
|
||||
|
||||
<!-- BEGIN OPERATIONAL RESULTS -->
|
||||
|
||||
### Восстановление и ограничения
|
||||
|
||||
| Воздействие | Реальное наблюдение | Результат |
|
||||
| --- | --- | --- |
|
||||
| Остановка worker, SIGKILL API до publish, restart broker | Длина stream 1 → 1; ready через 3.023 с после запуска worker | PASS, исходник сохранён |
|
||||
| Смерть worker после PUT, настоящий lease | Ready через 28.923 с; попыток 2 | PASS, без ускорения часов |
|
||||
| Остановка broker и заполнение очереди | Принято 100; следующий HTTP 429; восстановлено 100 за 24.64 с | PASS, потеряно 0 |
|
||||
| Перезапуск всего стенда | 10 реальных процессов, прежние volumes и сессия | PASS |
|
||||
| Отказ API | 114 успешных записей, 2 переходных ошибок; первая успешная запись через 2.171 с | PASS |
|
||||
| Тайм-аут decode, тестовый бюджет 50 мс | Время сценария 68.024 мс; retry, затем ready | PASS |
|
||||
| Ограничение адресного пространства, тестовый бюджет 32 MiB | Конечный resource_limit, replay завершился | PASS |
|
||||
| Staging cleanup, часы +2 дня | Кандидатов 23, повторных 0; активный исходник сохранён | PASS |
|
||||
|
||||
### Проверка геометрии реальных файлов
|
||||
|
||||
| Исходник/сценарий | Thumbnail | Crop | Optimized |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| small_alpha | 80×40 | 128×128 | 80×40 |
|
||||
| exif_orientation | 64×256 | 128×128 | 100×400 |
|
||||
| center_crop | 256×64 | 128×128 | 400×100 |
|
||||
| large_fit | 256×128 | 128×128 | 1280×640 |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
### Наблюдаемые retry с jitter
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| После попытки | Сохранённая задержка до следующей, с |
|
||||
| --- | ---: |
|
||||
| 1 | 1.080882 |
|
||||
| 2 | 2.447441 |
|
||||
| 3 | 2.182634 |
|
||||
| 4 | 8.307950 |
|
||||
|
||||
После 5-й временной ошибки зафиксирован `retry_exhausted`; явный replay завершается ready. При проверке расписания next_at искусственно сдвигается, так что таблица показывает записанные интервалы, а не фактически выжданные паузы.
|
||||
|
||||
<!-- END OPERATIONAL RESULTS -->
|
||||
|
||||
Raw JSON/CSV и PNG/SVG хранятся в `evidence/`. График и таблица исследования пересобираются `scripts/summarize_research.py`, операционные таблицы — `scripts/summarize_operations.py`; ручные PASS не заменяют assert сценариев.
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
# Исследование: уровень 5, трек 2
|
||||
|
||||
## Гипотеза и инварианты
|
||||
|
||||
Версионирование результатов, transactional outbox, блокировки PostgreSQL и повторяемые ключи S3 позволяют завершать принятые задачи при at-least-once доставке без дополнительных наборов вариантов и перезаписи нового результата старым.
|
||||
|
||||
Инварианты проверяются assertions: каждый проверяемый upload с 202 завершается ready; оригинал побайтно совпадает с принятым; у текущей версии ровно четыре объекта; метаданные и SHA-256 каждого варианта соответствуют скачанным байтам; повтор ready не меняет метаданные; устаревшая job не изменяет новую версию.
|
||||
|
||||
## Методика
|
||||
|
||||
`make research` останавливает обычные worker/publisher и запускает `tests/pipeline_faults.py` в ops. Создаются реальные HTTP-вложения и настоящие consumer-group доставки Valkey Streams. В отдельном процессе вызывается `os._exit(86)` до PUT, после всех PUT до COMMIT, после COMMIT до XACK. Неподтверждённая доставка затем захватывается другим consumer. Матрица содержит три повтора каждого из пяти основных сценариев, дополнительно проверяет смену уже опубликованной v1 на v2, failed/replay, пять временных ошибок и повтор публикации outbox.
|
||||
|
||||
Для ускорения матрицы lease переводится в прошлое SQL-командой; до этого отдельно проверяется отказ захвата действующего lease. Это не измерение реального времени ожидания lease. `tests/lifecycle.py` отдельно проверяет восстановление с настоящими часами, без изменения дедлайна. Конкурентные дубли запускаются восемью потоками, обычный worker последовательный. Время в таблице включает HTTP/БД, запуск Python-процессов и обработку; это время полного тестового сценария, а не latency одного изображения в production.
|
||||
|
||||
Один локальный Linux Docker Engine в Colima, arm64, 4 vCPU, около 8 GiB RAM. Лимиты: worker 1 CPU/768 MiB, API 2 реплики по 2 CPU/512 MiB, PostgreSQL 1 CPU/512 MiB, MinIO 0.5 CPU/768 MiB, broker 0.25 CPU/192 MiB; ops 1 CPU/512 MiB. Ограничения CPU — верхние пределы, а не выделенные ядра. Прочие лабораторные могут работать в этой же VM. Файл матрицы — PNG 320×200 из публичных fixtures. Отдельные геометрические тесты используют PNG/JPEG от 80×40 до 1600×800. Нагрузочного прогрева нет: исследуется корректность конкретных interleavings, не максимальный throughput. Точные лимиты и образы сохранены в `evidence/containers.json` и `compose.yaml`.
|
||||
|
||||
## Результаты матрицы
|
||||
|
||||
| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Падение до PUT | 3 | 1550.718 | 1321.347–1786.888 | 232.845 | 4 | 0 |
|
||||
| Падение после PUT | 3 | 1549.527 | 1334.593–1872.025 | 284.404 | 4 | 0 |
|
||||
| Падение после COMMIT до ack | 3 | 1383.915 | 1275.246–1590.414 | 178.915 | 4 | 0 |
|
||||
| 8 конкурентных дублей | 3 | 143.795 | 121.12–172.849 | 26.448 | 4 | 0 |
|
||||
| Старая версия после новой | 3 | 223.21 | 109.191–440.476 | 188.235 | 4 | 0 |
|
||||
|
||||

|
||||
|
||||
После падения до COMMIT требуются две управляемые попытки. После COMMIT повтор не выполняет обработку: достаточно одной попытки. При восьми конкурентных доставках обрабатывает один поток, остальные видят занятость или завершённый результат. После новой версии старые задания не создают объекты v1 и не изменяют публичные метаданные v2. Отдельно проверено обновление уже существующего результата v1 на v2 с изменённым качеством кодирования.
|
||||
|
||||
Raw: [pipeline-faults.json](evidence/pipeline-faults.json), [CSV](evidence/fault-summary.csv), [lifecycle.json](evidence/lifecycle.json), [backpressure.json](evidence/backpressure.json), [cleanup-and-limits.json](evidence/cleanup-and-limits.json).
|
||||
|
||||
## Ограничения
|
||||
|
||||
Это проверка конечного набора аварийных расписаний, а не математическое доказательство для всех interleavings. Исследование не покрывает потерю диска/volume, несколько узлов, длительные сетевые разделения и повреждение файловой системы. Локальная матрица не подтверждает SLA и не измеряет p95 под производственной нагрузкой; для выбранного трека это не заявляется. CPU/RAM ограничены конфигурацией, пиковый RSS декодера в этом исследовании не измеряется.
|
||||
|
||||
Физический PUT может повториться после аварии: одинаковый ключ заменяется теми же данными. Гарантируется один текущий логический набор результатов, а доставка остаётся at least once. Объекты прежних успешно опубликованных версий сохраняются: это история версий, не дубли текущего результата. Изменение алгоритма должно сопровождаться увеличением pipeline_version; старый бинарный worker нужно вывести из эксплуатации перед изменением кода существующей версии.
|
||||
@@ -0,0 +1,896 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import ConnectionPool
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from fastapi import FastAPI, Header, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel, ConfigDict, StringConstraints, ValidationError
|
||||
from typing_extensions import Annotated
|
||||
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname())
|
||||
|
||||
USERNAME_RE = r"^[a-z0-9_]{3,32}$"
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str, *, headers: Optional[Dict[str, str]] = None):
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionOut(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserOut
|
||||
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
title: str
|
||||
member_ids: List[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: List[str]
|
||||
|
||||
|
||||
class CreateUserIn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: Annotated[str, StringConstraints(pattern=USERNAME_RE, min_length=3, max_length=32)]
|
||||
display_name: Annotated[str, StringConstraints(min_length=1, max_length=100)]
|
||||
password: Annotated[str, StringConstraints(min_length=12, max_length=128)]
|
||||
|
||||
|
||||
class CreateChatIn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: Annotated[str, StringConstraints(min_length=1, max_length=100)]
|
||||
member_ids: List[str]
|
||||
|
||||
|
||||
class CreateMessageIn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
attachment_ids: List[str] = []
|
||||
text: Annotated[str, StringConstraints(max_length=4000)]
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self) -> None:
|
||||
self.connection_url = self._resolve_database_url()
|
||||
self.cache_url = self._resolve_cache_url()
|
||||
self.shared_cache = redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
|
||||
self.session_store = os.getenv("SESSION_STORE", "valkey")
|
||||
self.password_hasher = PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16)
|
||||
self.pool = ConnectionPool(
|
||||
self.connection_url, min_size=1,
|
||||
max_size=int(os.getenv("DB_POOL_MAX_SIZE", "10")),
|
||||
timeout=5, max_waiting=20, open=False,
|
||||
kwargs={"connect_timeout": 5, "autocommit": False,
|
||||
"options": "-c statement_timeout=5000 -c lock_timeout=500"},
|
||||
)
|
||||
|
||||
def _resolve_database_url(self) -> str:
|
||||
value = os.getenv("DATABASE_URL")
|
||||
if value:
|
||||
return value
|
||||
secret_path = os.getenv("DATABASE_URL_FILE")
|
||||
if secret_path and os.path.exists(secret_path):
|
||||
with open(secret_path, "r", encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.getenv("DEFAULT_DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
|
||||
|
||||
def _resolve_cache_url(self) -> str:
|
||||
value = os.getenv("CACHE_URL")
|
||||
if value:
|
||||
return value
|
||||
secret_path = os.getenv("CACHE_URL_FILE")
|
||||
if secret_path and os.path.exists(secret_path):
|
||||
with open(secret_path, "r", encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.getenv("DEFAULT_CACHE_URL", "redis://localhost:6379/0")
|
||||
|
||||
def connect(self):
|
||||
return self.pool.connection()
|
||||
|
||||
def cache_client(self):
|
||||
if os.getenv("CACHE_CLIENT_MODE", "shared") == "shared":
|
||||
return self.shared_cache
|
||||
return redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
|
||||
|
||||
def cache_session(self, session: Dict[str, Any]) -> None:
|
||||
if self.session_store != "valkey":
|
||||
return
|
||||
try:
|
||||
client = self.cache_client()
|
||||
ttl_seconds = max(1, int((self.parse_session_expiry(session["expires_at"]) - self.utc_now()).total_seconds()))
|
||||
payload = json.dumps({
|
||||
"token": session["token"],
|
||||
"user_id": session["user_id"],
|
||||
"expires_at": session["expires_at"],
|
||||
"revoked_at": session.get("revoked_at"),
|
||||
"user": session["user"],
|
||||
}, separators=(",", ":"), ensure_ascii=False)
|
||||
client.setex(f"session:{session['token']}", ttl_seconds, payload)
|
||||
except redis.RedisError:
|
||||
return
|
||||
|
||||
def get_cached_session(self, token: str, *, db_session) -> Optional[Dict[str, Any]]:
|
||||
if self.session_store != "valkey":
|
||||
return None
|
||||
try:
|
||||
client = self.cache_client()
|
||||
payload = client.get(f"session:{token}")
|
||||
if not payload:
|
||||
return None
|
||||
data = json.loads(payload)
|
||||
session = {
|
||||
"token": data["token"],
|
||||
"user_id": data["user_id"],
|
||||
"expires_at": data["expires_at"],
|
||||
"revoked_at": data.get("revoked_at"),
|
||||
"user": data["user"],
|
||||
}
|
||||
if not self.is_session_valid(session):
|
||||
client.delete(f"session:{token}")
|
||||
return None
|
||||
if session != db_session:
|
||||
client.delete(f"session:{token}")
|
||||
return None
|
||||
return db_session
|
||||
except (redis.RedisError, KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def delete_cached_session(self, token: str) -> None:
|
||||
if self.session_store != "valkey":
|
||||
return
|
||||
try:
|
||||
self.cache_client().delete(f"session:{token}")
|
||||
except redis.RedisError:
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def parse_session_expiry(value: Any) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(timezone.utc)
|
||||
if isinstance(value, str):
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
raise ValueError("invalid session expiry")
|
||||
|
||||
def init(self) -> None:
|
||||
migration_dir = Path(__file__).resolve().parent / "migrations"
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(202603)")
|
||||
cur.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
applied = set()
|
||||
cur.execute("SELECT version FROM schema_migrations")
|
||||
for (version,) in cur.fetchall():
|
||||
applied.add(version)
|
||||
|
||||
for migration_path in sorted(migration_dir.glob("*.sql")):
|
||||
version = migration_path.stem
|
||||
if version in applied:
|
||||
continue
|
||||
sql = migration_path.read_text(encoding="utf-8")
|
||||
cur.execute(sql)
|
||||
cur.execute(
|
||||
"INSERT INTO schema_migrations(version, applied_at) VALUES (%s, %s)",
|
||||
(version, self.utc_now()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def utc_now(self) -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
def isoformat_utc(self, value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
def hash_password(self, password: str) -> str:
|
||||
return self.password_hasher.hash(password)
|
||||
|
||||
def verify_password(self, password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
self.password_hasher.verify(password_hash, password)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def require_user_by_username(self, username: str, conn):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", (username,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": str(row[0]),
|
||||
"username": row[1],
|
||||
"display_name": row[2],
|
||||
"created_at": self.isoformat_utc(row[3]),
|
||||
"password_hash": row[4],
|
||||
}
|
||||
|
||||
def fetch_user(self, user_id: str, conn):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": str(row[0]),
|
||||
"username": row[1],
|
||||
"display_name": row[2],
|
||||
"created_at": self.isoformat_utc(row[3]),
|
||||
}
|
||||
|
||||
def fetch_session_by_token(self, token: str, conn):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT s.token, s.user_id, s.expires_at, s.revoked_at, u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token = %s",
|
||||
(token,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"token": row[0],
|
||||
"user_id": str(row[1]),
|
||||
"expires_at": self.isoformat_utc(row[2]),
|
||||
"revoked_at": row[3],
|
||||
"user": {
|
||||
"id": str(row[4]),
|
||||
"username": row[5],
|
||||
"display_name": row[6],
|
||||
"created_at": self.isoformat_utc(row[7]),
|
||||
},
|
||||
}
|
||||
|
||||
def list_member_ids(self, chat_id: str, conn) -> List[str]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat_id,))
|
||||
rows = cur.fetchall()
|
||||
return [str(row[0]) for row in rows]
|
||||
|
||||
def fetch_chat(self, chat_id: str, conn):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, title, created_at FROM chats WHERE id = %s", (chat_id,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": str(row[0]),
|
||||
"title": row[1],
|
||||
"created_at": self.isoformat_utc(row[2]),
|
||||
"member_ids": self.list_member_ids(str(row[0]), conn),
|
||||
}
|
||||
|
||||
def fetch_message(self, message_id: str, conn):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", (message_id,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": str(row[0]),
|
||||
"chat_id": str(row[1]),
|
||||
"sender_id": str(row[2]),
|
||||
"text": row[3],
|
||||
"created_at": self.isoformat_utc(row[4]),
|
||||
"attachment_ids": self.message_attachment_ids(str(row[0]), conn),
|
||||
}
|
||||
|
||||
def fetch_chat_membership(self, chat_id: str, user_id: str, conn) -> bool:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def insert_user(self, username: str, display_name: str, password: str):
|
||||
now = self.utc_now()
|
||||
user_id = uuid.uuid4()
|
||||
password_hash = self.hash_password(password)
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (%s,%s,%s,%s,%s)",
|
||||
(user_id, username, display_name, password_hash, now),
|
||||
)
|
||||
conn.commit()
|
||||
return {
|
||||
"id": str(user_id),
|
||||
"username": username,
|
||||
"display_name": display_name,
|
||||
"created_at": self.isoformat_utc(now),
|
||||
}
|
||||
|
||||
def insert_chat(self, title: str, member_ids: List[str], actor_id: str):
|
||||
now = self.utc_now()
|
||||
chat_id = uuid.uuid4()
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("INSERT INTO chats(id,title,created_at) VALUES (%s,%s,%s)", (chat_id, title, now))
|
||||
for member_id in sorted(set(member_ids)):
|
||||
cur.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (chat_id, member_id))
|
||||
conn.commit()
|
||||
return {
|
||||
"id": str(chat_id),
|
||||
"title": title,
|
||||
"member_ids": sorted(set(member_ids)),
|
||||
"created_at": self.isoformat_utc(now),
|
||||
}
|
||||
|
||||
def insert_message(self, chat_id: str, sender_id: str, text: str, key: str, attachment_ids: List[str]):
|
||||
# Transaction-level lock serializes the same scope across all replicas.
|
||||
scope = json.dumps([sender_id, "POST", chat_id, key])
|
||||
request_body = {"text": text, "attachment_ids": attachment_ids}
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,))
|
||||
cur.execute("DELETE FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s AND expires_at <= NOW()", (sender_id, chat_id, key))
|
||||
cur.execute("SELECT body, response FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s", (sender_id, chat_id, key))
|
||||
previous = cur.fetchone()
|
||||
if previous:
|
||||
if previous[0] != request_body:
|
||||
raise ApiError(409, "conflict", "idempotency key reused with different body")
|
||||
return previous[1]
|
||||
for aid in attachment_ids:
|
||||
cur.execute("SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE", (aid, chat_id))
|
||||
att = cur.fetchone()
|
||||
if att is None:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
if att[0] == "failed":
|
||||
raise ApiError(409, "conflict", "attachment unavailable")
|
||||
now = self.utc_now()
|
||||
message_id = str(uuid.uuid4())
|
||||
message = {"id": message_id, "chat_id": chat_id, "sender_id": sender_id,
|
||||
"text": text, "created_at": self.isoformat_utc(now), "attachment_ids": attachment_ids}
|
||||
cur.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES (%s,%s,%s,%s,%s)",
|
||||
(message_id, chat_id, sender_id, text, now))
|
||||
for position, aid in enumerate(attachment_ids):
|
||||
cur.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, aid, position))
|
||||
cur.execute("INSERT INTO message_requests(sender_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s::jsonb,%s::jsonb,%s)",
|
||||
(sender_id, chat_id, key, json.dumps(request_body), json.dumps(message), now + timedelta(hours=24)))
|
||||
conn.commit()
|
||||
return message
|
||||
|
||||
def message_attachment_ids(self, message_id, conn):
|
||||
return [str(row[0]) for row in conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (message_id,)).fetchall()]
|
||||
|
||||
@staticmethod
|
||||
def cursor_uuid_key(value: str) -> int:
|
||||
try:
|
||||
return uuid.UUID(value).int
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
|
||||
def list_chats_for_user(self, user_id: str, limit: int, cursor: Optional[str]):
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
query = """
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE cm.user_id = %s
|
||||
"""
|
||||
params: List[Any] = [user_id]
|
||||
if cursor:
|
||||
prefix = f"user:{user_id}|"
|
||||
if not cursor.startswith(prefix):
|
||||
raise ApiError(400, "invalid_request", "cursor does not match user chat list")
|
||||
remainder = cursor[len(prefix):]
|
||||
if not remainder:
|
||||
raise ApiError(400, "invalid_request", "invalid cursor")
|
||||
try:
|
||||
cursor_ts_raw, cursor_id = remainder.split("|", 1)
|
||||
cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid cursor") from exc
|
||||
query += " AND (c.created_at > %s OR (c.created_at = %s AND c.id > %s::uuid))"
|
||||
params.extend([cursor_dt, cursor_dt, cursor_id])
|
||||
query += " ORDER BY c.created_at ASC, c.id ASC LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
cur.execute(query, tuple(params))
|
||||
rows = cur.fetchall()
|
||||
items = [
|
||||
{
|
||||
"id": str(row[0]),
|
||||
"title": row[1],
|
||||
"created_at": self.isoformat_utc(row[2]),
|
||||
"member_ids": self.list_member_ids(str(row[0]), conn),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
next_cursor = None
|
||||
if len(items) > limit:
|
||||
last_item = items[limit - 1]
|
||||
next_cursor = f"user:{user_id}|{last_item['created_at']}|{last_item['id']}"
|
||||
items = items[:limit]
|
||||
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
def list_messages_for_chat(self, chat_id: str, limit: int, cursor: Optional[str]):
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s"
|
||||
params: List[Any] = [chat_id]
|
||||
if cursor:
|
||||
prefix = f"chat:{chat_id}|"
|
||||
if not cursor.startswith(prefix):
|
||||
raise ApiError(400, "invalid_request", "cursor does not match chat")
|
||||
remainder = cursor[len(prefix):]
|
||||
if not remainder:
|
||||
raise ApiError(400, "invalid_request", "invalid cursor")
|
||||
try:
|
||||
cursor_ts_raw, cursor_id = remainder.split("|", 1)
|
||||
cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid cursor") from exc
|
||||
query += " AND (created_at, id) > (%s, %s::uuid)"
|
||||
params.extend([cursor_dt, cursor_id])
|
||||
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
cur.execute(query, tuple(params))
|
||||
rows = cur.fetchall()
|
||||
links = {}
|
||||
if rows:
|
||||
cur.execute("SELECT message_id, attachment_id FROM message_attachments WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position", ([str(row[0]) for row in rows],))
|
||||
for mid, aid in cur.fetchall():
|
||||
links.setdefault(str(mid), []).append(str(aid))
|
||||
|
||||
items = [{
|
||||
"id": str(row[0]),
|
||||
"chat_id": str(row[1]),
|
||||
"sender_id": str(row[2]),
|
||||
"text": row[3],
|
||||
"created_at": self.isoformat_utc(row[4]),
|
||||
"attachment_ids": links.get(str(row[0]), []),
|
||||
} for row in rows]
|
||||
|
||||
next_cursor = None
|
||||
if len(items) > limit:
|
||||
last_item = items[limit - 1]
|
||||
next_cursor = f"chat:{chat_id}|{last_item['created_at']}|{last_item['id']}"
|
||||
items = items[:limit]
|
||||
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
def create_session(self, user_id: str, username: str):
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = self.utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO sessions(id, token, user_id, expires_at, created_at) VALUES (%s,%s,%s,%s,%s)",
|
||||
(uuid.uuid4(), token, user_id, expires_at, self.utc_now()),
|
||||
)
|
||||
conn.commit()
|
||||
with self.connect() as conn:
|
||||
user = self.fetch_user(user_id, conn)
|
||||
if user is None:
|
||||
raise ApiError(500, "internal_error", "session user missing")
|
||||
session = {
|
||||
"token": token,
|
||||
"user_id": user_id,
|
||||
"expires_at": self.isoformat_utc(expires_at),
|
||||
"revoked_at": None,
|
||||
"user": user,
|
||||
}
|
||||
self.cache_session(session)
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": self.isoformat_utc(expires_at),
|
||||
"user": user,
|
||||
}
|
||||
|
||||
def get_session_by_token(self, token: str, conn=None):
|
||||
if conn is None:
|
||||
with self.connect() as db_conn:
|
||||
return self.get_session_by_token(token, db_conn)
|
||||
db_session = self.fetch_session_by_token(token, conn)
|
||||
if db_session is None or not self.is_session_valid(db_session):
|
||||
self.delete_cached_session(token)
|
||||
return None
|
||||
cached = self.get_cached_session(token, db_session=db_session)
|
||||
if cached is None:
|
||||
self.cache_session(db_session)
|
||||
return db_session
|
||||
|
||||
def revoke_current_session(self, token: str):
|
||||
with self.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE sessions SET revoked_at = %s WHERE token = %s AND revoked_at IS NULL", (self.utc_now(), token))
|
||||
conn.commit()
|
||||
self.delete_cached_session(token)
|
||||
|
||||
def is_session_valid(self, session) -> bool:
|
||||
if session is None:
|
||||
return False
|
||||
if session.get("revoked_at") is not None:
|
||||
return False
|
||||
try:
|
||||
expiry = self.parse_session_expiry(session["expires_at"])
|
||||
except ValueError:
|
||||
return False
|
||||
if expiry <= self.utc_now():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
store = Store()
|
||||
from attachments import ObjectStore
|
||||
object_store = ObjectStore(store, ApiError)
|
||||
app = FastAPI(title="MTUSI Messenger")
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def startup_event() -> None:
|
||||
store.pool.open(wait=True, timeout=10)
|
||||
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
def shutdown_event() -> None:
|
||||
store.pool.close()
|
||||
store.shared_cache.close()
|
||||
object_store.close()
|
||||
|
||||
|
||||
@app.exception_handler(psycopg.OperationalError)
|
||||
async def database_error_handler(request: Request, exc):
|
||||
return await api_error_handler(request, ApiError(503, "unavailable", "database unavailable"))
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
request_id = uuid.uuid4().hex
|
||||
request.state.request_id = request_id
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
response.headers["X-App-Revision"] = os.getenv("APP_REVISION", "lab4-v1")
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
body = {"error": {"code": exc.code, "message": exc.message, "request_id": getattr(request.state, "request_id", "")}}
|
||||
return JSONResponse(status_code=exc.status_code, headers={**exc.headers, "X-Request-Id": getattr(request.state, "request_id", ""), "X-Instance-Id": INSTANCE_ID}, content=body)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
request_id = getattr(request.state, "request_id", "")
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||
content={"error": {"code": "invalid_request", "message": "invalid request", "request_id": request_id}},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def generic_error_handler(request: Request, exc: Exception):
|
||||
request_id = getattr(request.state, "request_id", "")
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||
content={"error": {"code": "internal_error", "message": "internal error", "request_id": request_id}},
|
||||
)
|
||||
|
||||
|
||||
def get_auth_header(request: Request):
|
||||
auth = request.headers.get("Authorization")
|
||||
if not auth:
|
||||
raise ApiError(401, "unauthorized", "missing authorization", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
|
||||
scheme, _, token = auth.partition(" ")
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise ApiError(401, "unauthorized", "invalid bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
|
||||
return token.strip()
|
||||
|
||||
|
||||
def require_bearer_session(request: Request):
|
||||
token = get_auth_header(request)
|
||||
with store.connect() as conn:
|
||||
session = store.get_session_by_token(token, conn)
|
||||
if session is None or not store.is_session_valid(session):
|
||||
raise ApiError(401, "unauthorized", "invalid or expired session", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
|
||||
return session
|
||||
|
||||
|
||||
async def parse_json_body(request: Request) -> Dict[str, Any]:
|
||||
try:
|
||||
raw = await request.body()
|
||||
if not raw:
|
||||
return {}
|
||||
return __import__("json").loads(raw)
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "malformed JSON") from exc
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
try:
|
||||
with store.connect() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
object_store.ready()
|
||||
return {"status": "ok"}
|
||||
except Exception as exc:
|
||||
raise ApiError(503, "unavailable", "required storage unavailable") from exc
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user_endpoint(request: Request):
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "malformed JSON") from exc
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "invalid_request", "invalid user payload")
|
||||
try:
|
||||
payload = CreateUserIn.model_validate(body)
|
||||
except ValidationError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid user payload") from exc
|
||||
if not payload.username or not payload.display_name.strip():
|
||||
raise ApiError(400, "invalid_request", "invalid request")
|
||||
payload.display_name = payload.display_name.strip()
|
||||
try:
|
||||
user = store.insert_user(payload.username, payload.display_name, payload.password)
|
||||
return UserOut.model_validate(user)
|
||||
except Exception as exc:
|
||||
if "duplicate key" in str(exc).lower() or "unique" in str(exc).lower():
|
||||
raise ApiError(409, "conflict", "username already exists") from exc
|
||||
raise ApiError(503, "unavailable", "required storage unavailable") from exc
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def current_user(request: Request):
|
||||
session = require_bearer_session(request)
|
||||
user = session["user"]
|
||||
return UserOut.model_validate(user)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def create_session(request: Request):
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if not auth.lower().startswith("basic "):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
encoded = auth.split(" ", 1)[1].strip()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded.encode("ascii"), validate=True)
|
||||
raw = decoded.decode("utf-8")
|
||||
except Exception:
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
if ":" not in raw:
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
username, password = raw.split(":", 1)
|
||||
if not re_match(username):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
if not (12 <= len(password) <= 128):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
try:
|
||||
with store.connect() as conn:
|
||||
user = store.require_user_by_username(username, conn)
|
||||
if user is None or not store.verify_password(user["password_hash"], password):
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
|
||||
session = store.create_session(user["id"], username)
|
||||
return SessionOut.model_validate(session)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ApiError(503, "unavailable", "required storage unavailable") from exc
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(request: Request):
|
||||
session = require_bearer_session(request)
|
||||
token = session["token"]
|
||||
store.revoke_current_session(token)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat_endpoint(request: Request):
|
||||
session = require_bearer_session(request)
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "malformed JSON") from exc
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "invalid_request", "invalid chat payload")
|
||||
try:
|
||||
payload = CreateChatIn.model_validate(body)
|
||||
except ValidationError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid chat payload") from exc
|
||||
if len(payload.member_ids) < 1 or len(payload.member_ids) > 100:
|
||||
raise ApiError(400, "invalid_request", "invalid member_ids")
|
||||
title = payload.title.strip()
|
||||
if not title:
|
||||
raise ApiError(400, "invalid_request", "invalid title")
|
||||
normalized = []
|
||||
seen = set()
|
||||
for member_id in payload.member_ids:
|
||||
try:
|
||||
member_id = str(uuid.UUID(member_id))
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid member_ids") from exc
|
||||
if member_id in seen:
|
||||
raise ApiError(400, "invalid_request", "duplicate member_ids")
|
||||
seen.add(member_id)
|
||||
normalized.append(member_id)
|
||||
if session["user_id"] not in normalized:
|
||||
raise ApiError(400, "invalid_request", "current user must be a member")
|
||||
try:
|
||||
with store.connect() as conn:
|
||||
for member_id in normalized:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM users WHERE id = %s", (member_id,))
|
||||
if cur.fetchone() is None:
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
chat = store.insert_chat(title, normalized, session["user_id"])
|
||||
return ChatOut.model_validate(chat)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ApiError(503, "unavailable", "required storage unavailable") from exc
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request):
|
||||
session = require_bearer_session(request)
|
||||
limit = request.query_params.get("limit", "50")
|
||||
cursor = request.query_params.get("cursor")
|
||||
try:
|
||||
limit_int = int(limit)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid limit") from exc
|
||||
if not (1 <= limit_int <= 100):
|
||||
raise ApiError(400, "invalid_request", "invalid limit")
|
||||
result = store.list_chats_for_user(session["user_id"], limit_int, cursor)
|
||||
return {"items": [ChatOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(request: Request, chat_id: str):
|
||||
session = require_bearer_session(request)
|
||||
try:
|
||||
uuid.UUID(chat_id)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
|
||||
with store.connect() as conn:
|
||||
chat = store.fetch_chat(chat_id, conn)
|
||||
if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return ChatOut.model_validate(chat)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(request: Request, chat_id: str):
|
||||
session = require_bearer_session(request)
|
||||
try:
|
||||
uuid.UUID(chat_id)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "malformed JSON") from exc
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "invalid_request", "invalid message payload")
|
||||
try:
|
||||
payload = CreateMessageIn.model_validate(body)
|
||||
except ValidationError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid message payload") from exc
|
||||
if len(payload.attachment_ids) > 10:
|
||||
raise ApiError(400, "invalid_request", "too many attachments")
|
||||
try:
|
||||
payload.attachment_ids = [str(uuid.UUID(aid)) for aid in payload.attachment_ids]
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid attachment id") from exc
|
||||
if len(set(payload.attachment_ids)) != len(payload.attachment_ids):
|
||||
raise ApiError(400, "invalid_request", "duplicate attachment ids")
|
||||
if not payload.text.strip() and not payload.attachment_ids:
|
||||
raise ApiError(400, "invalid_request", "invalid text")
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise ApiError(400, "invalid_request", "invalid Idempotency-Key")
|
||||
with store.connect() as conn:
|
||||
chat = store.fetch_chat(chat_id, conn)
|
||||
if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
message = store.insert_message(str(uuid.UUID(chat_id)), session["user_id"], payload.text, key, payload.attachment_ids)
|
||||
return MessageOut.model_validate(message)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(request: Request, chat_id: str):
|
||||
session = require_bearer_session(request)
|
||||
try:
|
||||
uuid.UUID(chat_id)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
|
||||
limit = request.query_params.get("limit", "50")
|
||||
cursor = request.query_params.get("cursor")
|
||||
try:
|
||||
limit_int = int(limit)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "invalid limit") from exc
|
||||
if not (1 <= limit_int <= 100):
|
||||
raise ApiError(400, "invalid_request", "invalid limit")
|
||||
with store.connect() as conn:
|
||||
if not store.fetch_chat_membership(chat_id, session["user_id"], conn):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
result = store.list_messages_for_chat(chat_id, limit_int, cursor)
|
||||
return {"items": [MessageOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]}
|
||||
|
||||
|
||||
def re_match(value: str) -> bool:
|
||||
import re
|
||||
|
||||
return bool(re.fullmatch(USERNAME_RE, value))
|
||||
|
||||
|
||||
from attachments import register_routes
|
||||
register_routes(app, object_store, require_bearer_session)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False)
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
"""Bounded uploads into durable S3 staging and transactional job/outbox."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid
|
||||
import warnings
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from fastapi import Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg.rows import dict_row
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
MAX_BYTES = 10485760
|
||||
MAX_PIXELS = 40000000
|
||||
CHUNK = 65536
|
||||
# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is
|
||||
# retained as another bound, but its warning threshold is not the contract cap.
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at,pipeline_version,staging_key,variants'
|
||||
|
||||
|
||||
def canonical_id(value, error):
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except (ValueError, TypeError, AttributeError) as exc:
|
||||
raise error(400, 'invalid_request', 'invalid id') from exc
|
||||
|
||||
|
||||
def object_lock(conn, key, *, wait=True):
|
||||
function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
||||
return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0]
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, store, error):
|
||||
self.store, self.error = store, error
|
||||
self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'),
|
||||
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'),
|
||||
config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3,
|
||||
retries={'total_max_attempts': 1, 'mode': 'standard'},
|
||||
max_pool_connections=8, s3={'addressing_style': 'path'}))
|
||||
self.decode_lock = threading.BoundedSemaphore(1)
|
||||
# At most two payload spools per process; queued requests receive ASGI
|
||||
# backpressure instead of being read into an unbounded request.body().
|
||||
self.upload_slots = asyncio.Semaphore(2)
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
||||
|
||||
def authorize_chat(self, request, chat_id, authenticate):
|
||||
actor = authenticate(request)
|
||||
chat_id = canonical_id(chat_id, self.error)
|
||||
with self.store.connect() as conn:
|
||||
if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn):
|
||||
raise self.error(404, 'not_found', 'chat not found')
|
||||
return actor, chat_id
|
||||
|
||||
def validate_filename(self, name):
|
||||
if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name):
|
||||
raise self.error(400, 'invalid_request', 'invalid filename')
|
||||
|
||||
def inspect_image(self, stream, claimed_type):
|
||||
try:
|
||||
with self.decode_lock, warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
stream.seek(0)
|
||||
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
|
||||
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise self.error(413, 'payload_too_large', 'too many image pixels')
|
||||
if actual != claimed_type:
|
||||
raise self.error(415, 'unsupported_media_type', 'content type mismatch')
|
||||
image.verify()
|
||||
stream.seek(0)
|
||||
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
|
||||
image.load() # JPEG.verify alone does not ensure a full decode.
|
||||
stream.seek(0)
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
raise self.error(413, 'payload_too_large', 'too many image pixels') from exc
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
|
||||
raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc
|
||||
|
||||
def fetch(self, aid, conn):
|
||||
with conn.cursor(row_factory=dict_row) as cur:
|
||||
cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,))
|
||||
return cur.fetchone()
|
||||
|
||||
def public(self, row):
|
||||
return {'id': str(row['id']), 'chat_id': str(row['chat_id']),
|
||||
'owner_id': str(row['owner_id']), 'filename': row['filename'],
|
||||
'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
|
||||
'sha256': row['sha256'], 'status': row['state'], 'variants': [{k:v for k,v in x.items() if k != 'object_key'} for x in row['variants']], 'error_code': row['error_code'],
|
||||
'created_at': self.store.isoformat_utc(row['created_at']),
|
||||
'updated_at': self.store.isoformat_utc(row['updated_at'])}
|
||||
|
||||
def authorized(self, request, aid, authenticate):
|
||||
actor = authenticate(request)
|
||||
aid = canonical_id(aid, self.error)
|
||||
with self.store.connect() as conn:
|
||||
row = self.fetch(aid, conn)
|
||||
if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn):
|
||||
raise self.error(404, 'not_found', 'attachment not found')
|
||||
return row
|
||||
|
||||
def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum):
|
||||
from pipeline import create_job
|
||||
aid = str(uuid.uuid4())
|
||||
key = 'staging/' + aid
|
||||
now = self.store.utc_now()
|
||||
# The capacity lock serializes admission across every API replica.
|
||||
# S3 is written before COMMIT: an ambiguous commit may leave an orphan,
|
||||
# but a returned 202 always has both staging bytes and a durable job.
|
||||
with self.store.connect() as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(202605)')
|
||||
count = conn.execute("SELECT count(*) FROM jobs WHERE state IN ('queued','processing')").fetchone()[0]
|
||||
if count >= int(os.getenv('MAX_PENDING_JOBS', '100')):
|
||||
raise self.error(429, 'rate_limited', 'processing capacity exceeded')
|
||||
try:
|
||||
stream.seek(0)
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=stream,
|
||||
ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum})
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
||||
conn.execute("""INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,
|
||||
content_type,size_bytes,sha256,state,created_at,updated_at,staging_key)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,%s,%s)""",
|
||||
(aid,chat_id,actor_id,'results/'+aid+'/v1/original',filename,content_type,size,checksum,now,now,key))
|
||||
create_job(conn, aid, 1)
|
||||
row = self.fetch(aid, conn)
|
||||
return self.public(row)
|
||||
|
||||
def download(self, row):
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
||||
body = response['Body']
|
||||
if response['ContentLength'] != row['size_bytes']:
|
||||
body.close()
|
||||
raise self.error(503, 'unavailable', 'object unavailable')
|
||||
def chunks():
|
||||
try:
|
||||
while True:
|
||||
part = body.read(CHUNK)
|
||||
if not part:
|
||||
break
|
||||
yield part
|
||||
finally:
|
||||
body.close()
|
||||
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
|
||||
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
|
||||
'Content-Disposition': 'attachment'})
|
||||
|
||||
|
||||
def register_routes(app, objects, authenticate):
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload(request: Request, chat_id: str):
|
||||
actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate)
|
||||
filename = request.query_params.get('filename', '')
|
||||
objects.validate_filename(filename)
|
||||
claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower()
|
||||
if claimed not in ('image/png', 'image/jpeg'):
|
||||
raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required')
|
||||
length = request.headers.get('content-length')
|
||||
if length:
|
||||
try:
|
||||
expected = int(length)
|
||||
except ValueError as exc:
|
||||
raise objects.error(400, 'invalid_request', 'invalid content length') from exc
|
||||
if expected > MAX_BYTES:
|
||||
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
|
||||
if expected < 1:
|
||||
raise objects.error(415, 'unsupported_media_type', 'empty image')
|
||||
try:
|
||||
await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc
|
||||
try:
|
||||
# Temporary payloads are never persistent API storage. Every path,
|
||||
# including disconnect/cancellation/error, closes and unlinks the spool.
|
||||
with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream:
|
||||
checksum, size = hashlib.sha256(), 0
|
||||
try:
|
||||
iterator = request.stream().__aiter__()
|
||||
async with asyncio.timeout(30):
|
||||
while True:
|
||||
try:
|
||||
chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5)
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
|
||||
checksum.update(chunk)
|
||||
await run_in_threadpool(stream.write, chunk)
|
||||
except (asyncio.TimeoutError, ClientDisconnect) as exc:
|
||||
raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc
|
||||
if size < 1:
|
||||
raise objects.error(415, 'unsupported_media_type', 'empty image')
|
||||
await run_in_threadpool(objects.inspect_image, stream, claimed)
|
||||
return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id,
|
||||
filename, claimed, size, checksum.hexdigest())
|
||||
finally:
|
||||
objects.upload_slots.release()
|
||||
|
||||
@app.get('/api/v1/attachments/{aid}')
|
||||
def metadata(request: Request, aid: str):
|
||||
return objects.public(objects.authorized(request, aid, authenticate))
|
||||
|
||||
@app.get('/api/v1/attachments/{aid}/content')
|
||||
def content(request: Request, aid: str):
|
||||
row = objects.authorized(request, aid, authenticate)
|
||||
if row['state'] != 'ready':
|
||||
raise objects.error(409, 'conflict', 'attachment unavailable')
|
||||
variant = request.query_params.get('variant', 'original')
|
||||
if variant != 'original':
|
||||
selected = next((v for v in row['variants'] if v['name'] == variant), None)
|
||||
if selected is None:
|
||||
raise objects.error(404, 'not_found', 'variant not found')
|
||||
row = {**row, **selected}
|
||||
return objects.download(row)
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
name: mtusi-lab5
|
||||
x-api: &api
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-lab5-api:local}
|
||||
environment: &environment
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}@db:5432/messenger
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
CACHE_URL: redis://valkey:6379/0
|
||||
SESSION_STORE: valkey
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
DB_POOL_MAX_SIZE: 5
|
||||
CACHE_CLIENT_MODE: shared
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: lab4api
|
||||
S3_SECRET_ACCESS_KEY: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
cpus: ${API_CPUS:-2}
|
||||
mem_limit: 512m
|
||||
stop_grace_period: 40s
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, pg_isready -U messenger -d messenger]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.2-alpine
|
||||
command: [valkey-server, --appendonly, 'yes']
|
||||
volumes: [valkey_data:/data]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
s3:
|
||||
# RELEASE.2025-09-07T16-13-09Z, verified locally, linux/arm64.
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER: lab4root
|
||||
MINIO_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
MINIO_BROWSER: 'off'
|
||||
volumes: [s3_data:/data]
|
||||
cpus: 0.5
|
||||
mem_limit: 768m
|
||||
s3-init:
|
||||
# mc RELEASE.2025-08-13T08-35-41Z.
|
||||
image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727
|
||||
entrypoint: [/bin/sh, /bootstrap/init-s3.sh]
|
||||
environment:
|
||||
S3_ROOT_USER: lab4root
|
||||
S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
S3_API_USER: lab4api
|
||||
S3_API_PASSWORD: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh}
|
||||
volumes: ['./infra:/bootstrap:ro']
|
||||
depends_on: [s3]
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
migrate:
|
||||
<<: *api
|
||||
command: [python, migrate.py]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
api:
|
||||
<<: *api
|
||||
command: [uvicorn, 'app:app', --host, 0.0.0.0, --port, '8080', --timeout-graceful-shutdown, '35', --limit-concurrency, '64']
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_started}
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import socket; s=socket.create_connection(('127.0.0.1',8080),1); s.sendall(b'GET /health/ready HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n'); assert s.makefile('rb').readline().split()[1]==b'200'"]
|
||||
interval: 2s
|
||||
timeout: 4s
|
||||
retries: 2
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
ports: ['127.0.0.1:${APP_PORT:-8085}:8080']
|
||||
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
|
||||
depends_on: [api]
|
||||
cpus: 0.25
|
||||
mem_limit: 64m
|
||||
broker:
|
||||
image: valkey/valkey:8.0.2-alpine
|
||||
command: [valkey-server, --appendonly, 'yes', --appendfsync, always, --maxmemory-policy, noeviction]
|
||||
volumes: [broker_data:/data]
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
restart: unless-stopped
|
||||
publisher:
|
||||
<<: *api
|
||||
command: [python, publisher.py]
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_started}
|
||||
worker:
|
||||
<<: *api
|
||||
command: [python, worker.py]
|
||||
restart: unless-stopped
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
broker: {condition: service_started}
|
||||
janitor:
|
||||
<<: *api
|
||||
command: [python, janitor.py]
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
ops:
|
||||
<<: *api
|
||||
profiles: [tools]
|
||||
volumes: ['./tests:/app/tests:ro', './scripts:/app/scripts:ro']
|
||||
command: [python, manage.py, failed]
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
s3_data:
|
||||
broker_data:
|
||||
+15
-3
@@ -1,5 +1,17 @@
|
||||
# Доказательства
|
||||
# Доказательства лабораторной 5
|
||||
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
Файлы получены реальными запусками из этого репозитория. Credentials, тела запросов с паролями и сессионные токены сюда не записываются.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
- `check.txt`, `validate.txt`, `smoke.txt`, `readiness.txt`: публичные проверки и две здоровые API.
|
||||
- `pipeline-faults.json`: уровень 5, трек 2; матрица crash/дубли/версии и дополнительные retry/replay/outbox-сценарии.
|
||||
- `fault-summary.csv`, `pipeline-faults.png`, `pipeline-faults.svg`: сводка матрицы, генерируется `scripts/summarize_research.py`.
|
||||
- `lifecycle.json`: SIGKILL API, restart broker, сохранность принятого файла и реальный lease timeout.
|
||||
- `backpressure.json`: 100 загрузок при остановленном broker, 429 на следующей, полное восстановление.
|
||||
- `cleanup-and-limits.json`: настоящие child timeout/MemoryError с уменьшенными тестовыми бюджетами; cleanup с ускоренными часами.
|
||||
- `uploads-negative.json`, `private-storage.json`, `image-variants.json`: ограничения, авторизация, приватность и реальные варианты изображений.
|
||||
- `cache-consistency.txt`, `idempotency.json`: сохранённые возможности прошлых лабораторных.
|
||||
- `persistence.txt`, `persistence-before.json`, `persistence-after.json`, `persistence-verified.json`: сохранность и проверка фактического перезапуска всех сервисов.
|
||||
- `failover.txt`: потеря одной API с восстановлением записи.
|
||||
- `runtime.json`, `containers.json`: безопасная сводка Docker VM, лимитов и volumes.
|
||||
|
||||
Матрица не измеряет production SLA. Для ускоренной lease/cleanup/retry-проверки явно указано изменение тестовых часов/расписания; отдельный lifecycle использует настоящий lease.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"accepted_while_broker_down": 100,
|
||||
"next_upload_status": 429,
|
||||
"messages_available": true,
|
||||
"all_ready": 100,
|
||||
"admission_seconds": 2.997,
|
||||
"drain_seconds": 24.64,
|
||||
"lost_accepted": 0,
|
||||
"profile": "100 PNG 320x200, 8 upload clients, 1 serial worker; closed-loop; not a throughput benchmark"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
cache_key_exists=True
|
||||
ttl_seconds=3599
|
||||
avg_latency_ms=1.90
|
||||
p95_latency_ms=2.60
|
||||
stale_cache_rejected=True
|
||||
forged_identity_rejected=True
|
||||
concurrent_identity_checks=8/8
|
||||
malformed_cache_recovers=True
|
||||
repopulated_revoked_cache_rejected=True
|
||||
short_ttl_recovers_from_db=True
|
||||
logout_revokes_cache=True
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"case": "real_child_timeout_with_test_budget_50ms",
|
||||
"status": "PASS",
|
||||
"elapsed_ms": 68.024
|
||||
},
|
||||
{
|
||||
"case": "real_child_address_space_limit_32MiB",
|
||||
"status": "PASS",
|
||||
"error_code": "resource_limit"
|
||||
},
|
||||
{
|
||||
"case": "staging_dry_run_apply_repeat",
|
||||
"status": "PASS",
|
||||
"candidates": 23,
|
||||
"repeat_candidates": 0,
|
||||
"active_retained": true,
|
||||
"clock": "advanced_2_days"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
[
|
||||
{
|
||||
"name": "/mtusi-lab5-api-1",
|
||||
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
|
||||
"hostname": "a6c7dcf30787",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:20.56568455Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-api-2",
|
||||
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
|
||||
"hostname": "d43ff1cbdbbc",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:20.41170375Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-broker-1",
|
||||
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
|
||||
"hostname": "43400a668650",
|
||||
"service": "broker",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:28.257499705Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_broker_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-db-1",
|
||||
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
|
||||
"hostname": "f20a9e4d8783",
|
||||
"service": "db",
|
||||
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:51.02995726Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_postgres_data",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-janitor-1",
|
||||
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
|
||||
"hostname": "8ec2d9f5b984",
|
||||
"service": "janitor",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.911355944Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-lb-1",
|
||||
"id": "14adfe35371f51e2272c03a8a300f62311ff31ba28236048a75e12d648e6fd82",
|
||||
"hostname": "14adfe35371f",
|
||||
"service": "lb",
|
||||
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:14.230730447Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 67108864,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8085"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-migrate-1",
|
||||
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
|
||||
"hostname": "c265f53ce7e9",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:32:07.768599294Z",
|
||||
"health": null,
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-publisher-1",
|
||||
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
|
||||
"hostname": "acdc5d34e3b0",
|
||||
"service": "publisher",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:38.59219456Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-1",
|
||||
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
|
||||
"hostname": "554bcd156ad2",
|
||||
"service": "s3",
|
||||
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:51.22311738Z",
|
||||
"health": null,
|
||||
"nano_cpus": 500000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_s3_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-init-1",
|
||||
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
|
||||
"hostname": "d7a59a310290",
|
||||
"service": "s3-init",
|
||||
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:32:07.290372244Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-valkey-1",
|
||||
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
|
||||
"hostname": "8c620bff1a6c",
|
||||
"service": "valkey",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:50.863364684Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 134217728,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_valkey_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-worker-1",
|
||||
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
|
||||
"hostname": "72ecae4e2677",
|
||||
"service": "worker",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:31:38.594433513Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8085"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 2, "first_recovery_seconds": 2.171}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
scenario,repeats,mean_ms,min_ms,max_ms,stdev_ms,object_sets,objects_per_current_version,failures
|
||||
before_effect,3,1550.718,1321.347,1786.888,232.845,1,4,0
|
||||
after_effect,3,1549.527,1334.593,1872.025,284.404,1,4,0
|
||||
after_commit,3,1383.915,1275.246,1590.414,178.915,1,4,0
|
||||
concurrent_duplicates,3,143.795,121.12,172.849,26.448,1,4,0
|
||||
reorder_stale_version,3,223.21,109.191,440.476,188.235,1,4,0
|
||||
|
@@ -0,0 +1 @@
|
||||
{"parallel_requests": 16, "observed_instances": ["46b8beae40b9", "a684e742220a"], "unique_messages": 1, "conflict": 409, "semantic_replay": true, "user_scope": true}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"case": "small_alpha",
|
||||
"status": "PASS",
|
||||
"dimensions": {
|
||||
"thumbnail": [
|
||||
80,
|
||||
40
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
80,
|
||||
40
|
||||
]
|
||||
},
|
||||
"original_sha256": "2a4135aed00454b2566994964a2efffd9fd5593454d4d2f5b102aa3c42b2d3bd"
|
||||
},
|
||||
{
|
||||
"case": "exif_orientation",
|
||||
"status": "PASS",
|
||||
"dimensions": {
|
||||
"thumbnail": [
|
||||
64,
|
||||
256
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
100,
|
||||
400
|
||||
]
|
||||
},
|
||||
"original_sha256": "43ba669d1496cb5857d62be714452e087d474b2421f70f3cba3264dca491d026"
|
||||
},
|
||||
{
|
||||
"case": "center_crop",
|
||||
"status": "PASS",
|
||||
"dimensions": {
|
||||
"thumbnail": [
|
||||
256,
|
||||
64
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
400,
|
||||
100
|
||||
]
|
||||
},
|
||||
"original_sha256": "983bf89131f785c45b587c57640d62d223af08bfc71fcf28a27faed486af9583"
|
||||
},
|
||||
{
|
||||
"case": "large_fit",
|
||||
"status": "PASS",
|
||||
"dimensions": {
|
||||
"thumbnail": [
|
||||
256,
|
||||
128
|
||||
],
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
1280,
|
||||
640
|
||||
]
|
||||
},
|
||||
"original_sha256": "fd1aebd1ed7ce3d142d0e38b7dc2d3b6c106a71d399e6fad25706c1f3ad8850b"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"scenario": "stopped_worker_api_kill_outbox_broker_restart",
|
||||
"status": "PASS",
|
||||
"attachment_id": "882900a6-2551-4975-90ff-1a1ca424b0ee",
|
||||
"stream_length_before": 1,
|
||||
"stream_length_after": 1,
|
||||
"ready_ms": 3023.094,
|
||||
"containers_before": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"id": "a6c7dcf30787",
|
||||
"started_at": "2026-11-20T15:28:14.515508921Z",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"id": "d43ff1cbdbbc",
|
||||
"started_at": "2026-11-20T15:27:52.500502971Z",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-broker-1": {
|
||||
"id": "43400a668650",
|
||||
"started_at": "2026-11-20T15:27:50.923553697Z",
|
||||
"volumes": [
|
||||
"mtusi-lab5_broker_data"
|
||||
]
|
||||
}
|
||||
},
|
||||
"containers_after": {
|
||||
"mtusi-lab5-api-1": {
|
||||
"id": "a6c7dcf30787",
|
||||
"started_at": "2026-11-20T15:31:20.56568455Z",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-api-2": {
|
||||
"id": "d43ff1cbdbbc",
|
||||
"started_at": "2026-11-20T15:31:20.41170375Z",
|
||||
"volumes": []
|
||||
},
|
||||
"mtusi-lab5-broker-1": {
|
||||
"id": "43400a668650",
|
||||
"started_at": "2026-11-20T15:31:28.257499705Z",
|
||||
"volumes": [
|
||||
"mtusi-lab5_broker_data"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "crash_real_lease_expiry",
|
||||
"status": "PASS",
|
||||
"attachment_id": "4c7827d8-b5f2-4b58-a019-a5d26099c12b",
|
||||
"ready_ms": 28922.739,
|
||||
"job": {
|
||||
"job_id": "12c2c00c-8114-4c19-a9c6-2b4ffd42bd29",
|
||||
"state": "ready",
|
||||
"attempt": 2,
|
||||
"lease_until": null,
|
||||
"pipeline_version": 1,
|
||||
"published_at": "2026-11-20 15:32:05.409533+00:00",
|
||||
"deliveries": 2
|
||||
},
|
||||
"lease_clock": "real"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
[
|
||||
{
|
||||
"name": "/mtusi-lab5-api-1",
|
||||
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
|
||||
"hostname": "a6c7dcf30787",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.845602697Z",
|
||||
"health": "starting",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-api-2",
|
||||
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
|
||||
"hostname": "d43ff1cbdbbc",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.500502971Z",
|
||||
"health": "starting",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-broker-1",
|
||||
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
|
||||
"hostname": "43400a668650",
|
||||
"service": "broker",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:50.923553697Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_broker_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-db-1",
|
||||
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
|
||||
"hostname": "f20a9e4d8783",
|
||||
"service": "db",
|
||||
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:51.02995726Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_postgres_data",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-janitor-1",
|
||||
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
|
||||
"hostname": "8ec2d9f5b984",
|
||||
"service": "janitor",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.911355944Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-lb-1",
|
||||
"id": "65e0d941c1b56b64cfc79d597ded2d8a64f8aa32ce378e6f3c737525e7ab4f37",
|
||||
"hostname": "65e0d941c1b5",
|
||||
"service": "lb",
|
||||
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:51.271530136Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 67108864,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "",
|
||||
"HostPort": "8085"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-migrate-1",
|
||||
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
|
||||
"hostname": "c265f53ce7e9",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:27:36.704304537Z",
|
||||
"health": null,
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-publisher-1",
|
||||
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
|
||||
"hostname": "acdc5d34e3b0",
|
||||
"service": "publisher",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.978480354Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-1",
|
||||
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
|
||||
"hostname": "554bcd156ad2",
|
||||
"service": "s3",
|
||||
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:51.22311738Z",
|
||||
"health": null,
|
||||
"nano_cpus": 500000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_s3_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-init-1",
|
||||
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
|
||||
"hostname": "d7a59a310290",
|
||||
"service": "s3-init",
|
||||
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:27:36.140115508Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-valkey-1",
|
||||
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
|
||||
"hostname": "8c620bff1a6c",
|
||||
"service": "valkey",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:50.863364684Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 134217728,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_valkey_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-worker-1",
|
||||
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
|
||||
"hostname": "72ecae4e2677",
|
||||
"service": "worker",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:52.000472553Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,197 @@
|
||||
[
|
||||
{
|
||||
"name": "/mtusi-lab5-api-1",
|
||||
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
|
||||
"hostname": "a6c7dcf30787",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:26:31.479663938Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-api-2",
|
||||
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
|
||||
"hostname": "d43ff1cbdbbc",
|
||||
"service": "api",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:26:31.171165305Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-broker-1",
|
||||
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
|
||||
"hostname": "43400a668650",
|
||||
"service": "broker",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:25:26.510562536Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_broker_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-db-1",
|
||||
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
|
||||
"hostname": "f20a9e4d8783",
|
||||
"service": "db",
|
||||
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:18:03.662849926Z",
|
||||
"health": "healthy",
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_postgres_data",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-janitor-1",
|
||||
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
|
||||
"hostname": "8ec2d9f5b984",
|
||||
"service": "janitor",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:26:31.168446315Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-lb-1",
|
||||
"id": "65e0d941c1b56b64cfc79d597ded2d8a64f8aa32ce378e6f3c737525e7ab4f37",
|
||||
"hostname": "65e0d941c1b5",
|
||||
"service": "lb",
|
||||
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:26:31.646228277Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 67108864,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "",
|
||||
"HostPort": "8085"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-migrate-1",
|
||||
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
|
||||
"hostname": "c265f53ce7e9",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:27:36.704304537Z",
|
||||
"health": null,
|
||||
"nano_cpus": 2000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-publisher-1",
|
||||
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
|
||||
"hostname": "acdc5d34e3b0",
|
||||
"service": "publisher",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:38.479168598Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 201326592,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-1",
|
||||
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
|
||||
"hostname": "554bcd156ad2",
|
||||
"service": "s3",
|
||||
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:18:03.664833261Z",
|
||||
"health": null,
|
||||
"nano_cpus": 500000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_s3_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-init-1",
|
||||
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
|
||||
"hostname": "d7a59a310290",
|
||||
"service": "s3-init",
|
||||
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-20T15:27:36.140115508Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 536870912,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-valkey-1",
|
||||
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
|
||||
"hostname": "8c620bff1a6c",
|
||||
"service": "valkey",
|
||||
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:18:03.65752301Z",
|
||||
"health": null,
|
||||
"nano_cpus": 250000000,
|
||||
"memory_bytes": 134217728,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
{
|
||||
"name": "mtusi-lab5_valkey_data",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-worker-1",
|
||||
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
|
||||
"hostname": "72ecae4e2677",
|
||||
"service": "worker",
|
||||
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-20T15:27:38.475722085Z",
|
||||
"health": null,
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_bytes": 805306368,
|
||||
"published_ports": {},
|
||||
"volumes": []
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"running_processes_restarted": 10,
|
||||
"rows": [
|
||||
{
|
||||
"name": "/mtusi-lab5-api-1",
|
||||
"service": "api",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-api-2",
|
||||
"service": "api",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-broker-1",
|
||||
"service": "broker",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-db-1",
|
||||
"service": "db",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-janitor-1",
|
||||
"service": "janitor",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-lb-1",
|
||||
"service": "lb",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-publisher-1",
|
||||
"service": "publisher",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-s3-1",
|
||||
"service": "s3",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-valkey-1",
|
||||
"service": "valkey",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
},
|
||||
{
|
||||
"name": "/mtusi-lab5-worker-1",
|
||||
"service": "worker",
|
||||
"same_container": true,
|
||||
"process_restarted": true,
|
||||
"same_volumes": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8085"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,213 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"track": 2,
|
||||
"matrix_repeats": 3,
|
||||
"rows": [
|
||||
{
|
||||
"iteration": 1,
|
||||
"scenario": "before_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1786.888,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 1,
|
||||
"scenario": "after_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1441.962,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 1,
|
||||
"scenario": "after_commit",
|
||||
"status": "PASS",
|
||||
"attempts": 1,
|
||||
"recovery_ms": 1286.085,
|
||||
"objects": 4,
|
||||
"lease_expiry": "not_needed"
|
||||
},
|
||||
{
|
||||
"iteration": 1,
|
||||
"scenario": "concurrent_duplicates",
|
||||
"status": "PASS",
|
||||
"deliveries": 8,
|
||||
"effects": 1,
|
||||
"recovery_ms": 137.417,
|
||||
"outcomes": [
|
||||
"ready",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer"
|
||||
],
|
||||
"objects": 4
|
||||
},
|
||||
{
|
||||
"iteration": 1,
|
||||
"scenario": "reorder_stale_version",
|
||||
"status": "PASS",
|
||||
"old_deliveries": 5,
|
||||
"current_version": 2,
|
||||
"old_objects": 0,
|
||||
"objects": 4,
|
||||
"recovery_ms": 440.476
|
||||
},
|
||||
{
|
||||
"iteration": 2,
|
||||
"scenario": "before_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1543.92,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 2,
|
||||
"scenario": "after_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1872.025,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 2,
|
||||
"scenario": "after_commit",
|
||||
"status": "PASS",
|
||||
"attempts": 1,
|
||||
"recovery_ms": 1590.414,
|
||||
"objects": 4,
|
||||
"lease_expiry": "not_needed"
|
||||
},
|
||||
{
|
||||
"iteration": 2,
|
||||
"scenario": "concurrent_duplicates",
|
||||
"status": "PASS",
|
||||
"deliveries": 8,
|
||||
"effects": 1,
|
||||
"recovery_ms": 172.849,
|
||||
"outcomes": [
|
||||
"ready",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer"
|
||||
],
|
||||
"objects": 4
|
||||
},
|
||||
{
|
||||
"iteration": 2,
|
||||
"scenario": "reorder_stale_version",
|
||||
"status": "PASS",
|
||||
"old_deliveries": 5,
|
||||
"current_version": 2,
|
||||
"old_objects": 0,
|
||||
"objects": 4,
|
||||
"recovery_ms": 109.191
|
||||
},
|
||||
{
|
||||
"iteration": 3,
|
||||
"scenario": "before_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1321.347,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 3,
|
||||
"scenario": "after_effect",
|
||||
"status": "PASS",
|
||||
"attempts": 2,
|
||||
"recovery_ms": 1334.593,
|
||||
"objects": 4,
|
||||
"lease_expiry": "accelerated"
|
||||
},
|
||||
{
|
||||
"iteration": 3,
|
||||
"scenario": "after_commit",
|
||||
"status": "PASS",
|
||||
"attempts": 1,
|
||||
"recovery_ms": 1275.246,
|
||||
"objects": 4,
|
||||
"lease_expiry": "not_needed"
|
||||
},
|
||||
{
|
||||
"iteration": 3,
|
||||
"scenario": "concurrent_duplicates",
|
||||
"status": "PASS",
|
||||
"deliveries": 8,
|
||||
"effects": 1,
|
||||
"recovery_ms": 121.12,
|
||||
"outcomes": [
|
||||
"defer",
|
||||
"ready",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer",
|
||||
"defer"
|
||||
],
|
||||
"objects": 4
|
||||
},
|
||||
{
|
||||
"iteration": 3,
|
||||
"scenario": "reorder_stale_version",
|
||||
"status": "PASS",
|
||||
"old_deliveries": 5,
|
||||
"current_version": 2,
|
||||
"old_objects": 0,
|
||||
"objects": 4,
|
||||
"recovery_ms": 119.964
|
||||
},
|
||||
{
|
||||
"scenario": "published_v1_then_v2_then_stale_v1",
|
||||
"status": "PASS",
|
||||
"current_version": 2,
|
||||
"objects": 4
|
||||
},
|
||||
{
|
||||
"scenario": "permanent_failed_replay",
|
||||
"status": "PASS",
|
||||
"objects": 4
|
||||
},
|
||||
{
|
||||
"scenario": "bounded_retry_jitter",
|
||||
"status": "PASS",
|
||||
"max_attempts": 5,
|
||||
"delays_seconds": [
|
||||
1.080882,
|
||||
2.447441,
|
||||
2.182634,
|
||||
8.30795
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "outbox_republish",
|
||||
"status": "PASS",
|
||||
"objects": 4
|
||||
}
|
||||
],
|
||||
"accepted_in_database": 151,
|
||||
"invariants": {
|
||||
"accepted_tasks_lost": 0,
|
||||
"duplicate_variant_sets": 0,
|
||||
"stale_version_overwrites": 0
|
||||
},
|
||||
"limitations": [
|
||||
"matrix lease expiry accelerated with SQL",
|
||||
"single host, worker processing is serial except intentional duplicate probe",
|
||||
"no disk-loss or network partition proof"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 86 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 62 KiB |
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"anonymous_bucket_and_object_status": [
|
||||
403,
|
||||
403
|
||||
],
|
||||
"application_admin_operations_denied": [
|
||||
"create_bucket",
|
||||
"get_bucket_policy",
|
||||
"write_outside_prefix"
|
||||
],
|
||||
"visible_bucket_names": [
|
||||
"messenger-attachments"
|
||||
],
|
||||
"console_reachable": false
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: healthy API replicas observed through LB
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"ServerVersion": "27.4.0",
|
||||
"Architecture": "aarch64",
|
||||
"OSType": "linux",
|
||||
"NCPU": 4,
|
||||
"MemTotal": 8308158464,
|
||||
"KernelVersion": "6.8.0-50-generic"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8085"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"cases": [
|
||||
{
|
||||
"case": "corrupt_png",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "truncated_png",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "mime_spoof",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "unsupported_mime",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "empty_payload",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "unsafe_filename",
|
||||
"status": 400
|
||||
},
|
||||
{
|
||||
"case": "foreign_chat_upload",
|
||||
"status": 404
|
||||
},
|
||||
{
|
||||
"case": "unauthenticated_upload",
|
||||
"status": 401
|
||||
},
|
||||
{
|
||||
"case": "over_40_million_pixels",
|
||||
"status": 413
|
||||
},
|
||||
{
|
||||
"case": "oversized_content_length_before_body",
|
||||
"status": 413
|
||||
},
|
||||
{
|
||||
"case": "chunked_oversize",
|
||||
"status": 413
|
||||
},
|
||||
{
|
||||
"case": "foreign_attachment_metadata_and_bytes",
|
||||
"status": 404
|
||||
},
|
||||
{
|
||||
"case": "cross_chat_and_unknown_attachment",
|
||||
"status": 404
|
||||
},
|
||||
{
|
||||
"case": "attachment_idempotency_and_conflict",
|
||||
"status": "201/409"
|
||||
},
|
||||
{
|
||||
"case": "truncated_jpeg",
|
||||
"status": 415
|
||||
},
|
||||
{
|
||||
"case": "interrupted_http_body",
|
||||
"metadata_or_object_added": false
|
||||
}
|
||||
],
|
||||
"count": 21
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
+7
-4
@@ -1,7 +1,10 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура решения
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Главный файл — [compose.yaml](../compose.yaml). Все сервисы и volumes принадлежат проекту `mtusi-lab5`; наружу опубликован только HAProxy на `127.0.0.1:8085`.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
- `haproxy.cfg`: round-robin между API, health checks `/health/ready`, обновление адресов через Docker DNS.
|
||||
- `init-s3.sh`: повторяемая инициализация приватного bucket и отдельной application-учётной записи. Административные credentials есть только у MinIO/init.
|
||||
- `api-policy.json`: list своего bucket и get/put/delete только `originals/*` для совместимости, `staging/*`, `results/*`; административных прав нет.
|
||||
- `local.env.example`: пример фактических параметров. Секреты генерируются `scripts/bootstrap.sh` и хранятся в игнорируемом `.env`.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
`broker` и `valkey` — отдельные контейнеры: очередь Streams с AOF appendfsync=always и кэш сессий. У PostgreSQL, обоих Valkey и MinIO свои persistent volumes. Все зависимости запускаются из этого репозитория.
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},
|
||||
{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*","arn:aws:s3:::messenger-attachments/staging/*","arn:aws:s3:::messenger-attachments/results/*"]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 512
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
timeout connect 1s
|
||||
timeout client 45s
|
||||
timeout server 45s
|
||||
timeout check 1s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
|
||||
server-template api 8 api:8080 check
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ready=false
|
||||
for attempt in $(seq 1 60); do
|
||||
if mc alias set local http://s3:9000 "$S3_ROOT_USER" "$S3_ROOT_PASSWORD" >/dev/null 2>&1; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
[ "$ready" = true ] || exit 1
|
||||
mc mb --ignore-existing local/messenger-attachments >/dev/null
|
||||
mc anonymous set none local/messenger-attachments >/dev/null
|
||||
mc admin user add local "$S3_API_USER" "$S3_API_PASSWORD" >/dev/null
|
||||
mc admin policy create local lab4-attachments /bootstrap/api-policy.json >/dev/null
|
||||
mc admin policy attach local lab4-attachments --user "$S3_API_USER" >/dev/null
|
||||
printf '%s\n' 'PASS: private bucket and restricted application principal configured'
|
||||
@@ -0,0 +1,12 @@
|
||||
# scripts/bootstrap.sh generates these three secrets into ignored .env.
|
||||
POSTGRES_PASSWORD=replace-with-random-secret
|
||||
S3_ROOT_PASSWORD=replace-with-random-secret
|
||||
S3_API_PASSWORD=replace-with-random-secret
|
||||
APP_PORT=8085
|
||||
API_REPLICAS=2
|
||||
API_CPUS=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=20
|
||||
JOB_LEASE_SECONDS=30
|
||||
MAX_PENDING_JOBS=100
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""Conservative staging cleanup; all current queued/processing jobs are retained."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
from datetime import datetime,timezone
|
||||
from psycopg.rows import dict_row
|
||||
from app import store,object_store
|
||||
|
||||
stop=threading.Event()
|
||||
|
||||
|
||||
def sweep(apply=False,retention=None):
|
||||
retention=max(60,int(retention or os.getenv('STAGING_RETENTION_SECONDS','86400')))
|
||||
deleted=[]
|
||||
pages=object_store.client.get_paginator('list_objects_v2').paginate(Bucket=object_store.bucket,Prefix='staging/')
|
||||
for page in pages:
|
||||
for item in page.get('Contents',[]):
|
||||
if (datetime.now(timezone.utc)-item['LastModified']).total_seconds()<retention:continue
|
||||
# Admission holds this same lock from S3 put through DB COMMIT.
|
||||
with store.connect() as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(202605)')
|
||||
rows=conn.cursor(row_factory=dict_row).execute('SELECT id,state,staging_key FROM attachments WHERE staging_key=%s FOR UPDATE',(item['Key'],)).fetchall()
|
||||
if any(row['state'] in ('queued','processing','pending') for row in rows):continue
|
||||
head=object_store.client.head_object(Bucket=object_store.bucket,Key=item['Key'])
|
||||
if (datetime.now(timezone.utc)-head['LastModified']).total_seconds()<retention:continue
|
||||
if apply:
|
||||
object_store.client.delete_object(Bucket=object_store.bucket,Key=item['Key'])
|
||||
conn.execute('UPDATE attachments SET staging_key=NULL WHERE staging_key=%s',(item['Key'],))
|
||||
deleted.append(item['Key'])
|
||||
return {'apply':apply,'retention_seconds':retention,'staging_candidates':deleted}
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('--once',action='store_true');p.add_argument('--apply',action='store_true');args=p.parse_args()
|
||||
store.pool.open(wait=True,timeout=30)
|
||||
for sig in (signal.SIGINT,signal.SIGTERM):signal.signal(sig,lambda *_:stop.set())
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:print(json.dumps(sweep(args.apply or not args.once)),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True)
|
||||
if args.once:raise
|
||||
if args.once:break
|
||||
stop.wait(60)
|
||||
finally:
|
||||
store.pool.close();object_store.close()
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Trusted operator CLI, no public replay or arbitrary job submission endpoint."""
|
||||
import argparse
|
||||
import json
|
||||
from psycopg.rows import dict_row
|
||||
from app import store
|
||||
from pipeline import replay,upgrade
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('action',choices=['failed','replay','upgrade']);p.add_argument('id',nargs='?');args=p.parse_args()
|
||||
store.pool.open(wait=True,timeout=30)
|
||||
try:
|
||||
if args.action=='failed':
|
||||
with store.connect() as conn:
|
||||
rows=conn.cursor(row_factory=dict_row).execute("SELECT id,attachment_id,pipeline_version,attempt,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at LIMIT 100").fetchall()
|
||||
print(json.dumps(rows,default=str,indent=2))
|
||||
elif not args.id:p.error('id is required')
|
||||
elif args.action=='replay':replay(store,args.id);print('requeued')
|
||||
else:print(upgrade(store,args.id))
|
||||
finally:store.pool.close()
|
||||
@@ -0,0 +1,8 @@
|
||||
from app import store
|
||||
|
||||
if __name__ == '__main__':
|
||||
store.pool.open(wait=True, timeout=30)
|
||||
try:
|
||||
store.init()
|
||||
finally:
|
||||
store.pool.close()
|
||||
@@ -0,0 +1,41 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY,
|
||||
token TEXT NOT NULL UNIQUE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages(chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_members_user ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_created ON chats(created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_requests (
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
key VARCHAR(128) NOT NULL,
|
||||
body TEXT NOT NULL,
|
||||
response JSONB NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY(sender_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests(expires_at);
|
||||
@@ -0,0 +1,27 @@
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
state TEXT NOT NULL CHECK(state IN ('pending','ready','failed')),
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
CREATE INDEX attachments_state_age ON attachments(state, updated_at);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id, attachment_id),
|
||||
UNIQUE(message_id, position)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments(attachment_id);
|
||||
-- Preserve replay of lab3 text requests if an existing lab3 DB is upgraded.
|
||||
ALTER TABLE message_requests ALTER COLUMN body TYPE JSONB
|
||||
USING jsonb_build_object('text', body, 'attachment_ids', '[]'::jsonb);
|
||||
UPDATE message_requests SET response = response || '{"attachment_ids":[]}'::jsonb;
|
||||
@@ -0,0 +1,36 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_state_check CHECK(state IN ('pending','queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK(pipeline_version>0);
|
||||
ALTER TABLE attachments ADD COLUMN staging_key TEXT;
|
||||
ALTER TABLE attachments ADD COLUMN variants JSONB NOT NULL DEFAULT '[]';
|
||||
CREATE TABLE jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL CHECK(pipeline_version>0),
|
||||
state TEXT NOT NULL CHECK(state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt INTEGER NOT NULL DEFAULT 0,
|
||||
lease_until TIMESTAMPTZ,
|
||||
next_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE TABLE outbox (
|
||||
job_id UUID PRIMARY KEY REFERENCES jobs(id),
|
||||
published_at TIMESTAMPTZ,
|
||||
deliveries INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE job_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID REFERENCES jobs(id),
|
||||
event TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX jobs_due ON jobs(state,next_at);
|
||||
|
||||
-- Upgrade existing lab4 ready originals through the asynchronous pipeline.
|
||||
UPDATE attachments SET staging_key=object_key,state='queued',updated_at=now() WHERE state IN ('ready','pending');
|
||||
INSERT INTO jobs(id,attachment_id,pipeline_version,state)
|
||||
SELECT gen_random_uuid(),id,pipeline_version,'queued' FROM attachments WHERE state='queued';
|
||||
INSERT INTO outbox(job_id) SELECT id FROM jobs;
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""Database authority, version fencing and safe operator actions."""
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from psycopg.rows import dict_row
|
||||
import redis
|
||||
|
||||
STREAM = 'attachments.v1'
|
||||
GROUP = 'image-workers'
|
||||
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
|
||||
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '20')))
|
||||
LEASE = max(TIMEOUT + 5, int(os.getenv('JOB_LEASE_SECONDS', '30')))
|
||||
|
||||
|
||||
def broker():
|
||||
return redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
|
||||
decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
|
||||
|
||||
|
||||
def create_job(conn, aid, version):
|
||||
jid = str(uuid.uuid4())
|
||||
conn.execute("INSERT INTO jobs(id,attachment_id,pipeline_version,state) VALUES (%s,%s,%s,'queued')", (jid,aid,version))
|
||||
conn.execute('INSERT INTO outbox(job_id) VALUES (%s)', (jid,))
|
||||
audit(conn, jid, 'accepted')
|
||||
return jid
|
||||
|
||||
|
||||
def audit(conn, jid, event):
|
||||
conn.execute('INSERT INTO job_audit(job_id,event) VALUES (%s,%s)', (jid,event))
|
||||
|
||||
|
||||
def envelope(row):
|
||||
return dict(schema_version=1,job_id=str(row['id']),type='attachment.process.v1',
|
||||
attachment_id=str(row['attachment_id']),pipeline_version=row['pipeline_version'],
|
||||
attempt=row['attempt']+1,created_at=row['created_at'].isoformat().replace('+00:00','Z'))
|
||||
|
||||
|
||||
def fetch_job(conn, jid, suffix=''):
|
||||
return conn.cursor(row_factory=dict_row).execute('SELECT * FROM jobs WHERE id=%s '+suffix, (jid,)).fetchone()
|
||||
|
||||
|
||||
def replay(store, jid):
|
||||
with store.connect() as conn:
|
||||
job = fetch_job(conn,jid,'FOR UPDATE')
|
||||
if job is None:
|
||||
raise ValueError('unknown job')
|
||||
att = conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (job['attachment_id'],)).fetchone()
|
||||
if job['state'] != 'failed' or att['pipeline_version'] != job['pipeline_version'] or not att['staging_key']:
|
||||
raise ValueError('only current failed job with retained staging can be replayed')
|
||||
conn.execute("UPDATE jobs SET state='queued',attempt=0,error_code=NULL,next_at=now(),lease_until=NULL WHERE id=%s",(jid,))
|
||||
conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=now() WHERE id=%s",(att['id'],))
|
||||
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
|
||||
audit(conn,jid,'operator_replay')
|
||||
|
||||
|
||||
def upgrade(store, aid):
|
||||
# Only trusted CLI; never accept version/storage paths from HTTP clients.
|
||||
with store.connect() as conn:
|
||||
# Lock jobs before attachment, same order as the worker.
|
||||
conn.execute('SELECT id FROM jobs WHERE attachment_id=%s ORDER BY pipeline_version FOR UPDATE',(aid,)).fetchall()
|
||||
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(aid,)).fetchone()
|
||||
if att is None or (att['state'] != 'ready' and not att['staging_key']):
|
||||
raise ValueError('upgrade requires durable source')
|
||||
version=att['pipeline_version']+1
|
||||
# A completed immutable original is itself a durable source for upgrade.
|
||||
conn.execute("UPDATE attachments SET pipeline_version=%s,staging_key=CASE WHEN state='ready' THEN object_key ELSE staging_key END,state='queued',variants='[]',error_code=NULL,updated_at=now() WHERE id=%s",(version,aid))
|
||||
conn.execute("UPDATE jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",(aid,))
|
||||
return create_job(conn,aid,version)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Recoverable DB outbox -> persistent Valkey Streams (at least once)."""
|
||||
import argparse
|
||||
import json
|
||||
import signal
|
||||
import threading
|
||||
from psycopg.rows import dict_row
|
||||
from app import store
|
||||
from pipeline import broker, envelope, STREAM
|
||||
|
||||
stop=threading.Event()
|
||||
|
||||
|
||||
def publish_once(client):
|
||||
with store.connect() as conn:
|
||||
# Marking published happens AFTER XADD. A crash in between produces a
|
||||
# duplicate. DB remains authoritative; no acknowledged intent is lost.
|
||||
rows=conn.cursor(row_factory=dict_row).execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
|
||||
WHERE j.state IN ('queued','processing') AND j.next_at<=now()
|
||||
AND (o.published_at IS NULL OR (j.state='processing' AND j.lease_until<now()
|
||||
AND o.published_at<now()-interval '5 seconds'))
|
||||
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
|
||||
for row in rows:
|
||||
client.xadd(STREAM,{'job':json.dumps(envelope(row))})
|
||||
conn.execute('UPDATE outbox SET published_at=now(),deliveries=deliveries+1 WHERE job_id=%s',(row['id'],))
|
||||
return len(rows)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser();parser.add_argument('--once',action='store_true');args=parser.parse_args()
|
||||
store.pool.open(wait=True,timeout=30)
|
||||
client=broker()
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda *_:stop.set())
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
count=publish_once(client)
|
||||
if count:print(json.dumps({'published':count}),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'publisher_error':type(exc).__name__}),flush=True)
|
||||
if args.once:raise
|
||||
if args.once:break
|
||||
stop.wait(.25)
|
||||
finally:
|
||||
client.close();store.pool.close()
|
||||
@@ -0,0 +1,2 @@
|
||||
-r requirements-dev.txt
|
||||
matplotlib==3.10.0
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn==0.30.6
|
||||
psycopg[binary]==3.3.6
|
||||
psycopg-pool==3.2.6
|
||||
argon2-cffi==23.1.0
|
||||
python-dotenv==1.0.1
|
||||
redis==5.1.0
|
||||
boto3==1.42.0
|
||||
Pillow==13.0.0
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ ! -f .env ]; then
|
||||
umask 077
|
||||
python3 - <<'PY' > .env
|
||||
import secrets
|
||||
for name in ('POSTGRES_PASSWORD','S3_ROOT_PASSWORD','S3_API_PASSWORD'):
|
||||
print(name+'='+secrets.token_hex(24))
|
||||
print('APP_PORT=8085')
|
||||
PY
|
||||
fi
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
exec docker compose "$@"
|
||||
fi
|
||||
exec docker-compose "$@"
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sanitized real-container evidence. Never dump environment or labels wholesale."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-a','-q'],cwd=ROOT,text=True).split()
|
||||
result=[]
|
||||
for item in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)):
|
||||
result.append({'name':item['Name'],'id':item['Id'],'hostname':item['Config']['Hostname'],
|
||||
'service':item['Config']['Labels']['com.docker.compose.service'],
|
||||
'image_id':item['Image'],'status':item['State']['Status'],
|
||||
'started_at':item['State']['StartedAt'],
|
||||
'health':item['State'].get('Health',{}).get('Status'),
|
||||
'nano_cpus':item['HostConfig']['NanoCpus'],'memory_bytes':item['HostConfig']['Memory'],
|
||||
'published_ports':item['HostConfig']['PortBindings'],
|
||||
'volumes':[{'name':m['Name'],'destination':m['Destination']} for m in item['Mounts'] if m['Type']=='volume']})
|
||||
print(json.dumps(sorted(result,key=lambda c:c['name']),indent=2))
|
||||
@@ -0,0 +1,14 @@
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from app import store
|
||||
from psycopg.rows import dict_row
|
||||
store.pool.open(wait=True,timeout=20)
|
||||
try:
|
||||
with store.connect() as conn:
|
||||
rows=conn.cursor(row_factory=dict_row).execute('''SELECT j.id AS job_id,j.state,j.attempt,j.lease_until,
|
||||
a.pipeline_version,o.published_at,o.deliveries FROM jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
JOIN outbox o ON o.job_id=j.id WHERE a.id=%s ORDER BY j.pipeline_version''',(sys.argv[1],)).fetchall()
|
||||
print(json.dumps(rows,default=str))
|
||||
finally:store.pool.close()
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
trap 'scripts/compose.sh start publisher worker >/dev/null' EXIT
|
||||
scripts/compose.sh stop worker publisher
|
||||
scripts/compose.sh run --rm --no-deps ops python tests/pipeline_faults.py > evidence/pipeline-faults.json
|
||||
scripts/compose.sh run --rm --no-deps ops python tests/cleanup_and_limits.py > evidence/cleanup-and-limits.json
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
scripts/compose.sh restart db valkey broker s3 api lb publisher worker janitor
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?Required API hostname from X-Instance-Id}"
|
||||
for id in $(scripts/compose.sh ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 3 "$id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf '%s\n' 'Target is not a running API replica' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Synchronize operational result tables with real sanitized test artifacts."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
ROOT=Path(__file__).resolve().parents[1];E=ROOT/'evidence'
|
||||
load=lambda name:json.loads((E/name).read_text())
|
||||
life=load('lifecycle.json')['rows'];back=load('backpressure.json');cleanup=load('cleanup-and-limits.json')['rows'];variants=load('image-variants.json')['rows']
|
||||
faults=load('pipeline-faults.json')['rows'];verification=load('persistence-verified.json')
|
||||
failover=json.loads(next(s for s in (E/'failover.txt').read_text().splitlines() if s.startswith('{')))
|
||||
retry=next(r for r in faults if r['scenario']=='bounded_retry_jitter')
|
||||
content='### Восстановление и ограничения\n\n| Воздействие | Реальное наблюдение | Результат |\n| --- | --- | --- |\n'
|
||||
content+=f"| Остановка worker, SIGKILL API до publish, restart broker | Длина stream {life[0]['stream_length_before']} → {life[0]['stream_length_after']}; ready через {life[0]['ready_ms']/1000:.3f} с после запуска worker | PASS, исходник сохранён |\n"
|
||||
content+=f"| Смерть worker после PUT, настоящий lease | Ready через {life[1]['ready_ms']/1000:.3f} с; попыток {life[1]['job']['attempt']} | PASS, без ускорения часов |\n"
|
||||
content+=f"| Остановка broker и заполнение очереди | Принято {back['accepted_while_broker_down']}; следующий HTTP {back['next_upload_status']}; восстановлено {back['all_ready']} за {back['drain_seconds']} с | PASS, потеряно {back['lost_accepted']} |\n"
|
||||
content+=f"| Перезапуск всего стенда | {verification['running_processes_restarted']} реальных процессов, прежние volumes и сессия | PASS |\n"
|
||||
content+=f"| Отказ API | {failover['successful_writes']} успешных записей, {failover['failed_attempts']} переходных ошибок; первая успешная запись через {failover['first_recovery_seconds']} с | PASS |\n"
|
||||
content+=f"| Тайм-аут decode, тестовый бюджет 50 мс | Время сценария {cleanup[0]['elapsed_ms']} мс; retry, затем ready | PASS |\n"
|
||||
content+=f"| Ограничение адресного пространства, тестовый бюджет 32 MiB | Конечный {cleanup[1]['error_code']}, replay завершился | PASS |\n"
|
||||
content+=f"| Staging cleanup, часы +2 дня | Кандидатов {cleanup[2]['candidates']}, повторных {cleanup[2]['repeat_candidates']}; активный исходник сохранён | PASS |\n"
|
||||
content+='\n### Проверка геометрии реальных файлов\n\n| Исходник/сценарий | Thumbnail | Crop | Optimized |\n| --- | --- | --- | --- |\n'
|
||||
for row in variants:
|
||||
dims=lambda key:'×'.join(map(str,row['dimensions'][key]))
|
||||
content+=f"| {row['case']} | {dims('thumbnail')} | {dims('crop')} | {dims('optimized')} |\n"
|
||||
content+='\n### Наблюдаемые retry с jitter\n\n| После попытки | Сохранённая задержка до следующей, с |\n| --- | ---: |\n'
|
||||
for i,delay in enumerate(retry['delays_seconds'],1):content+=f'| {i} | {delay:.6f} |\n'
|
||||
content+=f"\nПосле {retry['max_attempts']}-й временной ошибки зафиксирован `retry_exhausted`; явный replay завершается ready. При проверке расписания next_at искусственно сдвигается, так что таблица показывает записанные интервалы, а не фактически выжданные паузы.\n\n"
|
||||
p=ROOT/'REPORT.md';s=p.read_text();a=s.index('<!-- BEGIN OPERATIONAL RESULTS -->');b=s.index('<!-- END OPERATIONAL RESULTS -->',a)
|
||||
p.write_text(s[:a]+'<!-- BEGIN OPERATIONAL RESULTS -->\n\n'+content+s[b:])
|
||||
print('Synchronized REPORT.md operational, geometry and retry tables')
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Generate research tables and static plots only from measured raw evidence."""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
os.environ.setdefault('MPLCONFIGDIR','/tmp/mtusi-lab5-matplotlib')
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
E=ROOT/'evidence'
|
||||
data=json.loads((E/'pipeline-faults.json').read_text())
|
||||
names=['before_effect','after_effect','after_commit','concurrent_duplicates','reorder_stale_version']
|
||||
labels=['Crash before PUT','Crash after PUT','Crash after COMMIT','8 duplicate jobs','Old job after v2']
|
||||
summary=[]
|
||||
for name in names:
|
||||
rows=[r for r in data['rows'] if r['scenario']==name]
|
||||
times=[r['recovery_ms'] for r in rows]
|
||||
summary.append(dict(scenario=name,repeats=len(rows),mean_ms=round(statistics.mean(times),3),
|
||||
min_ms=min(times),max_ms=max(times),stdev_ms=round(statistics.stdev(times),3),
|
||||
object_sets=1,objects_per_current_version=4,failures=sum(r['status']!='PASS' for r in rows)))
|
||||
with (E/'fault-summary.csv').open('w') as out:
|
||||
w=csv.DictWriter(out,fieldnames=list(summary[0]),lineterminator='\n');w.writeheader();w.writerows(summary)
|
||||
fig,axes=plt.subplots(1,2,figsize=(13,4.8))
|
||||
means=[r['mean_ms'] for r in summary]
|
||||
axes[0].bar(labels,means,color='#2878b5',yerr=[r['stdev_ms'] for r in summary],capsize=4)
|
||||
axes[0].set_ylabel('Scenario wall time, ms (mean +/- sample SD, n=3)')
|
||||
axes[0].tick_params(axis='x',rotation=25)
|
||||
axes[0].set_title('Process startup and accelerated lease recovery included')
|
||||
axes[1].bar(labels,[r['objects_per_current_version'] for r in summary],color='#32a77b')
|
||||
axes[1].axhline(4,color='black',linestyle='--',label='1 original + 3 variants')
|
||||
axes[1].set_ylim(0,5);axes[1].set_ylabel('Objects in current version prefix')
|
||||
axes[1].tick_params(axis='x',rotation=25);axes[1].set_title('No extra variant sets in tested scenarios');axes[1].legend()
|
||||
fig.tight_layout();fig.savefig(E/'pipeline-faults.png',dpi=160);fig.savefig(E/'pipeline-faults.svg');plt.close(fig)
|
||||
svg=E/'pipeline-faults.svg'
|
||||
svg.write_text('\n'.join(line.rstrip() for line in svg.read_text().splitlines())+'\n')
|
||||
table='| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |\n| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n'
|
||||
translations=['Падение до PUT','Падение после PUT','Падение после COMMIT до ack','8 конкурентных дублей','Старая версия после новой']
|
||||
for label,r in zip(translations,summary):
|
||||
table+=f"| {label} | {r['repeats']} | {r['mean_ms']} | {r['min_ms']}–{r['max_ms']} | {r['stdev_ms']} | {r['objects_per_current_version']} | {r['failures']} |\n"
|
||||
text='''# Исследование: уровень 5, трек 2
|
||||
|
||||
## Гипотеза и инварианты
|
||||
|
||||
Версионирование результатов, transactional outbox, блокировки PostgreSQL и повторяемые ключи S3 позволяют завершать принятые задачи при at-least-once доставке без дополнительных наборов вариантов и перезаписи нового результата старым.
|
||||
|
||||
Инварианты проверяются assertions: каждый проверяемый upload с 202 завершается ready; оригинал побайтно совпадает с принятым; у текущей версии ровно четыре объекта; метаданные и SHA-256 каждого варианта соответствуют скачанным байтам; повтор ready не меняет метаданные; устаревшая job не изменяет новую версию.
|
||||
|
||||
## Методика
|
||||
|
||||
`make research` останавливает обычные worker/publisher и запускает `tests/pipeline_faults.py` в ops. Создаются реальные HTTP-вложения и настоящие consumer-group доставки Valkey Streams. В отдельном процессе вызывается `os._exit(86)` до PUT, после всех PUT до COMMIT, после COMMIT до XACK. Неподтверждённая доставка затем захватывается другим consumer. Матрица содержит три повтора каждого из пяти основных сценариев, дополнительно проверяет смену уже опубликованной v1 на v2, failed/replay, пять временных ошибок и повтор публикации outbox.
|
||||
|
||||
Для ускорения матрицы lease переводится в прошлое SQL-командой; до этого отдельно проверяется отказ захвата действующего lease. Это не измерение реального времени ожидания lease. `tests/lifecycle.py` отдельно проверяет восстановление с настоящими часами, без изменения дедлайна. Конкурентные дубли запускаются восемью потоками, обычный worker последовательный. Время в таблице включает HTTP/БД, запуск Python-процессов и обработку; это время полного тестового сценария, а не latency одного изображения в production.
|
||||
|
||||
Один локальный Linux Docker Engine в Colima, arm64, 4 vCPU, около 8 GiB RAM. Лимиты: worker 1 CPU/768 MiB, API 2 реплики по 2 CPU/512 MiB, PostgreSQL 1 CPU/512 MiB, MinIO 0.5 CPU/768 MiB, broker 0.25 CPU/192 MiB; ops 1 CPU/512 MiB. Ограничения CPU — верхние пределы, а не выделенные ядра. Прочие лабораторные могут работать в этой же VM. Файл матрицы — PNG 320×200 из публичных fixtures. Отдельные геометрические тесты используют PNG/JPEG от 80×40 до 1600×800. Нагрузочного прогрева нет: исследуется корректность конкретных interleavings, не максимальный throughput. Точные лимиты и образы сохранены в `evidence/containers.json` и `compose.yaml`.
|
||||
|
||||
## Результаты матрицы
|
||||
|
||||
'''+table+'''
|
||||

|
||||
|
||||
После падения до COMMIT требуются две управляемые попытки. После COMMIT повтор не выполняет обработку: достаточно одной попытки. При восьми конкурентных доставках обрабатывает один поток, остальные видят занятость или завершённый результат. После новой версии старые задания не создают объекты v1 и не изменяют публичные метаданные v2. Отдельно проверено обновление уже существующего результата v1 на v2 с изменённым качеством кодирования.
|
||||
|
||||
Raw: [pipeline-faults.json](evidence/pipeline-faults.json), [CSV](evidence/fault-summary.csv), [lifecycle.json](evidence/lifecycle.json), [backpressure.json](evidence/backpressure.json), [cleanup-and-limits.json](evidence/cleanup-and-limits.json).
|
||||
|
||||
## Ограничения
|
||||
|
||||
Это проверка конечного набора аварийных расписаний, а не математическое доказательство для всех interleavings. Исследование не покрывает потерю диска/volume, несколько узлов, длительные сетевые разделения и повреждение файловой системы. Локальная матрица не подтверждает SLA и не измеряет p95 под производственной нагрузкой; для выбранного трека это не заявляется. CPU/RAM ограничены конфигурацией, пиковый RSS декодера в этом исследовании не измеряется.
|
||||
|
||||
Физический PUT может повториться после аварии: одинаковый ключ заменяется теми же данными. Гарантируется один текущий логический набор результатов, а доставка остаётся at least once. Объекты прежних успешно опубликованных версий сохраняются: это история версий, не дубли текущего результата. Изменение алгоритма должно сопровождаться увеличением pipeline_version; старый бинарный worker нужно вывести из эксплуатации перед изменением кода существующей версии.
|
||||
'''
|
||||
(ROOT/'RESEARCH.md').write_text(text)
|
||||
report=ROOT/'REPORT.md'
|
||||
if report.exists() and '<!-- BEGIN RESEARCH RESULTS -->' in report.read_text():
|
||||
content=report.read_text();a=content.index('<!-- BEGIN RESEARCH RESULTS -->');b=content.index('<!-- END RESEARCH RESULTS -->',a)
|
||||
content=content[:a]+'<!-- BEGIN RESEARCH RESULTS -->\n\n'+table+'\n\n\n'+content[b:]
|
||||
report.write_text(content)
|
||||
print('Generated RESEARCH.md, CSV, PNG/SVG and synchronized REPORT.md research table')
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Check that the persistence hook actually restarted every running service."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]/'evidence'
|
||||
before={c['name']:c for c in json.loads((root/'persistence-before.json').read_text()) if c['status']=='running'}
|
||||
after={c['name']:c for c in json.loads((root/'persistence-after.json').read_text())}
|
||||
rows=[]
|
||||
for name,old in before.items():
|
||||
new=after[name]
|
||||
assert old['id']==new['id'],name+': container unexpectedly replaced'
|
||||
assert old['started_at']!=new['started_at'],name+': process not restarted'
|
||||
assert old['volumes']==new['volumes'],name+': volumes changed'
|
||||
assert new['status']=='running',name+': not running'
|
||||
rows.append(dict(name=name,service=new['service'],same_container=True,process_restarted=True,same_volumes=True))
|
||||
print(json.dumps({'status':'PASS','running_processes_restarted':len(rows),'rows':rows},indent=2))
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for actual healthy Compose API containers and their public LB aliases."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',default='http://localhost:8085')
|
||||
parser.add_argument('--replicas',type=int,default=2)
|
||||
args=parser.parse_args()
|
||||
deadline=time.monotonic()+90
|
||||
while time.monotonic()<deadline:
|
||||
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-q','api'],cwd=ROOT,text=True).split()
|
||||
if len(ids)==args.replicas:
|
||||
state=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
|
||||
if all(c['State'].get('Health',{}).get('Status')=='healthy' for c in state):
|
||||
expected={c['Config']['Hostname'] for c in state}
|
||||
try:
|
||||
observed=set()
|
||||
for _ in range(16):
|
||||
with urllib.request.urlopen(args.base_url+'/health/ready',timeout=2) as response:
|
||||
observed.add(response.headers.get('X-Instance-Id'))
|
||||
if observed==expected:
|
||||
print('PASS: healthy API replicas observed through LB')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise SystemExit('API readiness/LB discovery exceeded 90 seconds')
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Real broker outage and bounded admission, followed by complete recovery."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from lifecycle import compose,probe
|
||||
from client import Client,check
|
||||
|
||||
|
||||
def main():
|
||||
c=Client('http://localhost:8085');started=time.monotonic()
|
||||
try:
|
||||
compose('stop','worker','publisher','broker')
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
# Default deployment admits at most 100 pending logical jobs across APIs.
|
||||
def upload(_):return Client(c.base).attachment(alice,chat)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:accepted=list(pool.map(upload,range(100)))
|
||||
fixture=accepted[0][1]
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=full.png',429,actor=alice,raw=fixture,headers={'Content-Type':'image/png'})
|
||||
c.message(alice,chat,'messages remain writable while image queue is full')
|
||||
for att,_ in (accepted[0],accepted[-1]):
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
|
||||
admitted_seconds=time.monotonic()-started
|
||||
# Every 202 was accepted while the broker was genuinely stopped.
|
||||
state=probe(accepted[0][0]['id'])
|
||||
check(state['published_at'] is None and state['state']=='queued','upload depended on unavailable broker')
|
||||
compose('start','broker','publisher','worker');recovery=time.monotonic()
|
||||
for att,data in accepted:
|
||||
ready=c.wait_ready(bob,att,wait_seconds=120)
|
||||
check(len(ready['variants'])==3 and ready['sha256']==hashlib.sha256(data).hexdigest(),'recovery lost accepted metadata')
|
||||
for att,data in (accepted[0],accepted[-1]):
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'recovery lost original bytes')
|
||||
print(json.dumps({'status':'PASS','accepted_while_broker_down':len(accepted),'next_upload_status':429,
|
||||
'messages_available':True,'all_ready':len(accepted),'admission_seconds':round(admitted_seconds,3),
|
||||
'drain_seconds':round(time.monotonic()-recovery,3),'lost_accepted':0,
|
||||
'profile':'100 PNG 320x200, 8 upload clients, 1 serial worker; closed-loop; not a throughput benchmark'},indent=2))
|
||||
finally:compose('start','broker','publisher','worker')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Research track: cache consistency and stale session protection for Valkey-backed sessions."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080")
|
||||
CACHER_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0")
|
||||
|
||||
|
||||
def http_json(method, path, *, expected=200, headers=None, body=None):
|
||||
req_headers = {"Accept": "application/json"}
|
||||
if headers:
|
||||
req_headers.update(headers)
|
||||
data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
if data is not None:
|
||||
req_headers.setdefault("Content-Type", "application/json")
|
||||
request = urllib.request.Request(BASE_URL + path, data=data, headers=req_headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as resp:
|
||||
payload = resp.read()
|
||||
if not payload:
|
||||
return {"status": resp.status, "headers": dict(resp.headers), "body": None}
|
||||
if resp.status not in (expected if isinstance(expected, (list, tuple, set)) else [expected]):
|
||||
raise AssertionError(f"Expected {expected}, got {resp.status}")
|
||||
return {"status": resp.status, "headers": dict(resp.headers), "body": json.loads(payload.decode("utf-8"))}
|
||||
except urllib.error.HTTPError as exc:
|
||||
payload = exc.read()
|
||||
if payload:
|
||||
try:
|
||||
payload = json.loads(payload.decode("utf-8"))
|
||||
except Exception:
|
||||
payload = payload.decode("utf-8", errors="replace")
|
||||
if exc.code not in (expected if isinstance(expected, (list, tuple, set)) else [expected]):
|
||||
raise AssertionError(f"Expected status {expected}, got {exc.code} for {method} {path}: {payload}")
|
||||
return {"status": exc.code, "headers": dict(exc.headers), "body": payload}
|
||||
|
||||
|
||||
def call_with_latency(method, path, *, expected=200, headers=None, body=None):
|
||||
start = time.perf_counter()
|
||||
result = http_json(method, path, expected=expected, headers=headers, body=body)
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000.0
|
||||
return result, elapsed_ms
|
||||
|
||||
|
||||
def main():
|
||||
import redis
|
||||
|
||||
r = redis.Redis.from_url(CACHER_URL, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
|
||||
|
||||
username = f"cache_{uuid.uuid4().hex[:12]}"
|
||||
password = "123456789012"
|
||||
body = {
|
||||
"username": username,
|
||||
"display_name": "Cache Consistency",
|
||||
"password": password,
|
||||
}
|
||||
registration = http_json("POST", "/api/v1/users", expected=201, body=body)
|
||||
assert registration["status"] == 201, registration
|
||||
|
||||
basic = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
session = http_json("POST", "/api/v1/auth/sessions", expected=201, headers={"Authorization": basic})
|
||||
token = session["body"]["token"]
|
||||
assert token, session
|
||||
|
||||
key = f"session:{token}"
|
||||
assert r.get(key), "session was not populated in Valkey"
|
||||
ttl = r.ttl(key)
|
||||
assert ttl and ttl > 0, f"session TTL must be positive, got {ttl}"
|
||||
|
||||
latencies = []
|
||||
for _ in range(10):
|
||||
_, elapsed_ms = call_with_latency("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
|
||||
latencies.append(elapsed_ms)
|
||||
avg = sum(latencies) / len(latencies)
|
||||
p95 = sorted(latencies)[int(len(latencies) * 0.95) - 1]
|
||||
|
||||
stale_payload = {
|
||||
"token": token,
|
||||
"user_id": str(uuid.uuid4()),
|
||||
"expires_at": "2000-01-01T00:00:00Z",
|
||||
"revoked_at": None,
|
||||
"user": {"id": str(uuid.uuid4()), "username": "stale", "display_name": "stale", "created_at": "2020-01-01T00:00:00Z"},
|
||||
}
|
||||
r.setex(key, 120, json.dumps(stale_payload, separators=(",", ":"), ensure_ascii=False))
|
||||
stale_result = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
|
||||
assert stale_result["status"] == 200, stale_result
|
||||
assert stale_result["body"]["username"] == username, stale_result
|
||||
|
||||
# An unexpired payload must not be able to substitute another identity.
|
||||
stale_payload["expires_at"] = "2100-01-01T00:00:00Z"
|
||||
r.setex(key, 120, json.dumps(stale_payload))
|
||||
forged = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})
|
||||
assert forged["body"]["id"] == session["body"]["user"]["id"], forged
|
||||
assert forged["body"]["username"] == username, forged
|
||||
|
||||
r.setex(key, 120, json.dumps(stale_payload))
|
||||
def read_identity(_):
|
||||
return http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})["body"]["id"]
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
identities = list(executor.map(read_identity, range(8)))
|
||||
assert identities == [session["body"]["user"]["id"]] * 8, identities
|
||||
|
||||
# Syntactically valid JSON with missing fields must be treated as a miss.
|
||||
r.setex(key, 120, "{}")
|
||||
malformed = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})
|
||||
assert malformed["body"]["username"] == username, malformed
|
||||
|
||||
r.setex(key, 1, json.dumps({
|
||||
"token": token,
|
||||
"user_id": str(uuid.uuid4()),
|
||||
"expires_at": "2100-01-01T00:00:00Z",
|
||||
"revoked_at": None,
|
||||
"user": {"id": str(uuid.uuid4()), "username": "ttl-recovery", "display_name": "ttl-recovery", "created_at": "2020-01-01T00:00:00Z"},
|
||||
}, separators=(",", ":"), ensure_ascii=False))
|
||||
time.sleep(2)
|
||||
short_ttl = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
|
||||
assert short_ttl["status"] == 200, short_ttl
|
||||
assert short_ttl["body"]["username"] == username, short_ttl
|
||||
|
||||
revoke = http_json("DELETE", "/api/v1/auth/sessions/current", expected=204, headers={"Authorization": "Bearer " + token})
|
||||
assert revoke["status"] == 204, revoke
|
||||
assert not r.get(key), "revoked session key should be deleted from cache"
|
||||
|
||||
after_logout = http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token})
|
||||
assert after_logout["status"] == 401, after_logout
|
||||
|
||||
# Simulate a writer repopulating the cache after logout's DELETE.
|
||||
stale_payload.update(session["body"])
|
||||
stale_payload["user_id"] = session["body"]["user"]["id"]
|
||||
stale_payload["revoked_at"] = None
|
||||
r.setex(key, 120, json.dumps(stale_payload))
|
||||
http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token})
|
||||
assert not r.exists(key), "stale key after logout must be removed"
|
||||
|
||||
print("cache_key_exists=True")
|
||||
print(f"ttl_seconds={ttl}")
|
||||
print(f"avg_latency_ms={avg:.2f}")
|
||||
print(f"p95_latency_ms={p95:.2f}")
|
||||
print("stale_cache_rejected=True")
|
||||
print("forged_identity_rejected=True")
|
||||
print("concurrent_identity_checks=8/8")
|
||||
print("malformed_cache_recovers=True")
|
||||
print("repopulated_revoked_cache_rejected=True")
|
||||
print("short_ttl_recovers_from_db=True")
|
||||
print("logout_revokes_cache=True")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Conservative cleanup and actual subprocess timeout boundary; worker stopped."""
|
||||
from datetime import datetime,timedelta,timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest.mock as mock
|
||||
import uuid
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from client import Client,check
|
||||
from app import store,object_store
|
||||
from pipeline import envelope,fetch_job,replay
|
||||
from worker import handle
|
||||
from janitor import sweep
|
||||
|
||||
|
||||
def main():
|
||||
store.pool.open(wait=True,timeout=20)
|
||||
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap()
|
||||
def upload():
|
||||
att,data=c.attachment(alice,chat)
|
||||
with store.connect() as conn:
|
||||
jid=conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0]
|
||||
msg=envelope(fetch_job(conn,jid))
|
||||
return att,msg
|
||||
rows=[]
|
||||
try:
|
||||
att,msg=upload()
|
||||
real_run=subprocess.run
|
||||
def slow_decoder(*args,**kwargs):
|
||||
return real_run([sys.executable,'-c','import time; time.sleep(2)'],timeout=.05,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
start=time.monotonic()
|
||||
with mock.patch('worker.subprocess.run',side_effect=slow_decoder):
|
||||
check(handle(msg)=='retry','decoder timeout not classified as transient')
|
||||
elapsed=time.monotonic()-start
|
||||
with store.connect() as conn:
|
||||
job=fetch_job(conn,msg['job_id']);check(job['state']=='queued' and job['error_code']=='processing_timeout','timeout not persisted')
|
||||
conn.execute('UPDATE jobs SET next_at=now() WHERE id=%s',(job['id'],))
|
||||
check(handle(msg)=='ready','timeout recovery failed')
|
||||
rows.append(dict(case='real_child_timeout_with_test_budget_50ms',status='PASS',elapsed_ms=round(elapsed*1000,3)))
|
||||
# Validate memory-exit classification without risking an OOM on the host.
|
||||
att2,msg2=upload()
|
||||
def exhausted(*args,**kwargs):
|
||||
return real_run([sys.executable,'-c',"import resource; resource.setrlimit(resource.RLIMIT_AS,(32*1024**2,32*1024**2));\ntry: x=bytearray(128*1024**2)\nexcept MemoryError: raise SystemExit(3)"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
with mock.patch('worker.subprocess.run',side_effect=exhausted):
|
||||
check(handle(msg2)=='failed','memory failure not terminal')
|
||||
with store.connect() as conn:check(fetch_job(conn,msg2['job_id'])['error_code']=='resource_limit','memory failure leaked details')
|
||||
replay(store,msg2['job_id']);check(handle(msg2)=='ready','memory-failed replay did not complete')
|
||||
rows.append(dict(case='real_child_address_space_limit_32MiB',status='PASS',error_code='resource_limit'))
|
||||
active,msg3=upload()
|
||||
orphan='staging/'+str(uuid.uuid4())
|
||||
object_store.client.put_object(Bucket=object_store.bucket,Key=orphan,Body=b'orphan')
|
||||
# Advance janitor's clock, leaving actual DB state/worker leases intact.
|
||||
class Future(datetime):
|
||||
@classmethod
|
||||
def now(cls,tz=None):return datetime.now(tz)+timedelta(days=2)
|
||||
with mock.patch('janitor.datetime',Future):
|
||||
dry=sweep(False,86400)
|
||||
check(orphan in dry['staging_candidates'],'orphan not detected')
|
||||
check('staging/'+active['id'] not in dry['staging_candidates'],'active source scheduled for deletion')
|
||||
object_store.client.head_object(Bucket=object_store.bucket,Key=orphan)
|
||||
applied=sweep(True,86400)
|
||||
check(orphan in applied['staging_candidates'],'orphan not deleted')
|
||||
repeat=sweep(True,86400)
|
||||
check(not repeat['staging_candidates'],'cleanup is not idempotent')
|
||||
check(handle(msg3)=='ready','cleanup lost current job')
|
||||
rows.append(dict(case='staging_dry_run_apply_repeat',status='PASS',candidates=len(applied['staging_candidates']),
|
||||
repeat_candidates=0,active_retained=True,clock='advanced_2_days'))
|
||||
print(json.dumps({'status':'PASS','rows':rows},indent=2))
|
||||
finally:store.pool.close();object_store.close()
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Isolated crashable consumer using a real Streams consumer-group delivery."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from app import store
|
||||
from pipeline import broker,envelope,fetch_job,STREAM
|
||||
from worker import handle
|
||||
import redis
|
||||
|
||||
p=argparse.ArgumentParser();p.add_argument('job_id');p.add_argument('--crash',choices=['before_effect','after_effect','after_commit']);p.add_argument('--recover',action='store_true');args=p.parse_args()
|
||||
store.pool.open(wait=True,timeout=20)
|
||||
r=broker();group='fault-probes'
|
||||
try:
|
||||
try:r.xgroup_create(STREAM,group,id='$',mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if 'BUSYGROUP' not in str(exc):raise
|
||||
with store.connect() as db:msg=envelope(fetch_job(db,args.job_id))
|
||||
if args.recover:
|
||||
entries=r.xautoclaim(STREAM,group,'recovered',min_idle_time=0,start_id='0-0',count=100)[1]
|
||||
target=next(((mid,fields) for mid,fields in entries if json.loads(fields['job'])['job_id']==args.job_id),None)
|
||||
if target is None:raise AssertionError('expected unacknowledged real broker delivery')
|
||||
mid,fields=target;msg=json.loads(fields['job'])
|
||||
else:
|
||||
target_id=r.xadd(STREAM,{'job':json.dumps(msg)})
|
||||
while True:
|
||||
entries=r.xreadgroup(group,'crashable',{STREAM:'>'},count=100,block=1000)
|
||||
target=next(((mid,fields) for _,batch in entries for mid,fields in batch if mid==target_id),None)
|
||||
if target:mid,_=target;break
|
||||
outcome=handle(msg,crash=args.crash)
|
||||
if outcome!='defer':r.xack(STREAM,group,mid)
|
||||
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
|
||||
finally:
|
||||
r.close();store.pool.close()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Concurrent shared-key requests through LB, real instance headers and history."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
c = Client(args.base_url)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(16)
|
||||
|
||||
def worker(_):
|
||||
local = Client(args.base_url)
|
||||
barrier.wait()
|
||||
return local.message(alice, chat, ' concurrent body ', key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
responses = list(pool.map(worker, range(16)))
|
||||
ids = {r['json']['id'] for r in responses}
|
||||
instances = {r['headers']['X-Instance-Id'] for r in responses}
|
||||
check(len(ids) == 1, 'concurrent duplicate messages')
|
||||
check(len(instances) >= 2, 'concurrent requests did not reach two replicas')
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'concurrent body'}, headers={'Idempotency-Key': key})
|
||||
again = c.call('POST', path, 201, actor=alice, body={'attachment_ids': [], 'text': ' concurrent body '}, headers={'Idempotency-Key': key}, schema='Message')
|
||||
check(again['json'] == responses[0]['json'], 'semantic replay differs')
|
||||
page = c.call('GET', path, actor=bob, schema='MessagePage')['json']
|
||||
check(len(page['items']) == 1, 'history contains duplicate effects')
|
||||
c.message(bob, chat, 'other user', key=key)
|
||||
print(json.dumps({'parallel_requests': 16, 'observed_instances': sorted(instances), 'unique_messages': len(ids), 'conflict': 409, 'semantic_replay': True, 'user_scope': True}))
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Actual variant geometry, orientation, alpha, encoder metadata and hashes."""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from PIL import Image
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
from client import Client,check
|
||||
|
||||
|
||||
def main():
|
||||
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();rows=[]
|
||||
cases=[]
|
||||
cases.append(('small_alpha',Image.new('RGBA',(80,40),(0,0,0,0)),'PNG',{},(80,40)))
|
||||
cases.append(('exif_orientation',Image.new('RGB',(400,100),'red'),'JPEG',{'exif':Image.Exif()},(64,256)))
|
||||
cases[-1][3]['exif'][274]=6
|
||||
center=Image.new('RGB',(400,100),'red');center.paste('blue',(150,0,250,100))
|
||||
cases.append(('center_crop',center,'PNG',{},(256,64)))
|
||||
cases.append(('large_fit',Image.new('RGB',(1600,800),'green'),'PNG',{},(256,128)))
|
||||
for name,image,fmt,options,expected in cases:
|
||||
data=io.BytesIO();image.save(data,fmt,**options);payload=data.getvalue()
|
||||
mime='image/png' if fmt=='PNG' else 'image/jpeg'
|
||||
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,
|
||||
raw=payload,headers={'Content-Type':mime},schema='Attachment')['json']
|
||||
ready=c.wait_ready(alice,att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==payload,'original changed')
|
||||
dimensions={}
|
||||
for variant in ready['variants']:
|
||||
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+variant['name'],actor=bob)
|
||||
check(hashlib.sha256(result['bytes']).hexdigest()==variant['sha256'],'false checksum')
|
||||
check(len(result['bytes'])==variant['size_bytes'],'false variant size')
|
||||
with Image.open(io.BytesIO(result['bytes'])) as output:
|
||||
check(not output.getexif(),'EXIF leaked')
|
||||
check(output.size==(variant['width'],variant['height']),'false dimensions')
|
||||
dimensions[variant['name']]=output.size
|
||||
check(output.format==('WEBP' if variant['name']=='optimized' else 'JPEG'),'wrong encoder')
|
||||
if variant['name']=='thumbnail':
|
||||
check(output.size==expected,'thumbnail orientation/scale incorrect')
|
||||
if name=='small_alpha':check(min(output.getpixel((10,10)))>245,'alpha not composited on white')
|
||||
if variant['name']=='crop':
|
||||
check(output.size==(128,128),'crop dimensions wrong')
|
||||
if name=='center_crop':check(output.getpixel((64,64))[2]>240,'crop not centered')
|
||||
if variant['name']=='optimized':
|
||||
check(max(output.size)<=1280,'optimized too large')
|
||||
if name=='large_fit':check(output.size==(1280,640),'optimized ratio wrong')
|
||||
rows.append(dict(case=name,status='PASS',dimensions=dimensions,original_sha256=hashlib.sha256(payload).hexdigest()))
|
||||
print(json.dumps({'status':'PASS','rows':rows},indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Host orchestration: stopped worker, API kill, broker restart, real lease."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
from client import Client,check
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
COMPOSE=[str(ROOT/'scripts/compose.sh')]
|
||||
|
||||
|
||||
def compose(*args,allow_failure=False):
|
||||
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,text=True,capture_output=True)
|
||||
if p.returncode and not allow_failure:
|
||||
raise AssertionError('Compose action failed: '+' '.join(args[:3])+': '+p.stderr[-500:])
|
||||
return p
|
||||
|
||||
|
||||
def probe(aid):
|
||||
return json.loads(compose('run','--rm','--no-deps','ops','python','scripts/query_pipeline.py',aid).stdout)[-1]
|
||||
|
||||
|
||||
def wait_api(c):
|
||||
deadline=time.monotonic()+60
|
||||
while time.monotonic()<deadline:
|
||||
try:c.call('GET','/health/ready',schema='Health');return
|
||||
except Exception:time.sleep(.3)
|
||||
raise AssertionError('API recovery timeout')
|
||||
|
||||
|
||||
def snapshot(services):
|
||||
ids=compose('ps','-q',*services).stdout.split()
|
||||
state=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
|
||||
return {x['Name'].lstrip('/'):{'id':x['Id'][:12],'started_at':x['State']['StartedAt'],
|
||||
'volumes':sorted(m['Name'] for m in x['Mounts'] if m['Type']=='volume')} for x in state}
|
||||
|
||||
|
||||
def main():
|
||||
c=Client(os.environ.get('BASE_URL','http://localhost:8085'))
|
||||
rows=[]
|
||||
try:
|
||||
compose('stop','worker','publisher')
|
||||
alice,bob,eve,chat=c.bootstrap();att,data=c.attachment(alice,chat)
|
||||
message=c.message(alice,chat,'',attachments=[att['id']])['json']
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
|
||||
check(probe(att['id'])['published_at'] is None,'publisher unexpectedly active')
|
||||
before=snapshot(['api','broker'])
|
||||
compose('kill','-s','SIGKILL','api');compose('start','api');wait_api(c)
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
page=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage')['json']
|
||||
check(any(m['id']==message['id'] for m in page['items']),'queued message lost')
|
||||
compose('start','publisher')
|
||||
deadline=time.monotonic()+20
|
||||
while probe(att['id'])['published_at'] is None:
|
||||
check(time.monotonic()<deadline,'outbox not delivered');time.sleep(.2)
|
||||
length_before=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
|
||||
compose('restart','broker')
|
||||
length_after=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
|
||||
check(length_after>=length_before>0,'broker lost durable messages')
|
||||
after=snapshot(['api','broker'])
|
||||
check(all(after[name]['started_at']!=row['started_at'] and after[name]['volumes']==row['volumes'] for name,row in before.items()),'restart did not preserve volumes')
|
||||
start=time.monotonic();compose('start','worker');c.wait_ready(bob,att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'accepted original lost')
|
||||
rows.append(dict(scenario='stopped_worker_api_kill_outbox_broker_restart',status='PASS',attachment_id=att['id'],
|
||||
stream_length_before=length_before,stream_length_after=length_after,ready_ms=round((time.monotonic()-start)*1000,3),
|
||||
containers_before=before,containers_after=after))
|
||||
compose('stop','worker','publisher')
|
||||
att,data=c.attachment(alice,chat);jid=probe(att['id'])['job_id']
|
||||
crash=compose('run','--rm','--no-deps','ops','python','tests/drive_job.py',jid,'--crash','after_effect',allow_failure=True)
|
||||
check(crash.returncode==86,'worker failpoint did not crash')
|
||||
check(probe(att['id'])['state']=='processing','processing lease missing')
|
||||
start=time.monotonic();compose('start','publisher','worker');c.wait_ready(bob,att)
|
||||
elapsed=time.monotonic()-start
|
||||
check(elapsed>=20,'real lease expiry not exercised')
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'lease recovery corrupted original')
|
||||
rows.append(dict(scenario='crash_real_lease_expiry',status='PASS',attachment_id=att['id'],ready_ms=round(elapsed*1000,3),
|
||||
job=probe(att['id']),lease_clock='real'))
|
||||
print(json.dumps({'status':'PASS','rows':rows},ensure_ascii=False,indent=2))
|
||||
finally:compose('start','api','broker','publisher','worker')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Track 2: real crash processes, broker redelivery, version fencing and retries.
|
||||
Run only with the normal worker and publisher stopped. Creates synthetic rows.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from PIL import Image
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from client import Client,check,validate
|
||||
from app import store,object_store
|
||||
from pipeline import broker,envelope,fetch_job,MAX_ATTEMPTS,replay,upgrade,STREAM
|
||||
from worker import handle
|
||||
from publisher import publish_once
|
||||
|
||||
|
||||
def main():
|
||||
store.pool.open(wait=True,timeout=20)
|
||||
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();r=broker()
|
||||
rows=[]
|
||||
def upload():
|
||||
att,data=c.attachment(alice,chat)
|
||||
with store.connect() as conn:
|
||||
jid=str(conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0])
|
||||
msg=envelope(fetch_job(conn,jid))
|
||||
validate(msg,json.loads((Path(__file__).resolve().parents[1]/'contracts/jobs.schema.json').read_text()))
|
||||
return att,data,jid,msg
|
||||
def metadata(att):return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
|
||||
def objects(att,version=1):
|
||||
return sorted(x['Key'] for x in object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix=f"results/{att['id']}/v{version}/").get('Contents',[]))
|
||||
def invariant(att,data,version=1):
|
||||
ready=metadata(att);check(ready['status']=='ready','lost accepted work')
|
||||
check(len(ready['variants'])==3,'wrong number of variants')
|
||||
check(len(objects(att,version))==4,'duplicate or missing objects')
|
||||
for name in ('original','thumbnail','crop','optimized'):
|
||||
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=bob)
|
||||
if name=='original':check(result['bytes']==data,'original mutated')
|
||||
else:
|
||||
v=next(x for x in ready['variants'] if x['name']==name)
|
||||
check(hashlib.sha256(result['bytes']).hexdigest()==v['sha256'],'variant checksum mismatch')
|
||||
with Image.open(io.BytesIO(result['bytes'])) as im:
|
||||
check((im.width,im.height)==(v['width'],v['height']),'false dimensions')
|
||||
check(not im.getexif(),'EXIF not stripped')
|
||||
return ready
|
||||
try:
|
||||
for iteration in range(3):
|
||||
for crash in ('before_effect','after_effect','after_commit'):
|
||||
att,data,jid,msg=upload();start=time.monotonic()
|
||||
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--crash',crash],capture_output=True,text=True)
|
||||
check(result.returncode==86,'failpoint did not kill the worker process')
|
||||
before=objects(att)
|
||||
check(len(before)==(0 if crash=='before_effect' else 4),'wrong crash window')
|
||||
with store.connect() as conn:
|
||||
j=fetch_job(conn,jid)
|
||||
if crash!='after_commit':
|
||||
check(j['state']=='processing','lease not persisted')
|
||||
check(handle(msg)=='defer','live lease was stolen')
|
||||
# Controlled clock acceleration for matrix speed. Separate
|
||||
# persistence test exercises real 30-second lease timeout.
|
||||
conn.execute("UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=%s",(jid,))
|
||||
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--recover'],capture_output=True,text=True)
|
||||
check(result.returncode==0,'redelivery process failed')
|
||||
check(json.loads(result.stdout)['outcome'] in ('ready','noop'),'redelivery did not complete')
|
||||
ready=invariant(att,data)
|
||||
# Successful duplicate must not update metadata or re-encode.
|
||||
check(handle(msg)=='noop','ready duplicate was re-executed')
|
||||
check(metadata(att)==ready,'ready changed after duplicate')
|
||||
with store.connect() as conn:attempts=fetch_job(conn,jid)['attempt']
|
||||
rows.append(dict(iteration=iteration+1,scenario=crash,status='PASS',attempts=attempts,
|
||||
recovery_ms=round((time.monotonic()-start)*1000,3),objects=4,lease_expiry='accelerated' if crash!='after_commit' else 'not_needed'))
|
||||
att,data,jid,msg=upload();start=time.monotonic()
|
||||
outcomes=[]
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
outcomes=list(pool.map(lambda _:handle(msg),range(8)))
|
||||
invariant(att,data)
|
||||
check(outcomes.count('ready')==1,'concurrent duplicate produced multiple effects')
|
||||
rows.append(dict(iteration=iteration+1,scenario='concurrent_duplicates',status='PASS',deliveries=8,
|
||||
effects=1,recovery_ms=round((time.monotonic()-start)*1000,3),outcomes=outcomes,objects=4))
|
||||
# New version overtakes an unprocessed old version.
|
||||
att,data,jid,old=upload();new_jid=upgrade(store,att['id'])
|
||||
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
|
||||
start=time.monotonic();check(handle(new)=='ready','new version failed')
|
||||
newest=invariant(att,data,2)
|
||||
for _ in range(5):check(handle(old)=='noop','stale job not fenced')
|
||||
check(metadata(att)==newest and not objects(att,1),'old version overwrote new')
|
||||
rows.append(dict(iteration=iteration+1,scenario='reorder_stale_version',status='PASS',old_deliveries=5,
|
||||
current_version=2,old_objects=0,objects=4,recovery_ms=round((time.monotonic()-start)*1000,3)))
|
||||
# Upgrade an already published result, then redeliver the old version.
|
||||
att,data,jid,old=upload();check(handle(old)=='ready','v1 failed')
|
||||
v1=invariant(att,data)
|
||||
new_jid=upgrade(store,att['id'])
|
||||
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
|
||||
check(handle(new)=='ready','upgrade failed');v2=invariant(att,data,2)
|
||||
check(v1['variants']!=v2['variants'],'version settings did not change result')
|
||||
check(handle(old)=='noop' and metadata(att)==v2,'published old version replaced upgraded result')
|
||||
rows.append(dict(scenario='published_v1_then_v2_then_stale_v1',status='PASS',current_version=2,objects=4))
|
||||
# Permanent errors, explicit audited replay, ready replay refused.
|
||||
att,data,jid,msg=upload();check(handle(msg,force_error='permanent')=='failed','permanent failure retried')
|
||||
check(metadata(att)['status']=='failed','failed not exposed')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-test'})
|
||||
replay(store,jid);check(handle(msg)=='ready','operator replay did not recover')
|
||||
invariant(att,data)
|
||||
try:replay(store,jid);raise AssertionError('ready replay allowed')
|
||||
except ValueError:pass
|
||||
rows.append(dict(scenario='permanent_failed_replay',status='PASS',objects=4))
|
||||
# Transient retry scheduling respects backoff and exhausts the bound.
|
||||
att,data,jid,msg=upload();delays=[]
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
check(handle(msg,force_error='temporary')=='retry','temporary failure not handled')
|
||||
with store.connect() as conn:
|
||||
j=fetch_job(conn,jid)
|
||||
if attempt<MAX_ATTEMPTS-1:
|
||||
delay=conn.execute('SELECT extract(epoch FROM next_at-now()) FROM jobs WHERE id=%s',(jid,)).fetchone()[0]
|
||||
check(delay>0,'no backoff');delays.append(float(delay))
|
||||
check(handle(msg)=='defer','retry ignored schedule')
|
||||
conn.execute("UPDATE jobs SET next_at=now()-interval '1 second' WHERE id=%s",(jid,))
|
||||
check(metadata(att)['error_code']=='retry_exhausted','retry was not bounded')
|
||||
replay(store,jid);check(handle(msg)=='ready','retry exhausted replay failed');invariant(att,data)
|
||||
rows.append(dict(scenario='bounded_retry_jitter',status='PASS',max_attempts=MAX_ATTEMPTS,delays_seconds=delays))
|
||||
# COMMIT -> publish recovery, and a lost publish confirm duplicates safely.
|
||||
att,data,jid,msg=upload()
|
||||
with store.connect() as conn:check(conn.execute('SELECT published_at FROM outbox WHERE job_id=%s',(jid,)).fetchone()[0] is None,'outbox already published')
|
||||
publish_once(r)
|
||||
with store.connect() as conn:
|
||||
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
|
||||
publish_once(r)
|
||||
check(handle(msg)=='ready' and handle(msg)=='noop','republish was unsafe');invariant(att,data)
|
||||
rows.append(dict(scenario='outbox_republish',status='PASS',objects=4))
|
||||
with store.connect() as conn:
|
||||
accepted=conn.execute('SELECT count(*) FROM attachments').fetchone()[0]
|
||||
result={'status':'PASS','track':2,'matrix_repeats':3,'rows':rows,'accepted_in_database':accepted,
|
||||
'invariants':{'accepted_tasks_lost':0,'duplicate_variant_sets':0,'stale_version_overwrites':0},
|
||||
'limitations':['matrix lease expiry accelerated with SQL','single host, worker processing is serial except intentional duplicate probe','no disk-loss or network partition proof']}
|
||||
print(json.dumps(result,ensure_ascii=False,indent=2))
|
||||
finally:r.close();store.pool.close();object_store.close()
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Anonymous access outside API and least-privilege negative checks."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from botocore.exceptions import ClientError
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from client import Client,check,run
|
||||
|
||||
|
||||
def main():
|
||||
from app import store,object_store
|
||||
c=Client('http://lb:8080',timeout=15)
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
att,_=c.attachment(alice,chat)
|
||||
c.wait_ready(alice,att)
|
||||
key='results/'+att['id']+'/v1/original'
|
||||
anonymous=[]
|
||||
for suffix in ('','/'+key):
|
||||
try:
|
||||
urllib.request.urlopen('http://s3:9000/'+object_store.bucket+suffix,timeout=3)
|
||||
raise AssertionError('private bucket allowed anonymous access')
|
||||
except urllib.error.HTTPError as exc:
|
||||
check(exc.code==403,'anonymous access failed for an unexpected reason')
|
||||
anonymous.append(exc.code)
|
||||
# MinIO returns a filtered ListBuckets response to a ListBucket principal.
|
||||
visible=[b['Name'] for b in object_store.client.list_buckets().get('Buckets',[])]
|
||||
check(set(visible)<={object_store.bucket},'unrelated bucket names exposed')
|
||||
denied=[]
|
||||
for name,operation in [('create_bucket',lambda:object_store.client.create_bucket(Bucket='must-not-create-lab4')),
|
||||
('get_bucket_policy',lambda:object_store.client.get_bucket_policy(Bucket=object_store.bucket)),
|
||||
('write_outside_prefix',lambda:object_store.client.put_object(Bucket=object_store.bucket,Key='outside-policy',Body=b'no'))]:
|
||||
try:
|
||||
operation()
|
||||
raise AssertionError('application credentials have excessive privileges: '+name)
|
||||
except ClientError as exc:
|
||||
check(exc.response['Error']['Code']=='AccessDenied','unexpected permission error')
|
||||
denied.append(name)
|
||||
# The console is bound to 127.0.0.1 inside S3 and browser UI is disabled.
|
||||
import socket
|
||||
try:
|
||||
socket.create_connection(('s3',9001),timeout=1).close()
|
||||
raise AssertionError('S3 console reachable from another container')
|
||||
except OSError:
|
||||
pass
|
||||
print(json.dumps({'status':'PASS','anonymous_bucket_and_object_status':anonymous,
|
||||
'application_admin_operations_denied':denied,'visible_bucket_names':visible,'console_reachable':False},indent=2))
|
||||
object_store.close()
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Contract boundaries and object authorization; runs in Compose ops network."""
|
||||
import hashlib
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import quote, urlsplit
|
||||
import uuid
|
||||
import zlib
|
||||
from PIL import Image
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from client import Client, check, run
|
||||
|
||||
|
||||
def main():
|
||||
c=Client('http://lb:8080',timeout=15)
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
path='/api/v1/chats/'+chat['id']+'/attachments'
|
||||
fixture=(Path(__file__).parent/'fixtures/sample.png').read_bytes()
|
||||
cases=[]
|
||||
def bad(label,status,data=fixture,mime='image/png',filename='test.png',actor=None):
|
||||
c.call('POST',path+'?filename='+quote(filename,safe=''),status,actor=actor or alice,raw=data,headers={'Content-Type':mime})
|
||||
cases.append({'case':label,'status':status})
|
||||
bad('corrupt_png',415,b'not an image')
|
||||
bad('truncated_png',415,fixture[:50])
|
||||
bad('mime_spoof',415,mime='image/jpeg')
|
||||
bad('unsupported_mime',415,mime='application/octet-stream')
|
||||
bad('empty_payload',415,b'')
|
||||
for filename in ('../secret.png','a\\b.png','x\x00.png','x\n.png','x\u202e.png','x'*256):
|
||||
bad('unsafe_filename',400,filename=filename)
|
||||
bad('foreign_chat_upload',404,actor=eve)
|
||||
c.call('POST',path+'?filename=x.png',401,raw=fixture,headers={'Content-Type':'image/png'})
|
||||
cases.append({'case':'unauthenticated_upload','status':401})
|
||||
# Oversized dimensions are rejected using headers before a full raster decode.
|
||||
data=bytearray(fixture)
|
||||
data[16:24]=struct.pack('>II',10000,4001)
|
||||
data[29:33]=struct.pack('>I',zlib.crc32(data[12:29])&0xffffffff)
|
||||
bad('over_40_million_pixels',413,bytes(data))
|
||||
# No large allocation or body transfer: reject declared size before reading.
|
||||
parsed=urlsplit(c.base)
|
||||
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=10)
|
||||
conn.putrequest('POST',path+'?filename=big.png')
|
||||
for k,v in {'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png','Content-Length':str(10485761)}.items():
|
||||
conn.putheader(k,v)
|
||||
conn.endheaders()
|
||||
response=conn.getresponse()
|
||||
check(response.status==413,'oversized content-length accepted')
|
||||
response.read();conn.close()
|
||||
cases.append({'case':'oversized_content_length_before_body','status':413})
|
||||
# True streamed overflow, no Content-Length; response is read after the last
|
||||
# chunk, even if the peer stopped accepting bytes once it knew the limit.
|
||||
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=15)
|
||||
conn.putrequest('POST',path+'?filename=chunked.png')
|
||||
conn.putheader('Authorization','Bearer '+alice['token'])
|
||||
conn.putheader('Content-Type','image/png')
|
||||
conn.putheader('Transfer-Encoding','chunked')
|
||||
conn.endheaders()
|
||||
try:
|
||||
block=b'x'*65536
|
||||
for _ in range(161):
|
||||
conn.send(b'10000\r\n'+block+b'\r\n')
|
||||
conn.send(b'0\r\n\r\n')
|
||||
except (BrokenPipeError,ConnectionResetError):
|
||||
pass
|
||||
response=conn.getresponse()
|
||||
check(response.status==413,'streamed byte limit not enforced')
|
||||
response.read();conn.close()
|
||||
cases.append({'case':'chunked_oversize','status':413})
|
||||
att,content=c.attachment(alice,chat)
|
||||
# Another member can reference an attachment; ownership is not required.
|
||||
msg=c.message(bob,chat,'',attachments=[att['id']])['json']
|
||||
check(msg['attachment_ids']==[att['id']],'member could not link attachment')
|
||||
for suffix in ('','/content'):
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve)
|
||||
c.wait_ready(alice,att)
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=unknown',404,actor=alice)
|
||||
cases.append({'case':'foreign_attachment_metadata_and_bytes','status':404})
|
||||
other=c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'other','member_ids':[alice['id']]},schema='Chat')['json']
|
||||
other_path='/api/v1/chats/'+other['id']+'/messages'
|
||||
for aid in (att['id'],str(uuid.uuid4())):
|
||||
c.call('POST',other_path,404,actor=alice,body={'text':'','attachment_ids':[aid]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
cases.append({'case':'cross_chat_and_unknown_attachment','status':404})
|
||||
message_path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
for ids in ([att['id'],att['id']],['bad-id'],[str(uuid.uuid4()) for _ in range(11)]):
|
||||
c.call('POST',message_path,400,actor=alice,body={'text':'','attachment_ids':ids},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
key=uuid.uuid4().hex
|
||||
first=c.message(alice,chat,'',key=key,attachments=[att['id']])['json']
|
||||
second=c.message(alice,chat,'',key=key,attachments=[att['id']])['json']
|
||||
check(first==second,'attachment message replay changed')
|
||||
c.call('POST',message_path,409,actor=alice,body={'text':'different','attachment_ids':[att['id']]},headers={'Idempotency-Key':key})
|
||||
cases.append({'case':'attachment_idempotency_and_conflict','status':'201/409'})
|
||||
# Full JPEG validation, including truncated entropy-coded body.
|
||||
stream=io.BytesIO()
|
||||
Image.new('RGB',(320,200),'blue').save(stream,'JPEG')
|
||||
jpeg=stream.getvalue()
|
||||
uploaded=c.call('POST',path+'?filename=photo.jpg',202,actor=alice,raw=jpeg,headers={'Content-Type':'image/jpeg'},schema='Attachment')['json']
|
||||
c.wait_ready(alice,uploaded)
|
||||
downloaded=c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',actor=bob)
|
||||
check(downloaded['bytes']==jpeg,'JPEG original changed')
|
||||
bad('truncated_jpeg',415,jpeg[:-20],mime='image/jpeg')
|
||||
# A disconnect before receiving all bytes creates neither intent nor object.
|
||||
from app import store,object_store
|
||||
store.pool.open(wait=True,timeout=15)
|
||||
try:
|
||||
def counts():
|
||||
with store.connect() as db:
|
||||
n=db.execute('SELECT count(*) FROM attachments').fetchone()[0]
|
||||
return n,len(object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='staging/').get('Contents',[]))
|
||||
before=counts()
|
||||
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=5)
|
||||
conn.putrequest('POST',path+'?filename=interrupted.png')
|
||||
conn.putheader('Authorization','Bearer '+alice['token'])
|
||||
conn.putheader('Content-Type','image/png')
|
||||
conn.putheader('Content-Length',str(len(fixture)))
|
||||
conn.endheaders();conn.send(fixture[:50]);conn.close()
|
||||
time.sleep(1)
|
||||
check(counts()==before,'aborted upload persisted partial data')
|
||||
cases.append({'case':'interrupted_http_body','metadata_or_object_added':False})
|
||||
finally:
|
||||
store.pool.close();object_store.close()
|
||||
print(json.dumps({'status':'PASS','cases':cases,'count':len(cases)},ensure_ascii=False,indent=2))
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Isolated decoder/encoder. No network or credentials; bounded by parent timeout."""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
# Address space bound complements the whole-container cgroup memory cap.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (640*1024**2,640*1024**2))
|
||||
Image.MAX_IMAGE_PIXELS = 40000000
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def fit(image, cap):
|
||||
w,h=image.size
|
||||
ratio=min(1,cap/w,cap/h)
|
||||
return image.resize((max(1,int(w*ratio)),max(1,int(h*ratio))), Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def main():
|
||||
source,target=Path(sys.argv[1]),Path(sys.argv[2])
|
||||
version=int(sys.argv[3]) if len(sys.argv)>3 else 1
|
||||
jpeg_quality,webp_quality=(85,80) if version==1 else (90,85)
|
||||
with Image.open(source,formats=['PNG','JPEG']) as im:
|
||||
if im.width*im.height>40000000:
|
||||
raise ValueError('pixels')
|
||||
im.load()
|
||||
image=ImageOps.exif_transpose(im).convert('RGBA')
|
||||
white=Image.new('RGBA',image.size,'white')
|
||||
white.alpha_composite(image)
|
||||
rgb=white.convert('RGB')
|
||||
variants=[]
|
||||
for name,content_type,output in (
|
||||
('thumbnail','image/jpeg',fit(rgb,256)),
|
||||
('crop','image/jpeg',ImageOps.fit(rgb,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5))),
|
||||
('optimized','image/webp',fit(rgb,1280))):
|
||||
path=target/name
|
||||
# Explicit empty EXIF; encoder gets a fresh image without source metadata.
|
||||
clean=Image.new('RGB',output.size);clean.paste(output)
|
||||
clean.save(path,format='WEBP' if name=='optimized' else 'JPEG',quality=webp_quality if name=='optimized' else jpeg_quality,exif=b'')
|
||||
payload=path.read_bytes()
|
||||
variants.append(dict(name=name,content_type=content_type,width=output.width,height=output.height,
|
||||
size_bytes=len(payload),sha256=hashlib.sha256(payload).hexdigest()))
|
||||
(target/'manifest.json').write_text(json.dumps(variants))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:
|
||||
main()
|
||||
except MemoryError:
|
||||
sys.exit(3)
|
||||
except Exception:
|
||||
# Deliberately no decoder traceback or client-controlled payload in logs.
|
||||
sys.exit(2)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Fenced, serial worker; DB row locks protect all effects of one job/version."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from psycopg.rows import dict_row
|
||||
import redis
|
||||
from app import store, object_store
|
||||
from pipeline import audit, broker, fetch_job, GROUP, LEASE, MAX_ATTEMPTS, STREAM, TIMEOUT
|
||||
|
||||
stop=threading.Event()
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def failpoint(name, selected):
|
||||
# CLI/in-process test only, never from broker or HTTP.
|
||||
if name == selected:
|
||||
os._exit(86)
|
||||
|
||||
|
||||
def valid_message(msg):
|
||||
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
|
||||
if not isinstance(msg,dict) or not required<=msg.keys() or set(msg)-required-{'traceparent','tracestate'}:
|
||||
return False
|
||||
try:
|
||||
uuid.UUID(msg['job_id']);uuid.UUID(msg['attachment_id'])
|
||||
datetime.fromisoformat(msg['created_at'].replace('Z','+00:00'))
|
||||
return (type(msg['schema_version']) is int and msg['schema_version']==1 and msg['type']=='attachment.process.v1'
|
||||
and type(msg['attempt']) is int and msg['attempt']>=1
|
||||
and type(msg['pipeline_version']) is int and msg['pipeline_version']>=1)
|
||||
except (ValueError,TypeError,AttributeError):return False
|
||||
|
||||
|
||||
def claim(msg):
|
||||
with store.connect() as conn:
|
||||
job=fetch_job(conn,msg['job_id'],'FOR UPDATE SKIP LOCKED')
|
||||
if job is None:return 'defer',None
|
||||
if str(job['attachment_id'])!=msg['attachment_id'] or job['pipeline_version']!=msg['pipeline_version']:
|
||||
return 'invalid',None
|
||||
if job['state'] in ('ready','failed','superseded'):
|
||||
audit(conn,job['id'],'duplicate_noop')
|
||||
return 'noop',job
|
||||
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
|
||||
if att['pipeline_version']!=job['pipeline_version']:
|
||||
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],))
|
||||
audit(conn,job['id'],'stale_noop');return 'noop',job
|
||||
now=datetime.now(timezone.utc)
|
||||
if job['next_at']>now or (job['state']=='processing' and job['lease_until'] and job['lease_until']>now):
|
||||
return 'defer',job
|
||||
if job['attempt']>=MAX_ATTEMPTS:
|
||||
conn.execute("UPDATE jobs SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(job['id'],))
|
||||
conn.execute("UPDATE attachments SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(att['id'],))
|
||||
audit(conn,job['id'],'retry_exhausted');return 'noop',job
|
||||
job=conn.cursor(row_factory=dict_row).execute("""UPDATE jobs SET state='processing',attempt=attempt+1,
|
||||
lease_until=now()+(%s*interval '1 second'),updated_at=now() WHERE id=%s RETURNING *""",(LEASE,job['id'])).fetchone()
|
||||
conn.execute("UPDATE attachments SET state='processing',updated_at=now() WHERE id=%s",(att['id'],))
|
||||
audit(conn,job['id'],'claimed')
|
||||
return 'claimed',job
|
||||
|
||||
|
||||
def record_error(job, code, permanent):
|
||||
with store.connect() as conn:
|
||||
current=fetch_job(conn,job['id'],'FOR UPDATE')
|
||||
if current['state']!='processing' or current['attempt']!=job['attempt']:return
|
||||
final=permanent or current['attempt']>=MAX_ATTEMPTS
|
||||
state='failed' if final else 'queued'
|
||||
code=code if permanent or not final else 'retry_exhausted'
|
||||
delay=random.uniform(.5,1.5)*min(30,2**(current['attempt']-1))
|
||||
conn.execute("UPDATE jobs SET state=%s,error_code=%s,next_at=now()+(%s*interval '1 second'),lease_until=NULL,updated_at=now() WHERE id=%s",(state,code,delay,job['id']))
|
||||
conn.execute("UPDATE attachments SET state=%s,error_code=%s,updated_at=now() WHERE id=%s AND pipeline_version=%s",
|
||||
(state,code if final else None,job['attachment_id'],job['pipeline_version']))
|
||||
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],))
|
||||
audit(conn,job['id'],'failed:'+code if final else 'retry:'+code)
|
||||
|
||||
|
||||
def handle(msg, crash=None, force_error=None):
|
||||
if not valid_message(msg):return 'invalid'
|
||||
action,job=claim(msg)
|
||||
if action!='claimed':return action
|
||||
try:
|
||||
with store.connect() as conn:
|
||||
current=fetch_job(conn,job['id'],'FOR UPDATE')
|
||||
if current['state']!='processing' or current['attempt']!=job['attempt']:return 'defer'
|
||||
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
|
||||
if att['pipeline_version']!=job['pipeline_version']:
|
||||
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],));return 'noop'
|
||||
if force_error=='permanent':raise Permanent('invalid_image')
|
||||
if force_error=='temporary':raise TimeoutError()
|
||||
failpoint('before_effect',crash)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root=Path(folder)
|
||||
response=object_store.client.get_object(Bucket=object_store.bucket,Key=att['staging_key'])
|
||||
try:payload=response['Body'].read(10485761)
|
||||
finally:response['Body'].close()
|
||||
if len(payload)!=att['size_bytes'] or hashlib.sha256(payload).hexdigest()!=att['sha256']:
|
||||
raise Permanent('staging_corrupt')
|
||||
(root/'input').write_bytes(payload)
|
||||
result=subprocess.run([sys.executable,'transform.py',str(root/'input'),str(root),str(job['pipeline_version'])],timeout=TIMEOUT,
|
||||
env={'PATH':os.environ.get('PATH','/usr/local/bin:/usr/bin:/bin'),'LANG':'C.UTF-8'},
|
||||
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False)
|
||||
if result.returncode:
|
||||
raise Permanent('resource_limit' if result.returncode in (3,-9) else 'invalid_image')
|
||||
variants=json.loads((root/'manifest.json').read_text())
|
||||
prefix=f"results/{att['id']}/v{job['pipeline_version']}/"
|
||||
key=prefix+'original'
|
||||
def put(key,data,mime):
|
||||
object_store.client.put_object(Bucket=object_store.bucket,Key=key,Body=data,
|
||||
ContentType=mime,Metadata={'sha256':hashlib.sha256(data).hexdigest()})
|
||||
put(key,payload,att['content_type'])
|
||||
for variant in variants:
|
||||
variant['object_key']=prefix+variant['name']
|
||||
put(variant['object_key'],(root/variant['name']).read_bytes(),variant['content_type'])
|
||||
failpoint('after_effect',crash)
|
||||
conn.execute("""UPDATE attachments SET state='ready',error_code=NULL,object_key=%s,
|
||||
variants=%s::jsonb,updated_at=now() WHERE id=%s AND pipeline_version=%s""",
|
||||
(key,json.dumps(variants),att['id'],job['pipeline_version']))
|
||||
conn.execute("UPDATE jobs SET state='ready',error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=%s",(job['id'],))
|
||||
audit(conn,job['id'],'ready')
|
||||
failpoint('after_commit',crash)
|
||||
return 'ready'
|
||||
except Permanent as exc:
|
||||
record_error(job,str(exc),True);return 'failed'
|
||||
except ClientError as exc:
|
||||
code=exc.response.get('Error',{}).get('Code','')
|
||||
record_error(job,'staging_missing' if code in ('NoSuchKey','404') else 'storage_unavailable',code in ('NoSuchKey','404'))
|
||||
return 'retry'
|
||||
except (BotoCoreError,TimeoutError,subprocess.TimeoutExpired):
|
||||
record_error(job,'processing_timeout',False);return 'retry'
|
||||
except Exception:
|
||||
# Includes a lost/ambiguous DB COMMIT: never delete effects. Lease and
|
||||
# durable outbox safely recover; a ready duplicate cannot be rerun.
|
||||
print(json.dumps({'worker_error':'infrastructure','job_id':str(job['id'])}),flush=True)
|
||||
return 'defer'
|
||||
|
||||
|
||||
def ack(client,mid):
|
||||
client.xack(STREAM,GROUP,mid)
|
||||
client.xdel(STREAM,mid)
|
||||
|
||||
|
||||
def main():
|
||||
store.pool.open(wait=True,timeout=30)
|
||||
client=broker();consumer=str(uuid.uuid4())
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:stop.set())
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
try:client.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if 'BUSYGROUP' not in str(exc):raise
|
||||
reclaimed=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE*1000,start_id='0-0',count=1)[1]
|
||||
entries=reclaimed
|
||||
if not entries:
|
||||
batches=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
|
||||
entries=batches[0][1] if batches else []
|
||||
for mid,fields in entries:
|
||||
try:msg=json.loads(fields.get('job','null'))
|
||||
except ValueError:msg=None
|
||||
outcome=handle(msg)
|
||||
# A failed attempt has a durable retry intent/failed record.
|
||||
# Defer leaves the delivery pending for lease-based reclaim.
|
||||
if outcome!='defer':ack(client,mid)
|
||||
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'worker_error':type(exc).__name__}),flush=True);stop.wait(1)
|
||||
finally:
|
||||
client.close();object_store.close();store.pool.close()
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
Reference in New Issue
Block a user