ЛР 5 - БВТ2303 - Савин - на 5 #104

Open
healby wants to merge 3 commits from healby/mtusi2026lab5:main into main
71 changed files with 6406 additions and 28 deletions
+22
View File
@@ -0,0 +1,22 @@
FROM python:3.11.11-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
DATABASE_URL=postgresql://messenger:messenger@db:5432/messenger \
SESSION_TTL_SECONDS=3600
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
ARG APP_REVISION=lab5-v1
ENV APP_REVISION=$APP_REVISION
COPY app.py attachments.py pipeline.py worker.py publisher.py transform.py janitor.py manage.py migrate.py /app/
COPY migrations /app/migrations
COPY scripts /app/scripts
COPY tests /app/tests
COPY contracts /app/contracts
COPY lab.json /app/lab.json
EXPOSE 8080
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]
+27 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://localhost:8085
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,29 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: up down research test-lifecycle test-backpressure test-extra
up:
./scripts/bootstrap.sh
./scripts/compose.sh up -d --build
$(PYTHON) scripts/wait_ready.py --base-url "http://localhost:8085"
down:
./scripts/compose.sh down
research:
./scripts/research.sh
# These manage real containers; run sequentially, never concurrently.
test-lifecycle:
$(PYTHON) tests/lifecycle.py
test-backpressure:
$(PYTHON) tests/backpressure.py
test-extra:
./scripts/compose.sh run --rm --no-deps ops python tests/uploads_negative.py
./scripts/compose.sh run --rm --no-deps ops python tests/image_variants.py
./scripts/compose.sh run --rm --no-deps ops python tests/private_storage.py
./scripts/compose.sh run --rm --no-deps -e BASE_URL=http://lb:8080 ops python tests/cache_consistency.py
$(PYTHON) tests/idempotency_concurrent.py --base-url "$(BASE_URL)"
+18
View File
@@ -0,0 +1,18 @@
## Лабораторная 5
Автор: healby; группа не указана. Целевая оценка: 5. Трек 2: идемпотентность versioned pipeline.
Сохранён мессенджер из lab4; upload теперь возвращает 202 после устойчивого staging и транзакции metadata/job/outbox. Добавлены отдельные Valkey Streams broker, publisher, worker и cleanup; thumbnail/crop/WebP, retry/backoff/jitter, failed registry/replay, lease и защита версий.
Запуск: `make up`, `make check`, `make test`. Полные команды и реальные доказательства: [REPORT.md](REPORT.md), исследование с таблицей/графиком: [RESEARCH.md](RESEARCH.md).
Проверены public smoke, OpenAPI, отрицательные сценарии, приватность S3, варианты, дубли/аварии/версии, restart API/broker, реальный lease, backpressure, persistence всех сервисов и failover API. Ограничения: один host, без HA дисков/backup; ускоренные часы в части fault-матрицы обозначены отдельно.
Источники: официальный Valkey/Pillow/PostgreSQL и исходные контракты курса; Codex использован для реализации, тестов и документации.
- [x] Весь обязательный минимум сохранён с объявленным переходом upload 201 → 202.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены, реальный вывод приложен.
- [x] Сценарии отказов/безопасности воспроизведены.
- [x] Локальные секреты исключены из Git, токены не включены в отчёт.
- [x] README задания сохранён, REPORT.md описывает именно это решение.
+207 -20
View File
@@ -1,37 +1,224 @@
# Отчёт — лабораторная 5
# Лабораторная 5. Очереди и асинхронная обработка изображений
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: Савин Александр БВТ2303;
- Целевая оценка: **5**, выполнены уровни 3 и 4; **трек 2 — доказанная идемпотентность pipeline**.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git, Python 3.9+ для публичных HTTP-проверок, Python 3.11+ для локального окружения исследования, Docker Engine с Compose v2. На macOS использована Colima, Linux arm64, 4 vCPU и около 8 GiB RAM. Docker-версии и фактические ресурсы: [runtime.json](evidence/runtime.json), [containers.json](evidence/containers.json). Синтетический HTTP-стенд доступен только на `127.0.0.1:8085`.
## Архитектура и решения
```sh
git clone https://gitea.privcloud.tech/healby/mtusi2026lab5.git
cd mtusi2026lab5
./scripts/bootstrap.sh
./scripts/compose.sh config --quiet
make up
make check
make test
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
# Полная проверка OpenAPI и генератор графиков:
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-research.txt
make validate PYTHON=.venv/bin/python
# Проверки предыдущих возможностей и вариантов:
make test-extra
# Автоматические crash/retry/version/cleanup-сценарии:
make research
# Эти сценарии управляют сервисами; выполнять последовательно:
python3 tests/lifecycle.py > evidence/lifecycle.json
python3 tests/backpressure.py > evidence/backpressure.json
python3 scripts/container_snapshot.py > evidence/persistence-before.json
make test-persistence ACTION_SCRIPT=scripts/restart.sh > evidence/persistence.txt
python3 scripts/container_snapshot.py > evidence/persistence-after.json
python3 scripts/verify_persistence.py > evidence/persistence-verified.json
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
./scripts/compose.sh start api
python3 scripts/wait_ready.py
python3 scripts/container_snapshot.py > evidence/containers.json
.venv/bin/python scripts/summarize_research.py
python3 scripts/summarize_operations.py
```
`bootstrap.sh` создаёт `.env` с тремя случайными секретами и портом 8085, не перезаписывая существующий файл. `.env` исключён из Git и Docker build context. Миграции PostgreSQL и создание приватного bucket/ограниченной учётной записи S3 выполняют одноразовые сервисы `migrate` и `s3-init`. Повторный запуск безопасен. Зафиксированы PostgreSQL 16.6, Valkey 8.0.2, HAProxy 3.0.5, Python 3.11.11, MinIO/mc по digest; Python-библиотеки — в [requirements.txt](requirements.txt).
Безопасная остановка сохраняет volumes: `make down`. Намеренное удаление **только учебных данных пятой работы**: `./scripts/compose.sh down -v`. После удаления volumes старые пользователи, сессии, очередь и файлы восстановить нельзя. Проекты лабораторных 2–4 имеют другие Compose names, порты и volumes.
## Архитектура и переход с lab4
```mermaid
flowchart LR
C[Клиент] --> LB[HAProxy]
LB --> A[API ×2]
A --> DB[(PostgreSQL: metadata, jobs, outbox)]
A --> S[(MinIO: staging)]
DB --> P[Publisher]
P --> B[(Valkey Streams + AOF)]
B --> W[Worker]
W --> D[Pillow subprocess]
W --> S3[(MinIO: results/id/version)]
W --> DB
A --> K[(Valkey: кэш сессий)]
```
Из предыдущих работ сохранены Argon2id-пароли, устойчивые сессии и отзыв, авторизация участников чата, Unicode, keyset-пагинация, атомарная идемпотентность сообщений на 24 часа, две API-реплики и балансировка, проверка файлов, приватный S3 и авторизованная потоковая выдача оригинала. Источник истины для состояния — PostgreSQL; кэш не может подменить сессию/пользователя. Байты хранятся в MinIO.
Объявленный переход: upload `201 ready` заменён на `202 queued`; метаданные доступны до готовности, content отвечает 409, а queued/processing можно прикрепить к сообщению. Failed прикреплять нельзя. Новая миграция переводит существующие ready/pending-вложения lab4 в очередь, используя старый оригинал как источник; новая установка получает пустые таблицы. Автоматический reconciler из исследовательского трека lab4 заменён подходящей для асинхронной работы очисткой staging; прежний алгоритм нельзя применять к текущим queued jobs.
### Границы устойчивости
1. API аутентифицирует клиента, проверяет чат/имя/MIME/bytes/пиксели/целостность. Приём идёт небольшими блоками в ограниченный временный spool; это ещё не durable staging.
2. Под общей advisory-блокировкой приёма проверяется лимит активных jobs. Полный оригинал записывается в приватный `staging/<UUID>` MinIO на persistent volume.
3. В одной транзакции PostgreSQL создаются attachment, job и outbox. После COMMIT возвращается **снимок queued**. Падение после S3 до COMMIT может оставить только непринятый orphan; он убирается после retention. Такой запрос не получит подтверждённого 202.
4. Publisher читает outbox, выполняет XADD и только затем сохраняет published_at. Ответ XADD при `appendfsync=always` — граница подтверждения брокером. Если процесс упал после XADD до COMMIT outbox, публикация повторится. Если упал после COMMIT API до publish, сохранённый outbox будет найден при запуске publisher.
5. Worker использует consumer group и `XREADGROUP COUNT 1`. Краткая транзакция фиксирует processing, attempt и lease. Следующая транзакция блокирует job и attachment, проверяет поколение/попытку и держит блокировки до записи всех S3-эффектов и COMMIT ready. Живой worker не теряет право на эффект только из-за истечения времени lease.
6. После устойчивого результата worker выполняет XACK, затем XDEL обработанной доставки. При временной ошибке ack допустим только после сохранения нового намерения retry в БД. Необработанная доставка остаётся pending; `XAUTOCLAIM` позволяет её восстановить после остановки consumer.
Staging уже находится в том же MinIO, что и результаты: приём файла включает S3 PUT. Асинхронная граница отделяет resize/encode и публикацию конечного `results/` от HTTP, а не устраняет передачу байтов клиентом. Readiness требует PostgreSQL и S3; брокер, worker и необязательный кэш не включены, поскольку 202 можно безопасно принять через staging/outbox и при их остановке.
### Варианты, состояния и лимиты
| Вариант | Правила |
| --- | --- |
| original | Побайтно исходный PNG/JPEG; SHA-256 и размер из upload |
| thumbnail | JPEG, вписан в 256×256, без увеличения, размеры округляются вниз, минимум 1 px |
| crop | JPEG, центральный квадрат 128×128, маленькие допускается увеличивать |
| optimized | WebP, вписан в 1280×1280, без увеличения |
EXIF orientation применяется до геометрии; прозрачность компонуется на белый фон. Метаданные EXIF не переносятся в варианты; оригинал сохраняет исходные байты. Для v1 JPEG quality=85, WebP quality=80; для v2 и следующих JPEG=90, WebP=85. Optimized не обязан быть меньше каждого небольшого оригинала.
В API доступны queued → processing → ready/failed и processing → queued при временной ошибке. Error_code безопасный, без внутренних адресов и traceback. Готовность фиксируется после оригинала и всех трёх вариантов. Content проверяет авторизацию и статус для каждого запроса; redirect или публичная ссылка S3 не используются.
HTTP ограничен 10 MiB, 40 млн пикселей, 30 секундами передачи и 5 секундами простоя; одновременно принимаются два файла на API. Обработка выполняется в отдельном subprocess: 20 секунд на decode/encode, RLIMIT_AS 640 MiB, весь контейнер worker — 768 MiB. Процесс обработки получает минимальное окружение без credentials. Чрезмерно затратное, но формально допустимое по пикселям изображение может получить конечный resource_limit; 40 млн пикселей — защитный максимум, не обещание обработки при любом содержимом и выделенной памяти.
Обычный worker последовательный, prefetch/count=1; число отдельных workers можно увеличить через `--scale worker=N`. Максимум 100 queued/processing jobs контролируется транзакционно на всех API. При заполнении — 429 до записи нового staging. Это ограничение текущей работы, не общий лимит хранения всей истории файлов.
### Retry, failed, replay и cleanup
Временные S3-сбои и timeout повторяются до 5 управляемых попыток: задержка `uniform(0.5,1.5) × min(30, 2^(attempt−1))` секунд. Сам broker redelivery не создаёт новую логическую job. Неисправный/отсутствующий staging, invalid_image или resource_limit дают failed; исчерпание временных попыток — retry_exhausted. Registry failed — таблица jobs, а не отдельная очередь с потерянным контекстом. Операторский replay сбрасывает бюджет попыток текущей failed-job, пишет audit и не допускается для ready или отсутствующего staging.
```sh
./scripts/compose.sh run --rm --no-deps ops python manage.py failed
./scripts/compose.sh run --rm --no-deps ops python manage.py replay JOB_UUID
./scripts/compose.sh run --rm --no-deps ops python manage.py upgrade ATTACHMENT_UUID
# Очистка: безопасный dry-run по умолчанию:
./scripts/compose.sh run --rm --no-deps ops python janitor.py --once
./scripts/compose.sh run --rm --no-deps ops python janitor.py --once --apply
```
Lease по умолчанию 30 секунд и не меньше timeout+5. Зависшая processing после смерти worker захватывается заново; старый процесс дополнительно ограничен блокировками и версией. Повторы получают те же ключи `(attachment_id, pipeline_version, variant)`. Старый job не может откатить ready или опубликовать результат поверх новой версии. PUT после аварии может повториться — это идемпотентность эффекта, а не exactly-once доставка.
Janitor запускается раз в минуту и удаляет staging старше суток только после проверки состояния под блокировками. Он сохраняет queued/processing независимо от возраста, удаляет orphan после неудавшегося upload и удерживает failed-источники сутки для replay. Dry-run ничего не меняет, повторный apply идемпотентен. Готовые объекты и старые опубликованные версии автоматически не удаляются. Источник новой версии может ссылаться на готовый original предыдущей версии; такой объект не попадает в staging cleanup.
### Где настраивается
Реальные локальные параметры показаны в [infra/local.env.example](infra/local.env.example); `bootstrap.sh` генерирует секреты отдельно. В исходном `.env.example` заданы логические интерфейсы задания, а не готовая конфигурация этого стенда.
| Переменная/настройка | Реальная реализация |
| --- | --- |
| APP_PORT, API_REPLICAS | `.env` → Compose: порт 8085, две API |
| DATABASE_URL_FILE, CACHE_URL_FILE | Используются прямые DATABASE_URL/CACHE_URL в окружении контейнеров; значения собираются Compose из локальных секретов |
| S3_*_FILE | Прямые S3_ACCESS_KEY_ID/S3_SECRET_ACCESS_KEY в контейнерах; только s3-init получает административные credentials |
| BROKER_URL_FILE | Прямой BROKER_URL=`redis://broker:6379/0`, внутренняя сеть, порт наружу не опубликован |
| WORKER_CONCURRENCY | Внутри worker фиксировано 1, горизонтальное число задаётся `--scale worker=N` |
| JOB_MAX_ATTEMPTS | `.env`, по умолчанию 5 |
| JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS | `.env`, 20 и 30 секунд; минимальный lease=timeout+5 |
| MAX_PENDING_JOBS | `.env`, 100 на весь проект |
| STAGING_RETENTION_SECONDS | `.env`, 86400, janitor не допускает меньше 60 секунд |
| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | Фиксированные контрактом 10 MiB/40 млн в `attachments.py`, пиксельный лимит также в `transform.py` |
| SESSION_TTL_SECONDS | 3600 в Compose, проверяется PostgreSQL, TTL не продлевается рестартом |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдение | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура и контракт | `make check`, `make validate PYTHON=.venv/bin/python` | PASS | [check](evidence/check.txt), [validate](evidence/validate.txt) |
| Публичный минимум 1–5 | `make test` | PASS: 202, сообщение, ready, SHA оригинала, настоящий JPEG thumbnail | [smoke](evidence/smoke.txt) |
| Сессии/кэш из lab2 | cache_consistency.py | Подделанный/устаревший кэш отвергнут, logout сохраняет отзыв, 8/8 идентификаций верны | [cache](evidence/cache-consistency.txt) |
| Две API и message idempotency | idempotency_concurrent.py | 16 запросов через две реплики → одно сообщение | [idempotency](evidence/idempotency.json) |
| Приватный S3, least privilege | private_storage.py | Анонимные list/get 403; admin/посторонний prefix запрещены; console недоступна | [private storage](evidence/private-storage.json) |
| Ограничения и авторизация файлов | uploads_negative.py | 21 отрицательный сценарий, включая обрыв HTTP без сохранения частичного файла | [negative](evidence/uploads-negative.json) |
| Worker не нужен для приёма | lifecycle.py | 202 queued, content 409, сообщение принято при остановленном worker | [lifecycle](evidence/lifecycle.json) |
| Устойчивость API/outbox | lifecycle.py | SIGKILL обеих API после 202 до publish; прежнее вложение завершается | [lifecycle](evidence/lifecycle.json) |
| Persistence брокера | lifecycle.py | Очередь сохранена после restart broker на прежнем volume | [lifecycle](evidence/lifecycle.json) |
| Thumbnail/crop/WebP | image_variants.py | EXIF, белый alpha, no-upscale, crop по центру, 1280×640, размеры/hash реальны | [variants](evidence/image-variants.json) |
| Разрыв COMMIT/publish и повтор publish | pipeline_faults.py, lifecycle.py | Outbox доставлен после восстановления; повтор не размножает эффект | [faults](evidence/pipeline-faults.json) |
| Падение после S3 до ack | drive_job.py + pipeline_faults.py | Exit 86, незавершённая broker delivery восстановлена, четыре объекта | [faults](evidence/pipeline-faults.json) |
| Lease после смерти worker | lifecycle.py | Реальное ожидание lease, та же job становится ready на второй попытке | [lifecycle](evidence/lifecycle.json) |
| Backoff/jitter/failed/replay | pipeline_faults.py | Пять попыток, retry_exhausted, безопасный replay; ready replay отвергнут | [faults](evidence/pipeline-faults.json) |
| Время/память subprocess | cleanup_and_limits.py | Реальный timeout на тестовом бюджете 50 мс и MemoryError при 32 MiB классифицированы | [limits](evidence/cleanup-and-limits.json) |
| Staging retention/cleanup | cleanup_and_limits.py | Dry-run/apply/repeat; активный staging сохранён, orphan удалён | [cleanup](evidence/cleanup-and-limits.json) |
| Backpressure и отказ брокера | backpressure.py | 100 приняты без broker, следующий 429, сообщения доступны, все 100 ready после восстановления | [backpressure](evidence/backpressure.json) |
| Перезапуск всех сервисов | `make test-persistence` + snapshots | Сессия/данные/файл сохранены; StartedAt изменён у всех 10 работающих контейнеров, volumes прежние | [persistence](evidence/persistence.txt), [проверка hook](evidence/persistence-verified.json) |
| Отказ одной API | `make test-failover` | Запись восстановлена в допустимый бюджет, подтверждённые сообщения/файл сохранены | [failover](evidence/failover.txt) |
| Уровень 5, трек 2 | `make research` | Три повтора crash/duplicates/reorder, новая версия защищена, инварианты выполнены | [RESEARCH.md](RESEARCH.md) |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Реальный вывод основных команд:
## Эксперименты и отказы
```text
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
```
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
`make check` не запускает приложение; сквозные и инфраструктурные доказательства приведены отдельно. Снимки контейнеров намеренно не содержат окружение и credentials.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
## Исследование уровня 5
Гипотеза: outbox, version fencing и устойчивые ключи S3 сохраняют логический результат при дублировании/изменении порядка/аварии worker. Методика, аппаратные ограничения, размеры файлов и границы выводов изложены в [RESEARCH.md](RESEARCH.md). Сценарии включают реальный consumer-group redelivery и отдельные процессы, завершённые `os._exit(86)`.
<!-- BEGIN RESEARCH RESULTS -->
| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| Падение до PUT | 3 | 1550.718 | 1321.347–1786.888 | 232.845 | 4 | 0 |
| Падение после PUT | 3 | 1549.527 | 1334.593–1872.025 | 284.404 | 4 | 0 |
| Падение после COMMIT до ack | 3 | 1383.915 | 1275.246–1590.414 | 178.915 | 4 | 0 |
| 8 конкурентных дублей | 3 | 143.795 | 121.12–172.849 | 26.448 | 4 | 0 |
| Старая версия после новой | 3 | 223.21 | 109.191–440.476 | 188.235 | 4 | 0 |
![Исследование pipeline](evidence/pipeline-faults.png)
<!-- END RESEARCH RESULTS -->
Время таблицы — длительность тестового сценария с запуском процессов; в матрице lease ускорен SQL-командой. Это не время ожидания реального lease и не production latency. Отдельный lifecycle использует реальные часы. Дополнительные результаты автоматически сведены ниже.
<!-- BEGIN OPERATIONAL RESULTS -->
### Восстановление и ограничения
| Воздействие | Реальное наблюдение | Результат |
| --- | --- | --- |
| Остановка worker, SIGKILL API до publish, restart broker | Длина stream 1 → 1; ready через 3.023 с после запуска worker | PASS, исходник сохранён |
| Смерть worker после PUT, настоящий lease | Ready через 28.923 с; попыток 2 | PASS, без ускорения часов |
| Остановка broker и заполнение очереди | Принято 100; следующий HTTP 429; восстановлено 100 за 24.64 с | PASS, потеряно 0 |
| Перезапуск всего стенда | 10 реальных процессов, прежние volumes и сессия | PASS |
| Отказ API | 114 успешных записей, 2 переходных ошибок; первая успешная запись через 2.171 с | PASS |
| Тайм-аут decode, тестовый бюджет 50 мс | Время сценария 68.024 мс; retry, затем ready | PASS |
| Ограничение адресного пространства, тестовый бюджет 32 MiB | Конечный resource_limit, replay завершился | PASS |
| Staging cleanup, часы +2 дня | Кандидатов 23, повторных 0; активный исходник сохранён | PASS |
### Проверка геометрии реальных файлов
| Исходник/сценарий | Thumbnail | Crop | Optimized |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| small_alpha | 80×40 | 128×128 | 80×40 |
| exif_orientation | 64×256 | 128×128 | 100×400 |
| center_crop | 256×64 | 128×128 | 400×100 |
| large_fit | 256×128 | 128×128 | 1280×640 |
## Ограничения и следующие шаги
### Наблюдаемые retry с jitter
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
| После попытки | Сохранённая задержка до следующей, с |
| --- | ---: |
| 1 | 1.080882 |
| 2 | 2.447441 |
| 3 | 2.182634 |
| 4 | 8.307950 |
После 5-й временной ошибки зафиксирован `retry_exhausted`; явный replay завершается ready. При проверке расписания next_at искусственно сдвигается, так что таблица показывает записанные интервалы, а не фактически выжданные паузы.
<!-- END OPERATIONAL RESULTS -->
Raw JSON/CSV и PNG/SVG хранятся в `evidence/`. График и таблица исследования пересобираются `scripts/summarize_research.py`, операционные таблицы — `scripts/summarize_operations.py`; ручные PASS не заменяют assert сценариев.
+37
View File
@@ -0,0 +1,37 @@
# Исследование: уровень 5, трек 2
## Гипотеза и инварианты
Версионирование результатов, transactional outbox, блокировки PostgreSQL и повторяемые ключи S3 позволяют завершать принятые задачи при at-least-once доставке без дополнительных наборов вариантов и перезаписи нового результата старым.
Инварианты проверяются assertions: каждый проверяемый upload с 202 завершается ready; оригинал побайтно совпадает с принятым; у текущей версии ровно четыре объекта; метаданные и SHA-256 каждого варианта соответствуют скачанным байтам; повтор ready не меняет метаданные; устаревшая job не изменяет новую версию.
## Методика
`make research` останавливает обычные worker/publisher и запускает `tests/pipeline_faults.py` в ops. Создаются реальные HTTP-вложения и настоящие consumer-group доставки Valkey Streams. В отдельном процессе вызывается `os._exit(86)` до PUT, после всех PUT до COMMIT, после COMMIT до XACK. Неподтверждённая доставка затем захватывается другим consumer. Матрица содержит три повтора каждого из пяти основных сценариев, дополнительно проверяет смену уже опубликованной v1 на v2, failed/replay, пять временных ошибок и повтор публикации outbox.
Для ускорения матрицы lease переводится в прошлое SQL-командой; до этого отдельно проверяется отказ захвата действующего lease. Это не измерение реального времени ожидания lease. `tests/lifecycle.py` отдельно проверяет восстановление с настоящими часами, без изменения дедлайна. Конкурентные дубли запускаются восемью потоками, обычный worker последовательный. Время в таблице включает HTTP/БД, запуск Python-процессов и обработку; это время полного тестового сценария, а не latency одного изображения в production.
Один локальный Linux Docker Engine в Colima, arm64, 4 vCPU, около 8 GiB RAM. Лимиты: worker 1 CPU/768 MiB, API 2 реплики по 2 CPU/512 MiB, PostgreSQL 1 CPU/512 MiB, MinIO 0.5 CPU/768 MiB, broker 0.25 CPU/192 MiB; ops 1 CPU/512 MiB. Ограничения CPU — верхние пределы, а не выделенные ядра. Прочие лабораторные могут работать в этой же VM. Файл матрицы — PNG 320×200 из публичных fixtures. Отдельные геометрические тесты используют PNG/JPEG от 80×40 до 1600×800. Нагрузочного прогрева нет: исследуется корректность конкретных interleavings, не максимальный throughput. Точные лимиты и образы сохранены в `evidence/containers.json` и `compose.yaml`.
## Результаты матрицы
| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| Падение до PUT | 3 | 1550.718 | 1321.347–1786.888 | 232.845 | 4 | 0 |
| Падение после PUT | 3 | 1549.527 | 1334.593–1872.025 | 284.404 | 4 | 0 |
| Падение после COMMIT до ack | 3 | 1383.915 | 1275.246–1590.414 | 178.915 | 4 | 0 |
| 8 конкурентных дублей | 3 | 143.795 | 121.12–172.849 | 26.448 | 4 | 0 |
| Старая версия после новой | 3 | 223.21 | 109.191–440.476 | 188.235 | 4 | 0 |
![Время сценариев и количество объектов](evidence/pipeline-faults.png)
После падения до COMMIT требуются две управляемые попытки. После COMMIT повтор не выполняет обработку: достаточно одной попытки. При восьми конкурентных доставках обрабатывает один поток, остальные видят занятость или завершённый результат. После новой версии старые задания не создают объекты v1 и не изменяют публичные метаданные v2. Отдельно проверено обновление уже существующего результата v1 на v2 с изменённым качеством кодирования.
Raw: [pipeline-faults.json](evidence/pipeline-faults.json), [CSV](evidence/fault-summary.csv), [lifecycle.json](evidence/lifecycle.json), [backpressure.json](evidence/backpressure.json), [cleanup-and-limits.json](evidence/cleanup-and-limits.json).
## Ограничения
Это проверка конечного набора аварийных расписаний, а не математическое доказательство для всех interleavings. Исследование не покрывает потерю диска/volume, несколько узлов, длительные сетевые разделения и повреждение файловой системы. Локальная матрица не подтверждает SLA и не измеряет p95 под производственной нагрузкой; для выбранного трека это не заявляется. CPU/RAM ограничены конфигурацией, пиковый RSS декодера в этом исследовании не измеряется.
Физический PUT может повториться после аварии: одинаковый ключ заменяется теми же данными. Гарантируется один текущий логический набор результатов, а доставка остаётся at least once. Объекты прежних успешно опубликованных версий сохраняются: это история версий, не дубли текущего результата. Изменение алгоритма должно сопровождаться увеличением pipeline_version; старый бинарный worker нужно вывести из эксплуатации перед изменением кода существующей версии.
+896
View File
@@ -0,0 +1,896 @@
from __future__ import annotations
import base64
import json
import os
import secrets
import socket
import re
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import psycopg
from psycopg_pool import ConnectionPool
import redis
from argon2 import PasswordHasher
from fastapi import FastAPI, Header, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.concurrency import run_in_threadpool
from pydantic import BaseModel, ConfigDict, StringConstraints, ValidationError
from typing_extensions import Annotated
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname())
USERNAME_RE = r"^[a-z0-9_]{3,32}$"
class ApiError(Exception):
def __init__(self, status_code: int, code: str, message: str, *, headers: Optional[Dict[str, str]] = None):
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers or {}
class UserOut(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
username: str
display_name: str
created_at: str
class SessionOut(BaseModel):
model_config = ConfigDict(extra="forbid")
token: str
token_type: str
expires_at: str
user: UserOut
class ChatOut(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
title: str
member_ids: List[str]
created_at: str
class MessageOut(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: List[str]
class CreateUserIn(BaseModel):
model_config = ConfigDict(extra="forbid")
username: Annotated[str, StringConstraints(pattern=USERNAME_RE, min_length=3, max_length=32)]
display_name: Annotated[str, StringConstraints(min_length=1, max_length=100)]
password: Annotated[str, StringConstraints(min_length=12, max_length=128)]
class CreateChatIn(BaseModel):
model_config = ConfigDict(extra="forbid")
title: Annotated[str, StringConstraints(min_length=1, max_length=100)]
member_ids: List[str]
class CreateMessageIn(BaseModel):
model_config = ConfigDict(extra="forbid")
attachment_ids: List[str] = []
text: Annotated[str, StringConstraints(max_length=4000)]
class Store:
def __init__(self) -> None:
self.connection_url = self._resolve_database_url()
self.cache_url = self._resolve_cache_url()
self.shared_cache = redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
self.session_store = os.getenv("SESSION_STORE", "valkey")
self.password_hasher = PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16)
self.pool = ConnectionPool(
self.connection_url, min_size=1,
max_size=int(os.getenv("DB_POOL_MAX_SIZE", "10")),
timeout=5, max_waiting=20, open=False,
kwargs={"connect_timeout": 5, "autocommit": False,
"options": "-c statement_timeout=5000 -c lock_timeout=500"},
)
def _resolve_database_url(self) -> str:
value = os.getenv("DATABASE_URL")
if value:
return value
secret_path = os.getenv("DATABASE_URL_FILE")
if secret_path and os.path.exists(secret_path):
with open(secret_path, "r", encoding="utf-8") as handle:
return handle.read().strip()
return os.getenv("DEFAULT_DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger")
def _resolve_cache_url(self) -> str:
value = os.getenv("CACHE_URL")
if value:
return value
secret_path = os.getenv("CACHE_URL_FILE")
if secret_path and os.path.exists(secret_path):
with open(secret_path, "r", encoding="utf-8") as handle:
return handle.read().strip()
return os.getenv("DEFAULT_CACHE_URL", "redis://localhost:6379/0")
def connect(self):
return self.pool.connection()
def cache_client(self):
if os.getenv("CACHE_CLIENT_MODE", "shared") == "shared":
return self.shared_cache
return redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
def cache_session(self, session: Dict[str, Any]) -> None:
if self.session_store != "valkey":
return
try:
client = self.cache_client()
ttl_seconds = max(1, int((self.parse_session_expiry(session["expires_at"]) - self.utc_now()).total_seconds()))
payload = json.dumps({
"token": session["token"],
"user_id": session["user_id"],
"expires_at": session["expires_at"],
"revoked_at": session.get("revoked_at"),
"user": session["user"],
}, separators=(",", ":"), ensure_ascii=False)
client.setex(f"session:{session['token']}", ttl_seconds, payload)
except redis.RedisError:
return
def get_cached_session(self, token: str, *, db_session) -> Optional[Dict[str, Any]]:
if self.session_store != "valkey":
return None
try:
client = self.cache_client()
payload = client.get(f"session:{token}")
if not payload:
return None
data = json.loads(payload)
session = {
"token": data["token"],
"user_id": data["user_id"],
"expires_at": data["expires_at"],
"revoked_at": data.get("revoked_at"),
"user": data["user"],
}
if not self.is_session_valid(session):
client.delete(f"session:{token}")
return None
if session != db_session:
client.delete(f"session:{token}")
return None
return db_session
except (redis.RedisError, KeyError, TypeError, ValueError):
return None
def delete_cached_session(self, token: str) -> None:
if self.session_store != "valkey":
return
try:
self.cache_client().delete(f"session:{token}")
except redis.RedisError:
return
@staticmethod
def parse_session_expiry(value: Any) -> datetime:
if isinstance(value, datetime):
return value.astimezone(timezone.utc)
if isinstance(value, str):
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
raise ValueError("invalid session expiry")
def init(self) -> None:
migration_dir = Path(__file__).resolve().parent / "migrations"
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(202603)")
cur.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
"""
)
applied = set()
cur.execute("SELECT version FROM schema_migrations")
for (version,) in cur.fetchall():
applied.add(version)
for migration_path in sorted(migration_dir.glob("*.sql")):
version = migration_path.stem
if version in applied:
continue
sql = migration_path.read_text(encoding="utf-8")
cur.execute(sql)
cur.execute(
"INSERT INTO schema_migrations(version, applied_at) VALUES (%s, %s)",
(version, self.utc_now()),
)
conn.commit()
def utc_now(self) -> datetime:
return datetime.now(timezone.utc)
def isoformat_utc(self, value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def hash_password(self, password: str) -> str:
return self.password_hasher.hash(password)
def verify_password(self, password_hash: str, password: str) -> bool:
try:
self.password_hasher.verify(password_hash, password)
return True
except Exception:
return False
def require_user_by_username(self, username: str, conn):
with conn.cursor() as cur:
cur.execute("SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", (username,))
row = cur.fetchone()
if row is None:
return None
return {
"id": str(row[0]),
"username": row[1],
"display_name": row[2],
"created_at": self.isoformat_utc(row[3]),
"password_hash": row[4],
}
def fetch_user(self, user_id: str, conn):
with conn.cursor() as cur:
cur.execute("SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = cur.fetchone()
if row is None:
return None
return {
"id": str(row[0]),
"username": row[1],
"display_name": row[2],
"created_at": self.isoformat_utc(row[3]),
}
def fetch_session_by_token(self, token: str, conn):
with conn.cursor() as cur:
cur.execute(
"SELECT s.token, s.user_id, s.expires_at, s.revoked_at, u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token = %s",
(token,),
)
row = cur.fetchone()
if row is None:
return None
return {
"token": row[0],
"user_id": str(row[1]),
"expires_at": self.isoformat_utc(row[2]),
"revoked_at": row[3],
"user": {
"id": str(row[4]),
"username": row[5],
"display_name": row[6],
"created_at": self.isoformat_utc(row[7]),
},
}
def list_member_ids(self, chat_id: str, conn) -> List[str]:
with conn.cursor() as cur:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat_id,))
rows = cur.fetchall()
return [str(row[0]) for row in rows]
def fetch_chat(self, chat_id: str, conn):
with conn.cursor() as cur:
cur.execute("SELECT id, title, created_at FROM chats WHERE id = %s", (chat_id,))
row = cur.fetchone()
if row is None:
return None
return {
"id": str(row[0]),
"title": row[1],
"created_at": self.isoformat_utc(row[2]),
"member_ids": self.list_member_ids(str(row[0]), conn),
}
def fetch_message(self, message_id: str, conn):
with conn.cursor() as cur:
cur.execute("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", (message_id,))
row = cur.fetchone()
if row is None:
return None
return {
"id": str(row[0]),
"chat_id": str(row[1]),
"sender_id": str(row[2]),
"text": row[3],
"created_at": self.isoformat_utc(row[4]),
"attachment_ids": self.message_attachment_ids(str(row[0]), conn),
}
def fetch_chat_membership(self, chat_id: str, user_id: str, conn) -> bool:
with conn.cursor() as cur:
cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
return cur.fetchone() is not None
def insert_user(self, username: str, display_name: str, password: str):
now = self.utc_now()
user_id = uuid.uuid4()
password_hash = self.hash_password(password)
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (%s,%s,%s,%s,%s)",
(user_id, username, display_name, password_hash, now),
)
conn.commit()
return {
"id": str(user_id),
"username": username,
"display_name": display_name,
"created_at": self.isoformat_utc(now),
}
def insert_chat(self, title: str, member_ids: List[str], actor_id: str):
now = self.utc_now()
chat_id = uuid.uuid4()
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute("INSERT INTO chats(id,title,created_at) VALUES (%s,%s,%s)", (chat_id, title, now))
for member_id in sorted(set(member_ids)):
cur.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (chat_id, member_id))
conn.commit()
return {
"id": str(chat_id),
"title": title,
"member_ids": sorted(set(member_ids)),
"created_at": self.isoformat_utc(now),
}
def insert_message(self, chat_id: str, sender_id: str, text: str, key: str, attachment_ids: List[str]):
# Transaction-level lock serializes the same scope across all replicas.
scope = json.dumps([sender_id, "POST", chat_id, key])
request_body = {"text": text, "attachment_ids": attachment_ids}
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,))
cur.execute("DELETE FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s AND expires_at <= NOW()", (sender_id, chat_id, key))
cur.execute("SELECT body, response FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s", (sender_id, chat_id, key))
previous = cur.fetchone()
if previous:
if previous[0] != request_body:
raise ApiError(409, "conflict", "idempotency key reused with different body")
return previous[1]
for aid in attachment_ids:
cur.execute("SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE", (aid, chat_id))
att = cur.fetchone()
if att is None:
raise ApiError(404, "not_found", "attachment not found")
if att[0] == "failed":
raise ApiError(409, "conflict", "attachment unavailable")
now = self.utc_now()
message_id = str(uuid.uuid4())
message = {"id": message_id, "chat_id": chat_id, "sender_id": sender_id,
"text": text, "created_at": self.isoformat_utc(now), "attachment_ids": attachment_ids}
cur.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES (%s,%s,%s,%s,%s)",
(message_id, chat_id, sender_id, text, now))
for position, aid in enumerate(attachment_ids):
cur.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, aid, position))
cur.execute("INSERT INTO message_requests(sender_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s::jsonb,%s::jsonb,%s)",
(sender_id, chat_id, key, json.dumps(request_body), json.dumps(message), now + timedelta(hours=24)))
conn.commit()
return message
def message_attachment_ids(self, message_id, conn):
return [str(row[0]) for row in conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (message_id,)).fetchall()]
@staticmethod
def cursor_uuid_key(value: str) -> int:
try:
return uuid.UUID(value).int
except (ValueError, TypeError):
return 0
def list_chats_for_user(self, user_id: str, limit: int, cursor: Optional[str]):
with self.connect() as conn:
with conn.cursor() as cur:
query = """
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE cm.user_id = %s
"""
params: List[Any] = [user_id]
if cursor:
prefix = f"user:{user_id}|"
if not cursor.startswith(prefix):
raise ApiError(400, "invalid_request", "cursor does not match user chat list")
remainder = cursor[len(prefix):]
if not remainder:
raise ApiError(400, "invalid_request", "invalid cursor")
try:
cursor_ts_raw, cursor_id = remainder.split("|", 1)
cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid cursor") from exc
query += " AND (c.created_at > %s OR (c.created_at = %s AND c.id > %s::uuid))"
params.extend([cursor_dt, cursor_dt, cursor_id])
query += " ORDER BY c.created_at ASC, c.id ASC LIMIT %s"
params.append(limit + 1)
cur.execute(query, tuple(params))
rows = cur.fetchall()
items = [
{
"id": str(row[0]),
"title": row[1],
"created_at": self.isoformat_utc(row[2]),
"member_ids": self.list_member_ids(str(row[0]), conn),
}
for row in rows
]
next_cursor = None
if len(items) > limit:
last_item = items[limit - 1]
next_cursor = f"user:{user_id}|{last_item['created_at']}|{last_item['id']}"
items = items[:limit]
return {"items": items, "next_cursor": next_cursor}
def list_messages_for_chat(self, chat_id: str, limit: int, cursor: Optional[str]):
with self.connect() as conn:
with conn.cursor() as cur:
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
prefix = f"chat:{chat_id}|"
if not cursor.startswith(prefix):
raise ApiError(400, "invalid_request", "cursor does not match chat")
remainder = cursor[len(prefix):]
if not remainder:
raise ApiError(400, "invalid_request", "invalid cursor")
try:
cursor_ts_raw, cursor_id = remainder.split("|", 1)
cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid cursor") from exc
query += " AND (created_at, id) > (%s, %s::uuid)"
params.extend([cursor_dt, cursor_id])
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
params.append(limit + 1)
cur.execute(query, tuple(params))
rows = cur.fetchall()
links = {}
if rows:
cur.execute("SELECT message_id, attachment_id FROM message_attachments WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position", ([str(row[0]) for row in rows],))
for mid, aid in cur.fetchall():
links.setdefault(str(mid), []).append(str(aid))
items = [{
"id": str(row[0]),
"chat_id": str(row[1]),
"sender_id": str(row[2]),
"text": row[3],
"created_at": self.isoformat_utc(row[4]),
"attachment_ids": links.get(str(row[0]), []),
} for row in rows]
next_cursor = None
if len(items) > limit:
last_item = items[limit - 1]
next_cursor = f"chat:{chat_id}|{last_item['created_at']}|{last_item['id']}"
items = items[:limit]
return {"items": items, "next_cursor": next_cursor}
def create_session(self, user_id: str, username: str):
token = secrets.token_urlsafe(32)
expires_at = self.utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO sessions(id, token, user_id, expires_at, created_at) VALUES (%s,%s,%s,%s,%s)",
(uuid.uuid4(), token, user_id, expires_at, self.utc_now()),
)
conn.commit()
with self.connect() as conn:
user = self.fetch_user(user_id, conn)
if user is None:
raise ApiError(500, "internal_error", "session user missing")
session = {
"token": token,
"user_id": user_id,
"expires_at": self.isoformat_utc(expires_at),
"revoked_at": None,
"user": user,
}
self.cache_session(session)
return {
"token": token,
"token_type": "Bearer",
"expires_at": self.isoformat_utc(expires_at),
"user": user,
}
def get_session_by_token(self, token: str, conn=None):
if conn is None:
with self.connect() as db_conn:
return self.get_session_by_token(token, db_conn)
db_session = self.fetch_session_by_token(token, conn)
if db_session is None or not self.is_session_valid(db_session):
self.delete_cached_session(token)
return None
cached = self.get_cached_session(token, db_session=db_session)
if cached is None:
self.cache_session(db_session)
return db_session
def revoke_current_session(self, token: str):
with self.connect() as conn:
with conn.cursor() as cur:
cur.execute("UPDATE sessions SET revoked_at = %s WHERE token = %s AND revoked_at IS NULL", (self.utc_now(), token))
conn.commit()
self.delete_cached_session(token)
def is_session_valid(self, session) -> bool:
if session is None:
return False
if session.get("revoked_at") is not None:
return False
try:
expiry = self.parse_session_expiry(session["expires_at"])
except ValueError:
return False
if expiry <= self.utc_now():
return False
return True
store = Store()
from attachments import ObjectStore
object_store = ObjectStore(store, ApiError)
app = FastAPI(title="MTUSI Messenger")
@app.on_event("startup")
def startup_event() -> None:
store.pool.open(wait=True, timeout=10)
@app.on_event("shutdown")
def shutdown_event() -> None:
store.pool.close()
store.shared_cache.close()
object_store.close()
@app.exception_handler(psycopg.OperationalError)
async def database_error_handler(request: Request, exc):
return await api_error_handler(request, ApiError(503, "unavailable", "database unavailable"))
@app.middleware("http")
async def request_id_middleware(request: Request, call_next):
request_id = uuid.uuid4().hex
request.state.request_id = request_id
response = await call_next(request)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
response.headers["X-App-Revision"] = os.getenv("APP_REVISION", "lab4-v1")
return response
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
body = {"error": {"code": exc.code, "message": exc.message, "request_id": getattr(request.state, "request_id", "")}}
return JSONResponse(status_code=exc.status_code, headers={**exc.headers, "X-Request-Id": getattr(request.state, "request_id", ""), "X-Instance-Id": INSTANCE_ID}, content=body)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
request_id = getattr(request.state, "request_id", "")
return JSONResponse(
status_code=400,
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
content={"error": {"code": "invalid_request", "message": "invalid request", "request_id": request_id}},
)
@app.exception_handler(Exception)
async def generic_error_handler(request: Request, exc: Exception):
request_id = getattr(request.state, "request_id", "")
return JSONResponse(
status_code=500,
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
content={"error": {"code": "internal_error", "message": "internal error", "request_id": request_id}},
)
def get_auth_header(request: Request):
auth = request.headers.get("Authorization")
if not auth:
raise ApiError(401, "unauthorized", "missing authorization", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
scheme, _, token = auth.partition(" ")
if scheme.lower() != "bearer" or not token:
raise ApiError(401, "unauthorized", "invalid bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
return token.strip()
def require_bearer_session(request: Request):
token = get_auth_header(request)
with store.connect() as conn:
session = store.get_session_by_token(token, conn)
if session is None or not store.is_session_valid(session):
raise ApiError(401, "unauthorized", "invalid or expired session", headers={"WWW-Authenticate": 'Bearer realm="messenger"'})
return session
async def parse_json_body(request: Request) -> Dict[str, Any]:
try:
raw = await request.body()
if not raw:
return {}
return __import__("json").loads(raw)
except Exception as exc:
raise ApiError(400, "invalid_request", "malformed JSON") from exc
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
try:
with store.connect() as conn:
with conn.cursor() as cur:
cur.execute("SELECT 1")
object_store.ready()
return {"status": "ok"}
except Exception as exc:
raise ApiError(503, "unavailable", "required storage unavailable") from exc
@app.post("/api/v1/users", status_code=201)
async def create_user_endpoint(request: Request):
try:
body = await request.json()
except Exception as exc:
raise ApiError(400, "invalid_request", "malformed JSON") from exc
if not isinstance(body, dict):
raise ApiError(400, "invalid_request", "invalid user payload")
try:
payload = CreateUserIn.model_validate(body)
except ValidationError as exc:
raise ApiError(400, "invalid_request", "invalid user payload") from exc
if not payload.username or not payload.display_name.strip():
raise ApiError(400, "invalid_request", "invalid request")
payload.display_name = payload.display_name.strip()
try:
user = store.insert_user(payload.username, payload.display_name, payload.password)
return UserOut.model_validate(user)
except Exception as exc:
if "duplicate key" in str(exc).lower() or "unique" in str(exc).lower():
raise ApiError(409, "conflict", "username already exists") from exc
raise ApiError(503, "unavailable", "required storage unavailable") from exc
@app.get("/api/v1/users/me")
def current_user(request: Request):
session = require_bearer_session(request)
user = session["user"]
return UserOut.model_validate(user)
@app.post("/api/v1/auth/sessions", status_code=201)
def create_session(request: Request):
auth = request.headers.get("Authorization", "")
if not auth.lower().startswith("basic "):
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
encoded = auth.split(" ", 1)[1].strip()
try:
decoded = base64.b64decode(encoded.encode("ascii"), validate=True)
raw = decoded.decode("utf-8")
except Exception:
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
if ":" not in raw:
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
username, password = raw.split(":", 1)
if not re_match(username):
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
if not (12 <= len(password) <= 128):
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
try:
with store.connect() as conn:
user = store.require_user_by_username(username, conn)
if user is None or not store.verify_password(user["password_hash"], password):
raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'})
session = store.create_session(user["id"], username)
return SessionOut.model_validate(session)
except ApiError:
raise
except Exception as exc:
raise ApiError(503, "unavailable", "required storage unavailable") from exc
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(request: Request):
session = require_bearer_session(request)
token = session["token"]
store.revoke_current_session(token)
return Response(status_code=204)
@app.post("/api/v1/chats", status_code=201)
async def create_chat_endpoint(request: Request):
session = require_bearer_session(request)
try:
body = await request.json()
except Exception as exc:
raise ApiError(400, "invalid_request", "malformed JSON") from exc
if not isinstance(body, dict):
raise ApiError(400, "invalid_request", "invalid chat payload")
try:
payload = CreateChatIn.model_validate(body)
except ValidationError as exc:
raise ApiError(400, "invalid_request", "invalid chat payload") from exc
if len(payload.member_ids) < 1 or len(payload.member_ids) > 100:
raise ApiError(400, "invalid_request", "invalid member_ids")
title = payload.title.strip()
if not title:
raise ApiError(400, "invalid_request", "invalid title")
normalized = []
seen = set()
for member_id in payload.member_ids:
try:
member_id = str(uuid.UUID(member_id))
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid member_ids") from exc
if member_id in seen:
raise ApiError(400, "invalid_request", "duplicate member_ids")
seen.add(member_id)
normalized.append(member_id)
if session["user_id"] not in normalized:
raise ApiError(400, "invalid_request", "current user must be a member")
try:
with store.connect() as conn:
for member_id in normalized:
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE id = %s", (member_id,))
if cur.fetchone() is None:
raise ApiError(400, "invalid_request", "unknown member")
chat = store.insert_chat(title, normalized, session["user_id"])
return ChatOut.model_validate(chat)
except ApiError:
raise
except Exception as exc:
raise ApiError(503, "unavailable", "required storage unavailable") from exc
@app.get("/api/v1/chats")
def list_chats(request: Request):
session = require_bearer_session(request)
limit = request.query_params.get("limit", "50")
cursor = request.query_params.get("cursor")
try:
limit_int = int(limit)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid limit") from exc
if not (1 <= limit_int <= 100):
raise ApiError(400, "invalid_request", "invalid limit")
result = store.list_chats_for_user(session["user_id"], limit_int, cursor)
return {"items": [ChatOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(request: Request, chat_id: str):
session = require_bearer_session(request)
try:
uuid.UUID(chat_id)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
with store.connect() as conn:
chat = store.fetch_chat(chat_id, conn)
if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn):
raise ApiError(404, "not_found", "chat not found")
return ChatOut.model_validate(chat)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(request: Request, chat_id: str):
session = require_bearer_session(request)
try:
uuid.UUID(chat_id)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
try:
body = await request.json()
except Exception as exc:
raise ApiError(400, "invalid_request", "malformed JSON") from exc
if not isinstance(body, dict):
raise ApiError(400, "invalid_request", "invalid message payload")
try:
payload = CreateMessageIn.model_validate(body)
except ValidationError as exc:
raise ApiError(400, "invalid_request", "invalid message payload") from exc
if len(payload.attachment_ids) > 10:
raise ApiError(400, "invalid_request", "too many attachments")
try:
payload.attachment_ids = [str(uuid.UUID(aid)) for aid in payload.attachment_ids]
except (ValueError, TypeError) as exc:
raise ApiError(400, "invalid_request", "invalid attachment id") from exc
if len(set(payload.attachment_ids)) != len(payload.attachment_ids):
raise ApiError(400, "invalid_request", "duplicate attachment ids")
if not payload.text.strip() and not payload.attachment_ids:
raise ApiError(400, "invalid_request", "invalid text")
key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise ApiError(400, "invalid_request", "invalid Idempotency-Key")
with store.connect() as conn:
chat = store.fetch_chat(chat_id, conn)
if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn):
raise ApiError(404, "not_found", "chat not found")
message = store.insert_message(str(uuid.UUID(chat_id)), session["user_id"], payload.text, key, payload.attachment_ids)
return MessageOut.model_validate(message)
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(request: Request, chat_id: str):
session = require_bearer_session(request)
try:
uuid.UUID(chat_id)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid chat_id") from exc
limit = request.query_params.get("limit", "50")
cursor = request.query_params.get("cursor")
try:
limit_int = int(limit)
except ValueError as exc:
raise ApiError(400, "invalid_request", "invalid limit") from exc
if not (1 <= limit_int <= 100):
raise ApiError(400, "invalid_request", "invalid limit")
with store.connect() as conn:
if not store.fetch_chat_membership(chat_id, session["user_id"], conn):
raise ApiError(404, "not_found", "chat not found")
result = store.list_messages_for_chat(chat_id, limit_int, cursor)
return {"items": [MessageOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]}
def re_match(value: str) -> bool:
import re
return bool(re.fullmatch(USERNAME_RE, value))
from attachments import register_routes
register_routes(app, object_store, require_bearer_session)
if __name__ == "__main__":
import uvicorn
uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False)
+240
View File
@@ -0,0 +1,240 @@
"""Bounded uploads into durable S3 staging and transactional job/outbox."""
import asyncio
import hashlib
import os
from pathlib import Path
import tempfile
import threading
import time
import unicodedata
import uuid
import warnings
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import Request
from fastapi.responses import StreamingResponse
from PIL import Image, UnidentifiedImageError
from psycopg.rows import dict_row
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
MAX_BYTES = 10485760
MAX_PIXELS = 40000000
CHUNK = 65536
# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is
# retained as another bound, but its warning threshold is not the contract cap.
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at,pipeline_version,staging_key,variants'
def canonical_id(value, error):
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError) as exc:
raise error(400, 'invalid_request', 'invalid id') from exc
def object_lock(conn, key, *, wait=True):
function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0]
class ObjectStore:
def __init__(self, store, error):
self.store, self.error = store, error
self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'),
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'),
config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3,
retries={'total_max_attempts': 1, 'mode': 'standard'},
max_pool_connections=8, s3={'addressing_style': 'path'}))
self.decode_lock = threading.BoundedSemaphore(1)
# At most two payload spools per process; queued requests receive ASGI
# backpressure instead of being read into an unbounded request.body().
self.upload_slots = asyncio.Semaphore(2)
def close(self):
self.client.close()
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
def authorize_chat(self, request, chat_id, authenticate):
actor = authenticate(request)
chat_id = canonical_id(chat_id, self.error)
with self.store.connect() as conn:
if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn):
raise self.error(404, 'not_found', 'chat not found')
return actor, chat_id
def validate_filename(self, name):
if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name):
raise self.error(400, 'invalid_request', 'invalid filename')
def inspect_image(self, stream, claimed_type):
try:
with self.decode_lock, warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
if image.width * image.height > MAX_PIXELS:
raise self.error(413, 'payload_too_large', 'too many image pixels')
if actual != claimed_type:
raise self.error(415, 'unsupported_media_type', 'content type mismatch')
image.verify()
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
image.load() # JPEG.verify alone does not ensure a full decode.
stream.seek(0)
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
raise self.error(413, 'payload_too_large', 'too many image pixels') from exc
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc
def fetch(self, aid, conn):
with conn.cursor(row_factory=dict_row) as cur:
cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,))
return cur.fetchone()
def public(self, row):
return {'id': str(row['id']), 'chat_id': str(row['chat_id']),
'owner_id': str(row['owner_id']), 'filename': row['filename'],
'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
'sha256': row['sha256'], 'status': row['state'], 'variants': [{k:v for k,v in x.items() if k != 'object_key'} for x in row['variants']], 'error_code': row['error_code'],
'created_at': self.store.isoformat_utc(row['created_at']),
'updated_at': self.store.isoformat_utc(row['updated_at'])}
def authorized(self, request, aid, authenticate):
actor = authenticate(request)
aid = canonical_id(aid, self.error)
with self.store.connect() as conn:
row = self.fetch(aid, conn)
if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn):
raise self.error(404, 'not_found', 'attachment not found')
return row
def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum):
from pipeline import create_job
aid = str(uuid.uuid4())
key = 'staging/' + aid
now = self.store.utc_now()
# The capacity lock serializes admission across every API replica.
# S3 is written before COMMIT: an ambiguous commit may leave an orphan,
# but a returned 202 always has both staging bytes and a durable job.
with self.store.connect() as conn:
conn.execute('SELECT pg_advisory_xact_lock(202605)')
count = conn.execute("SELECT count(*) FROM jobs WHERE state IN ('queued','processing')").fetchone()[0]
if count >= int(os.getenv('MAX_PENDING_JOBS', '100')):
raise self.error(429, 'rate_limited', 'processing capacity exceeded')
try:
stream.seek(0)
self.client.put_object(Bucket=self.bucket, Key=key, Body=stream,
ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum})
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
conn.execute("""INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,
content_type,size_bytes,sha256,state,created_at,updated_at,staging_key)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,%s,%s)""",
(aid,chat_id,actor_id,'results/'+aid+'/v1/original',filename,content_type,size,checksum,now,now,key))
create_job(conn, aid, 1)
row = self.fetch(aid, conn)
return self.public(row)
def download(self, row):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
body = response['Body']
if response['ContentLength'] != row['size_bytes']:
body.close()
raise self.error(503, 'unavailable', 'object unavailable')
def chunks():
try:
while True:
part = body.read(CHUNK)
if not part:
break
yield part
finally:
body.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
'Content-Disposition': 'attachment'})
def register_routes(app, objects, authenticate):
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
async def upload(request: Request, chat_id: str):
actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate)
filename = request.query_params.get('filename', '')
objects.validate_filename(filename)
claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower()
if claimed not in ('image/png', 'image/jpeg'):
raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required')
length = request.headers.get('content-length')
if length:
try:
expected = int(length)
except ValueError as exc:
raise objects.error(400, 'invalid_request', 'invalid content length') from exc
if expected > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
if expected < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
try:
await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30)
except asyncio.TimeoutError as exc:
raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc
try:
# Temporary payloads are never persistent API storage. Every path,
# including disconnect/cancellation/error, closes and unlinks the spool.
with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream:
checksum, size = hashlib.sha256(), 0
try:
iterator = request.stream().__aiter__()
async with asyncio.timeout(30):
while True:
try:
chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5)
except StopAsyncIteration:
break
size += len(chunk)
if size > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
checksum.update(chunk)
await run_in_threadpool(stream.write, chunk)
except (asyncio.TimeoutError, ClientDisconnect) as exc:
raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc
if size < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
await run_in_threadpool(objects.inspect_image, stream, claimed)
return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id,
filename, claimed, size, checksum.hexdigest())
finally:
objects.upload_slots.release()
@app.get('/api/v1/attachments/{aid}')
def metadata(request: Request, aid: str):
return objects.public(objects.authorized(request, aid, authenticate))
@app.get('/api/v1/attachments/{aid}/content')
def content(request: Request, aid: str):
row = objects.authorized(request, aid, authenticate)
if row['state'] != 'ready':
raise objects.error(409, 'conflict', 'attachment unavailable')
variant = request.query_params.get('variant', 'original')
if variant != 'original':
selected = next((v for v in row['variants'] if v['name'] == variant), None)
if selected is None:
raise objects.error(404, 'not_found', 'variant not found')
row = {**row, **selected}
return objects.download(row)
+143
View File
@@ -0,0 +1,143 @@
name: mtusi-lab5
x-api: &api
build: .
image: ${API_IMAGE:-mtusi-lab5-api:local}
environment: &environment
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}@db:5432/messenger
BROKER_URL: redis://broker:6379/0
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
CACHE_URL: redis://valkey:6379/0
SESSION_STORE: valkey
SESSION_TTL_SECONDS: 3600
DB_POOL_MAX_SIZE: 5
CACHE_CLIENT_MODE: shared
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: lab4api
S3_SECRET_ACCESS_KEY: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh}
cpus: ${API_CPUS:-2}
mem_limit: 512m
stop_grace_period: 40s
services:
db:
image: postgres:16.6-alpine
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}
volumes: [postgres_data:/var/lib/postgresql/data]
cpus: 1
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, pg_isready -U messenger -d messenger]
interval: 2s
timeout: 1s
retries: 30
valkey:
image: valkey/valkey:8.0.2-alpine
command: [valkey-server, --appendonly, 'yes']
volumes: [valkey_data:/data]
cpus: 0.25
mem_limit: 128m
s3:
# RELEASE.2025-09-07T16-13-09Z, verified locally, linux/arm64.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER: lab4root
MINIO_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh}
MINIO_BROWSER: 'off'
volumes: [s3_data:/data]
cpus: 0.5
mem_limit: 768m
s3-init:
# mc RELEASE.2025-08-13T08-35-41Z.
image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727
entrypoint: [/bin/sh, /bootstrap/init-s3.sh]
environment:
S3_ROOT_USER: lab4root
S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh}
S3_API_USER: lab4api
S3_API_PASSWORD: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh}
volumes: ['./infra:/bootstrap:ro']
depends_on: [s3]
cpus: 1
mem_limit: 512m
migrate:
<<: *api
command: [python, migrate.py]
depends_on:
db: {condition: service_healthy}
api:
<<: *api
command: [uvicorn, 'app:app', --host, 0.0.0.0, --port, '8080', --timeout-graceful-shutdown, '35', --limit-concurrency, '64']
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
valkey: {condition: service_started}
healthcheck:
test: [CMD, python, -c, "import socket; s=socket.create_connection(('127.0.0.1',8080),1); s.sendall(b'GET /health/ready HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n'); assert s.makefile('rb').readline().split()[1]==b'200'"]
interval: 2s
timeout: 4s
retries: 2
lb:
image: haproxy:3.0.5-alpine
ports: ['127.0.0.1:${APP_PORT:-8085}:8080']
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
depends_on: [api]
cpus: 0.25
mem_limit: 64m
broker:
image: valkey/valkey:8.0.2-alpine
command: [valkey-server, --appendonly, 'yes', --appendfsync, always, --maxmemory-policy, noeviction]
volumes: [broker_data:/data]
cpus: 0.25
mem_limit: 192m
restart: unless-stopped
publisher:
<<: *api
command: [python, publisher.py]
restart: unless-stopped
cpus: 0.25
mem_limit: 192m
depends_on:
migrate: {condition: service_completed_successfully}
broker: {condition: service_started}
worker:
<<: *api
command: [python, worker.py]
restart: unless-stopped
cpus: 1
mem_limit: 768m
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
broker: {condition: service_started}
janitor:
<<: *api
command: [python, janitor.py]
restart: unless-stopped
cpus: 0.25
mem_limit: 192m
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
ops:
<<: *api
profiles: [tools]
volumes: ['./tests:/app/tests:ro', './scripts:/app/scripts:ro']
command: [python, manage.py, failed]
cpus: 1
mem_limit: 512m
volumes:
postgres_data:
valkey_data:
s3_data:
broker_data:
+15 -3
View File
@@ -1,5 +1,17 @@
# Доказательства
# Доказательства лабораторной 5
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Файлы получены реальными запусками из этого репозитория. Credentials, тела запросов с паролями и сессионные токены сюда не записываются.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
- `check.txt`, `validate.txt`, `smoke.txt`, `readiness.txt`: публичные проверки и две здоровые API.
- `pipeline-faults.json`: уровень 5, трек 2; матрица crash/дубли/версии и дополнительные retry/replay/outbox-сценарии.
- `fault-summary.csv`, `pipeline-faults.png`, `pipeline-faults.svg`: сводка матрицы, генерируется `scripts/summarize_research.py`.
- `lifecycle.json`: SIGKILL API, restart broker, сохранность принятого файла и реальный lease timeout.
- `backpressure.json`: 100 загрузок при остановленном broker, 429 на следующей, полное восстановление.
- `cleanup-and-limits.json`: настоящие child timeout/MemoryError с уменьшенными тестовыми бюджетами; cleanup с ускоренными часами.
- `uploads-negative.json`, `private-storage.json`, `image-variants.json`: ограничения, авторизация, приватность и реальные варианты изображений.
- `cache-consistency.txt`, `idempotency.json`: сохранённые возможности прошлых лабораторных.
- `persistence.txt`, `persistence-before.json`, `persistence-after.json`, `persistence-verified.json`: сохранность и проверка фактического перезапуска всех сервисов.
- `failover.txt`: потеря одной API с восстановлением записи.
- `runtime.json`, `containers.json`: безопасная сводка Docker VM, лимитов и volumes.
Матрица не измеряет production SLA. Для ускоренной lease/cleanup/retry-проверки явно указано изменение тестовых часов/расписания; отдельный lifecycle использует настоящий lease.
+11
View File
@@ -0,0 +1,11 @@
{
"status": "PASS",
"accepted_while_broker_down": 100,
"next_upload_status": 429,
"messages_available": true,
"all_ready": 100,
"admission_seconds": 2.997,
"drain_seconds": 24.64,
"lost_accepted": 0,
"profile": "100 PNG 320x200, 8 upload clients, 1 serial worker; closed-loop; not a throughput benchmark"
}
+11
View File
@@ -0,0 +1,11 @@
cache_key_exists=True
ttl_seconds=3599
avg_latency_ms=1.90
p95_latency_ms=2.60
stale_cache_rejected=True
forged_identity_rejected=True
concurrent_identity_checks=8/8
malformed_cache_recovers=True
repopulated_revoked_cache_rejected=True
short_ttl_recovers_from_db=True
logout_revokes_cache=True
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+23
View File
@@ -0,0 +1,23 @@
{
"status": "PASS",
"rows": [
{
"case": "real_child_timeout_with_test_budget_50ms",
"status": "PASS",
"elapsed_ms": 68.024
},
{
"case": "real_child_address_space_limit_32MiB",
"status": "PASS",
"error_code": "resource_limit"
},
{
"case": "staging_dry_run_apply_repeat",
"status": "PASS",
"candidates": 23,
"repeat_candidates": 0,
"active_retained": true,
"clock": "advanced_2_days"
}
]
}
+197
View File
@@ -0,0 +1,197 @@
[
{
"name": "/mtusi-lab5-api-1",
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
"hostname": "a6c7dcf30787",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:31:20.56568455Z",
"health": "healthy",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-api-2",
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
"hostname": "d43ff1cbdbbc",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:31:20.41170375Z",
"health": "healthy",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-broker-1",
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
"hostname": "43400a668650",
"service": "broker",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:31:28.257499705Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_broker_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-db-1",
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
"hostname": "f20a9e4d8783",
"service": "db",
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
"status": "running",
"started_at": "2026-11-20T15:27:51.02995726Z",
"health": "healthy",
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_postgres_data",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"name": "/mtusi-lab5-janitor-1",
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
"hostname": "8ec2d9f5b984",
"service": "janitor",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.911355944Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-lb-1",
"id": "14adfe35371f51e2272c03a8a300f62311ff31ba28236048a75e12d648e6fd82",
"hostname": "14adfe35371f",
"service": "lb",
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
"status": "running",
"started_at": "2026-11-20T15:31:14.230730447Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 67108864,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8085"
}
]
},
"volumes": []
},
{
"name": "/mtusi-lab5-migrate-1",
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
"hostname": "c265f53ce7e9",
"service": "migrate",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "exited",
"started_at": "2026-11-20T15:32:07.768599294Z",
"health": null,
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-publisher-1",
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
"hostname": "acdc5d34e3b0",
"service": "publisher",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:31:38.59219456Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-s3-1",
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
"hostname": "554bcd156ad2",
"service": "s3",
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
"status": "running",
"started_at": "2026-11-20T15:27:51.22311738Z",
"health": null,
"nano_cpus": 500000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_s3_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-s3-init-1",
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
"hostname": "d7a59a310290",
"service": "s3-init",
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
"status": "exited",
"started_at": "2026-11-20T15:32:07.290372244Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-valkey-1",
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
"hostname": "8c620bff1a6c",
"service": "valkey",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:27:50.863364684Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 134217728,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_valkey_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-worker-1",
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
"hostname": "72ecae4e2677",
"service": "worker",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:31:38.594433513Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": []
}
]
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8085"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 2, "first_recovery_seconds": 2.171}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+6
View File
@@ -0,0 +1,6 @@
scenario,repeats,mean_ms,min_ms,max_ms,stdev_ms,object_sets,objects_per_current_version,failures
before_effect,3,1550.718,1321.347,1786.888,232.845,1,4,0
after_effect,3,1549.527,1334.593,1872.025,284.404,1,4,0
after_commit,3,1383.915,1275.246,1590.414,178.915,1,4,0
concurrent_duplicates,3,143.795,121.12,172.849,26.448,1,4,0
reorder_stale_version,3,223.21,109.191,440.476,188.235,1,4,0
1 scenario repeats mean_ms min_ms max_ms stdev_ms object_sets objects_per_current_version failures
2 before_effect 3 1550.718 1321.347 1786.888 232.845 1 4 0
3 after_effect 3 1549.527 1334.593 1872.025 284.404 1 4 0
4 after_commit 3 1383.915 1275.246 1590.414 178.915 1 4 0
5 concurrent_duplicates 3 143.795 121.12 172.849 26.448 1 4 0
6 reorder_stale_version 3 223.21 109.191 440.476 188.235 1 4 0
+1
View File
@@ -0,0 +1 @@
{"parallel_requests": 16, "observed_instances": ["46b8beae40b9", "a684e742220a"], "unique_messages": 1, "conflict": 409, "semantic_replay": true, "user_scope": true}
+81
View File
@@ -0,0 +1,81 @@
{
"status": "PASS",
"rows": [
{
"case": "small_alpha",
"status": "PASS",
"dimensions": {
"thumbnail": [
80,
40
],
"crop": [
128,
128
],
"optimized": [
80,
40
]
},
"original_sha256": "2a4135aed00454b2566994964a2efffd9fd5593454d4d2f5b102aa3c42b2d3bd"
},
{
"case": "exif_orientation",
"status": "PASS",
"dimensions": {
"thumbnail": [
64,
256
],
"crop": [
128,
128
],
"optimized": [
100,
400
]
},
"original_sha256": "43ba669d1496cb5857d62be714452e087d474b2421f70f3cba3264dca491d026"
},
{
"case": "center_crop",
"status": "PASS",
"dimensions": {
"thumbnail": [
256,
64
],
"crop": [
128,
128
],
"optimized": [
400,
100
]
},
"original_sha256": "983bf89131f785c45b587c57640d62d223af08bfc71fcf28a27faed486af9583"
},
{
"case": "large_fit",
"status": "PASS",
"dimensions": {
"thumbnail": [
256,
128
],
"crop": [
128,
128
],
"optimized": [
1280,
640
]
},
"original_sha256": "fd1aebd1ed7ce3d142d0e38b7dc2d3b6c106a71d399e6fad25706c1f3ad8850b"
}
]
}
+67
View File
@@ -0,0 +1,67 @@
{
"status": "PASS",
"rows": [
{
"scenario": "stopped_worker_api_kill_outbox_broker_restart",
"status": "PASS",
"attachment_id": "882900a6-2551-4975-90ff-1a1ca424b0ee",
"stream_length_before": 1,
"stream_length_after": 1,
"ready_ms": 3023.094,
"containers_before": {
"mtusi-lab5-api-1": {
"id": "a6c7dcf30787",
"started_at": "2026-11-20T15:28:14.515508921Z",
"volumes": []
},
"mtusi-lab5-api-2": {
"id": "d43ff1cbdbbc",
"started_at": "2026-11-20T15:27:52.500502971Z",
"volumes": []
},
"mtusi-lab5-broker-1": {
"id": "43400a668650",
"started_at": "2026-11-20T15:27:50.923553697Z",
"volumes": [
"mtusi-lab5_broker_data"
]
}
},
"containers_after": {
"mtusi-lab5-api-1": {
"id": "a6c7dcf30787",
"started_at": "2026-11-20T15:31:20.56568455Z",
"volumes": []
},
"mtusi-lab5-api-2": {
"id": "d43ff1cbdbbc",
"started_at": "2026-11-20T15:31:20.41170375Z",
"volumes": []
},
"mtusi-lab5-broker-1": {
"id": "43400a668650",
"started_at": "2026-11-20T15:31:28.257499705Z",
"volumes": [
"mtusi-lab5_broker_data"
]
}
}
},
{
"scenario": "crash_real_lease_expiry",
"status": "PASS",
"attachment_id": "4c7827d8-b5f2-4b58-a019-a5d26099c12b",
"ready_ms": 28922.739,
"job": {
"job_id": "12c2c00c-8114-4c19-a9c6-2b4ffd42bd29",
"state": "ready",
"attempt": 2,
"lease_until": null,
"pipeline_version": 1,
"published_at": "2026-11-20 15:32:05.409533+00:00",
"deliveries": 2
},
"lease_clock": "real"
}
]
}
+197
View File
@@ -0,0 +1,197 @@
[
{
"name": "/mtusi-lab5-api-1",
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
"hostname": "a6c7dcf30787",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.845602697Z",
"health": "starting",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-api-2",
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
"hostname": "d43ff1cbdbbc",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.500502971Z",
"health": "starting",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-broker-1",
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
"hostname": "43400a668650",
"service": "broker",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:27:50.923553697Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_broker_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-db-1",
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
"hostname": "f20a9e4d8783",
"service": "db",
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
"status": "running",
"started_at": "2026-11-20T15:27:51.02995726Z",
"health": "healthy",
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_postgres_data",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"name": "/mtusi-lab5-janitor-1",
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
"hostname": "8ec2d9f5b984",
"service": "janitor",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.911355944Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-lb-1",
"id": "65e0d941c1b56b64cfc79d597ded2d8a64f8aa32ce378e6f3c737525e7ab4f37",
"hostname": "65e0d941c1b5",
"service": "lb",
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
"status": "running",
"started_at": "2026-11-20T15:27:51.271530136Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 67108864,
"published_ports": {
"8080/tcp": [
{
"HostIp": "",
"HostPort": "8085"
}
]
},
"volumes": []
},
{
"name": "/mtusi-lab5-migrate-1",
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
"hostname": "c265f53ce7e9",
"service": "migrate",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "exited",
"started_at": "2026-11-20T15:27:36.704304537Z",
"health": null,
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-publisher-1",
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
"hostname": "acdc5d34e3b0",
"service": "publisher",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.978480354Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-s3-1",
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
"hostname": "554bcd156ad2",
"service": "s3",
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
"status": "running",
"started_at": "2026-11-20T15:27:51.22311738Z",
"health": null,
"nano_cpus": 500000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_s3_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-s3-init-1",
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
"hostname": "d7a59a310290",
"service": "s3-init",
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
"status": "exited",
"started_at": "2026-11-20T15:27:36.140115508Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-valkey-1",
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
"hostname": "8c620bff1a6c",
"service": "valkey",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:27:50.863364684Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 134217728,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_valkey_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-worker-1",
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
"hostname": "72ecae4e2677",
"service": "worker",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:52.000472553Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": []
}
]
+197
View File
@@ -0,0 +1,197 @@
[
{
"name": "/mtusi-lab5-api-1",
"id": "a6c7dcf307879d09ac07b0b887033559137681728328ba576f97fda75b221f10",
"hostname": "a6c7dcf30787",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:26:31.479663938Z",
"health": "healthy",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-api-2",
"id": "d43ff1cbdbbc9743e6702bf486fd2b08565d16938a1c43cd5f3580a8319fb951",
"hostname": "d43ff1cbdbbc",
"service": "api",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:26:31.171165305Z",
"health": "healthy",
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-broker-1",
"id": "43400a668650580ec394d0355b5d57094552b72ac2045c469024a488fae8819b",
"hostname": "43400a668650",
"service": "broker",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:25:26.510562536Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_broker_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-db-1",
"id": "f20a9e4d87839b3be31ca6c91db0b948689899ea06a88df9efa4d4d1ba6791d4",
"hostname": "f20a9e4d8783",
"service": "db",
"image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad",
"status": "running",
"started_at": "2026-11-20T15:18:03.662849926Z",
"health": "healthy",
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_postgres_data",
"destination": "/var/lib/postgresql/data"
}
]
},
{
"name": "/mtusi-lab5-janitor-1",
"id": "8ec2d9f5b984f65c1208a23cfa3605f764fef9f62af2173f494b6fbf0b67f4a6",
"hostname": "8ec2d9f5b984",
"service": "janitor",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:26:31.168446315Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-lb-1",
"id": "65e0d941c1b56b64cfc79d597ded2d8a64f8aa32ce378e6f3c737525e7ab4f37",
"hostname": "65e0d941c1b5",
"service": "lb",
"image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a",
"status": "running",
"started_at": "2026-11-20T15:26:31.646228277Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 67108864,
"published_ports": {
"8080/tcp": [
{
"HostIp": "",
"HostPort": "8085"
}
]
},
"volumes": []
},
{
"name": "/mtusi-lab5-migrate-1",
"id": "c265f53ce7e970448fe93e6cc543d2256e025efc3fb28f5b19757bafaa46f61a",
"hostname": "c265f53ce7e9",
"service": "migrate",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "exited",
"started_at": "2026-11-20T15:27:36.704304537Z",
"health": null,
"nano_cpus": 2000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-publisher-1",
"id": "acdc5d34e3b03523ae5209ee57234d257e370740fef663745d9981bc1744ee5a",
"hostname": "acdc5d34e3b0",
"service": "publisher",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:38.479168598Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 201326592,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-s3-1",
"id": "554bcd156ad20d89a0c2b1f8d57e17fb825e4d1af032553114f74f5a82a752d1",
"hostname": "554bcd156ad2",
"service": "s3",
"image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253",
"status": "running",
"started_at": "2026-11-20T15:18:03.664833261Z",
"health": null,
"nano_cpus": 500000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_s3_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-s3-init-1",
"id": "d7a59a3102909606b2c42b96efbeb186573aed665c4ff2b0eb4a6a4975b37759",
"hostname": "d7a59a310290",
"service": "s3-init",
"image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1",
"status": "exited",
"started_at": "2026-11-20T15:27:36.140115508Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 536870912,
"published_ports": {},
"volumes": []
},
{
"name": "/mtusi-lab5-valkey-1",
"id": "8c620bff1a6cf99ddfe2799e69b51943c76b0be02039dda64fd34d0a62f4ab83",
"hostname": "8c620bff1a6c",
"service": "valkey",
"image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866",
"status": "running",
"started_at": "2026-11-20T15:18:03.65752301Z",
"health": null,
"nano_cpus": 250000000,
"memory_bytes": 134217728,
"published_ports": {},
"volumes": [
{
"name": "mtusi-lab5_valkey_data",
"destination": "/data"
}
]
},
{
"name": "/mtusi-lab5-worker-1",
"id": "72ecae4e267759c0dccfbf0bd9244852c4da3942dacd70c9b5cc11cee5a0d0e7",
"hostname": "72ecae4e2677",
"service": "worker",
"image_id": "sha256:340a6a6f74699a98e63e9725282594990013b479be7ae533fddf40cbab866ada",
"status": "running",
"started_at": "2026-11-20T15:27:38.475722085Z",
"health": null,
"nano_cpus": 1000000000,
"memory_bytes": 805306368,
"published_ports": {},
"volumes": []
}
]
+76
View File
@@ -0,0 +1,76 @@
{
"status": "PASS",
"running_processes_restarted": 10,
"rows": [
{
"name": "/mtusi-lab5-api-1",
"service": "api",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-api-2",
"service": "api",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-broker-1",
"service": "broker",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-db-1",
"service": "db",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-janitor-1",
"service": "janitor",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-lb-1",
"service": "lb",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-publisher-1",
"service": "publisher",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-s3-1",
"service": "s3",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-valkey-1",
"service": "valkey",
"same_container": true,
"process_restarted": true,
"same_volumes": true
},
{
"name": "/mtusi-lab5-worker-1",
"service": "worker",
"same_container": true,
"process_restarted": true,
"same_volumes": true
}
]
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8085"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+213
View File
@@ -0,0 +1,213 @@
{
"status": "PASS",
"track": 2,
"matrix_repeats": 3,
"rows": [
{
"iteration": 1,
"scenario": "before_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1786.888,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 1,
"scenario": "after_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1441.962,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 1,
"scenario": "after_commit",
"status": "PASS",
"attempts": 1,
"recovery_ms": 1286.085,
"objects": 4,
"lease_expiry": "not_needed"
},
{
"iteration": 1,
"scenario": "concurrent_duplicates",
"status": "PASS",
"deliveries": 8,
"effects": 1,
"recovery_ms": 137.417,
"outcomes": [
"ready",
"defer",
"defer",
"defer",
"defer",
"defer",
"defer",
"defer"
],
"objects": 4
},
{
"iteration": 1,
"scenario": "reorder_stale_version",
"status": "PASS",
"old_deliveries": 5,
"current_version": 2,
"old_objects": 0,
"objects": 4,
"recovery_ms": 440.476
},
{
"iteration": 2,
"scenario": "before_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1543.92,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 2,
"scenario": "after_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1872.025,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 2,
"scenario": "after_commit",
"status": "PASS",
"attempts": 1,
"recovery_ms": 1590.414,
"objects": 4,
"lease_expiry": "not_needed"
},
{
"iteration": 2,
"scenario": "concurrent_duplicates",
"status": "PASS",
"deliveries": 8,
"effects": 1,
"recovery_ms": 172.849,
"outcomes": [
"ready",
"defer",
"defer",
"defer",
"defer",
"defer",
"defer",
"defer"
],
"objects": 4
},
{
"iteration": 2,
"scenario": "reorder_stale_version",
"status": "PASS",
"old_deliveries": 5,
"current_version": 2,
"old_objects": 0,
"objects": 4,
"recovery_ms": 109.191
},
{
"iteration": 3,
"scenario": "before_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1321.347,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 3,
"scenario": "after_effect",
"status": "PASS",
"attempts": 2,
"recovery_ms": 1334.593,
"objects": 4,
"lease_expiry": "accelerated"
},
{
"iteration": 3,
"scenario": "after_commit",
"status": "PASS",
"attempts": 1,
"recovery_ms": 1275.246,
"objects": 4,
"lease_expiry": "not_needed"
},
{
"iteration": 3,
"scenario": "concurrent_duplicates",
"status": "PASS",
"deliveries": 8,
"effects": 1,
"recovery_ms": 121.12,
"outcomes": [
"defer",
"ready",
"defer",
"defer",
"defer",
"defer",
"defer",
"defer"
],
"objects": 4
},
{
"iteration": 3,
"scenario": "reorder_stale_version",
"status": "PASS",
"old_deliveries": 5,
"current_version": 2,
"old_objects": 0,
"objects": 4,
"recovery_ms": 119.964
},
{
"scenario": "published_v1_then_v2_then_stale_v1",
"status": "PASS",
"current_version": 2,
"objects": 4
},
{
"scenario": "permanent_failed_replay",
"status": "PASS",
"objects": 4
},
{
"scenario": "bounded_retry_jitter",
"status": "PASS",
"max_attempts": 5,
"delays_seconds": [
1.080882,
2.447441,
2.182634,
8.30795
]
},
{
"scenario": "outbox_republish",
"status": "PASS",
"objects": 4
}
],
"accepted_in_database": 151,
"invariants": {
"accepted_tasks_lost": 0,
"duplicate_variant_sets": 0,
"stale_version_overwrites": 0
},
"limitations": [
"matrix lease expiry accelerated with SQL",
"single host, worker processing is serial except intentional duplicate probe",
"no disk-loss or network partition proof"
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 86 KiB

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 62 KiB

+16
View File
@@ -0,0 +1,16 @@
{
"status": "PASS",
"anonymous_bucket_and_object_status": [
403,
403
],
"application_admin_operations_denied": [
"create_bucket",
"get_bucket_policy",
"write_outside_prefix"
],
"visible_bucket_names": [
"messenger-attachments"
],
"console_reachable": false
}
+1
View File
@@ -0,0 +1 @@
PASS: healthy API replicas observed through LB
+8
View File
@@ -0,0 +1,8 @@
{
"ServerVersion": "27.4.0",
"Architecture": "aarch64",
"OSType": "linux",
"NCPU": 4,
"MemTotal": 8308158464,
"KernelVersion": "6.8.0-50-generic"
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8085"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+90
View File
@@ -0,0 +1,90 @@
{
"status": "PASS",
"cases": [
{
"case": "corrupt_png",
"status": 415
},
{
"case": "truncated_png",
"status": 415
},
{
"case": "mime_spoof",
"status": 415
},
{
"case": "unsupported_mime",
"status": 415
},
{
"case": "empty_payload",
"status": 415
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "unsafe_filename",
"status": 400
},
{
"case": "foreign_chat_upload",
"status": 404
},
{
"case": "unauthenticated_upload",
"status": 401
},
{
"case": "over_40_million_pixels",
"status": 413
},
{
"case": "oversized_content_length_before_body",
"status": 413
},
{
"case": "chunked_oversize",
"status": 413
},
{
"case": "foreign_attachment_metadata_and_bytes",
"status": 404
},
{
"case": "cross_chat_and_unknown_attachment",
"status": 404
},
{
"case": "attachment_idempotency_and_conflict",
"status": "201/409"
},
{
"case": "truncated_jpeg",
"status": 415
},
{
"case": "interrupted_http_body",
"metadata_or_object_added": false
}
],
"count": 21
}
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+7 -4
View File
@@ -1,7 +1,10 @@
# Инфраструктура студента
# Инфраструктура решения
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Главный файл — [compose.yaml](../compose.yaml). Все сервисы и volumes принадлежат проекту `mtusi-lab5`; наружу опубликован только HAProxy на `127.0.0.1:8085`.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
- `haproxy.cfg`: round-robin между API, health checks `/health/ready`, обновление адресов через Docker DNS.
- `init-s3.sh`: повторяемая инициализация приватного bucket и отдельной application-учётной записи. Административные credentials есть только у MinIO/init.
- `api-policy.json`: list своего bucket и get/put/delete только `originals/*` для совместимости, `staging/*`, `results/*`; административных прав нет.
- `local.env.example`: пример фактических параметров. Секреты генерируются `scripts/bootstrap.sh` и хранятся в игнорируемом `.env`.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
`broker` и `valkey` — отдельные контейнеры: очередь Streams с AOF appendfsync=always и кэш сессий. У PostgreSQL, обоих Valkey и MinIO свои persistent volumes. Все зависимости запускаются из этого репозитория.
+7
View File
@@ -0,0 +1,7 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},
{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*","arn:aws:s3:::messenger-attachments/staging/*","arn:aws:s3:::messenger-attachments/results/*"]}
]
}
+31
View File
@@ -0,0 +1,31 @@
global
log stdout format raw local0
maxconn 512
defaults
log global
mode http
timeout connect 1s
timeout client 45s
timeout server 45s
timeout check 1s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
frontend public
bind :8080
default_backend replicas
backend replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
server-template api 8 api:8080 check
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
ready=false
for attempt in $(seq 1 60); do
if mc alias set local http://s3:9000 "$S3_ROOT_USER" "$S3_ROOT_PASSWORD" >/dev/null 2>&1; then
ready=true
break
fi
sleep 1
done
[ "$ready" = true ] || exit 1
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$S3_API_USER" "$S3_API_PASSWORD" >/dev/null
mc admin policy create local lab4-attachments /bootstrap/api-policy.json >/dev/null
mc admin policy attach local lab4-attachments --user "$S3_API_USER" >/dev/null
printf '%s\n' 'PASS: private bucket and restricted application principal configured'
+12
View File
@@ -0,0 +1,12 @@
# scripts/bootstrap.sh generates these three secrets into ignored .env.
POSTGRES_PASSWORD=replace-with-random-secret
S3_ROOT_PASSWORD=replace-with-random-secret
S3_API_PASSWORD=replace-with-random-secret
APP_PORT=8085
API_REPLICAS=2
API_CPUS=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=30
MAX_PENDING_JOBS=100
STAGING_RETENTION_SECONDS=86400
+48
View File
@@ -0,0 +1,48 @@
"""Conservative staging cleanup; all current queued/processing jobs are retained."""
import argparse
import json
import os
import signal
import threading
from datetime import datetime,timezone
from psycopg.rows import dict_row
from app import store,object_store
stop=threading.Event()
def sweep(apply=False,retention=None):
retention=max(60,int(retention or os.getenv('STAGING_RETENTION_SECONDS','86400')))
deleted=[]
pages=object_store.client.get_paginator('list_objects_v2').paginate(Bucket=object_store.bucket,Prefix='staging/')
for page in pages:
for item in page.get('Contents',[]):
if (datetime.now(timezone.utc)-item['LastModified']).total_seconds()<retention:continue
# Admission holds this same lock from S3 put through DB COMMIT.
with store.connect() as conn:
conn.execute('SELECT pg_advisory_xact_lock(202605)')
rows=conn.cursor(row_factory=dict_row).execute('SELECT id,state,staging_key FROM attachments WHERE staging_key=%s FOR UPDATE',(item['Key'],)).fetchall()
if any(row['state'] in ('queued','processing','pending') for row in rows):continue
head=object_store.client.head_object(Bucket=object_store.bucket,Key=item['Key'])
if (datetime.now(timezone.utc)-head['LastModified']).total_seconds()<retention:continue
if apply:
object_store.client.delete_object(Bucket=object_store.bucket,Key=item['Key'])
conn.execute('UPDATE attachments SET staging_key=NULL WHERE staging_key=%s',(item['Key'],))
deleted.append(item['Key'])
return {'apply':apply,'retention_seconds':retention,'staging_candidates':deleted}
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--once',action='store_true');p.add_argument('--apply',action='store_true');args=p.parse_args()
store.pool.open(wait=True,timeout=30)
for sig in (signal.SIGINT,signal.SIGTERM):signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:print(json.dumps(sweep(args.apply or not args.once)),flush=True)
except Exception as exc:
print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True)
if args.once:raise
if args.once:break
stop.wait(60)
finally:
store.pool.close();object_store.close()
+19
View File
@@ -0,0 +1,19 @@
"""Trusted operator CLI, no public replay or arbitrary job submission endpoint."""
import argparse
import json
from psycopg.rows import dict_row
from app import store
from pipeline import replay,upgrade
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['failed','replay','upgrade']);p.add_argument('id',nargs='?');args=p.parse_args()
store.pool.open(wait=True,timeout=30)
try:
if args.action=='failed':
with store.connect() as conn:
rows=conn.cursor(row_factory=dict_row).execute("SELECT id,attachment_id,pipeline_version,attempt,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at LIMIT 100").fetchall()
print(json.dumps(rows,default=str,indent=2))
elif not args.id:p.error('id is required')
elif args.action=='replay':replay(store,args.id);print('requeued')
else:print(upgrade(store,args.id))
finally:store.pool.close()
+8
View File
@@ -0,0 +1,8 @@
from app import store
if __name__ == '__main__':
store.pool.open(wait=True, timeout=30)
try:
store.init()
finally:
store.pool.close()
+41
View File
@@ -0,0 +1,41 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS sessions (
id UUID PRIMARY KEY,
token TEXT NOT NULL UNIQUE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages(chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_chat_members_user ON chat_members(user_id, chat_id);
CREATE INDEX IF NOT EXISTS idx_chats_created ON chats(created_at, id);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE message_requests (
sender_id UUID NOT NULL REFERENCES users(id),
chat_id UUID NOT NULL REFERENCES chats(id),
key VARCHAR(128) NOT NULL,
body TEXT NOT NULL,
response JSONB NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY(sender_id, chat_id, key)
);
CREATE INDEX message_requests_expiry ON message_requests(expires_at);
+27
View File
@@ -0,0 +1,27 @@
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
object_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type TEXT NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 CHAR(64) NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX attachments_state_age ON attachments(state, updated_at);
CREATE TABLE message_attachments (
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments(id),
position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id, attachment_id),
UNIQUE(message_id, position)
);
CREATE INDEX message_attachments_attachment ON message_attachments(attachment_id);
-- Preserve replay of lab3 text requests if an existing lab3 DB is upgraded.
ALTER TABLE message_requests ALTER COLUMN body TYPE JSONB
USING jsonb_build_object('text', body, 'attachment_ids', '[]'::jsonb);
UPDATE message_requests SET response = response || '{"attachment_ids":[]}'::jsonb;
+36
View File
@@ -0,0 +1,36 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CONSTRAINT attachments_state_check CHECK(state IN ('pending','queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK(pipeline_version>0);
ALTER TABLE attachments ADD COLUMN staging_key TEXT;
ALTER TABLE attachments ADD COLUMN variants JSONB NOT NULL DEFAULT '[]';
CREATE TABLE jobs (
id UUID PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL CHECK(pipeline_version>0),
state TEXT NOT NULL CHECK(state IN ('queued','processing','ready','failed','superseded')),
attempt INTEGER NOT NULL DEFAULT 0,
lease_until TIMESTAMPTZ,
next_at TIMESTAMPTZ NOT NULL DEFAULT now(),
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE TABLE outbox (
job_id UUID PRIMARY KEY REFERENCES jobs(id),
published_at TIMESTAMPTZ,
deliveries INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE job_audit (
id BIGSERIAL PRIMARY KEY,
job_id UUID REFERENCES jobs(id),
event TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX jobs_due ON jobs(state,next_at);
-- Upgrade existing lab4 ready originals through the asynchronous pipeline.
UPDATE attachments SET staging_key=object_key,state='queued',updated_at=now() WHERE state IN ('ready','pending');
INSERT INTO jobs(id,attachment_id,pipeline_version,state)
SELECT gen_random_uuid(),id,pipeline_version,'queued' FROM attachments WHERE state='queued';
INSERT INTO outbox(job_id) SELECT id FROM jobs;
+68
View File
@@ -0,0 +1,68 @@
"""Database authority, version fencing and safe operator actions."""
import json
import os
import uuid
from psycopg.rows import dict_row
import redis
STREAM = 'attachments.v1'
GROUP = 'image-workers'
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '20')))
LEASE = max(TIMEOUT + 5, int(os.getenv('JOB_LEASE_SECONDS', '30')))
def broker():
return redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
def create_job(conn, aid, version):
jid = str(uuid.uuid4())
conn.execute("INSERT INTO jobs(id,attachment_id,pipeline_version,state) VALUES (%s,%s,%s,'queued')", (jid,aid,version))
conn.execute('INSERT INTO outbox(job_id) VALUES (%s)', (jid,))
audit(conn, jid, 'accepted')
return jid
def audit(conn, jid, event):
conn.execute('INSERT INTO job_audit(job_id,event) VALUES (%s,%s)', (jid,event))
def envelope(row):
return dict(schema_version=1,job_id=str(row['id']),type='attachment.process.v1',
attachment_id=str(row['attachment_id']),pipeline_version=row['pipeline_version'],
attempt=row['attempt']+1,created_at=row['created_at'].isoformat().replace('+00:00','Z'))
def fetch_job(conn, jid, suffix=''):
return conn.cursor(row_factory=dict_row).execute('SELECT * FROM jobs WHERE id=%s '+suffix, (jid,)).fetchone()
def replay(store, jid):
with store.connect() as conn:
job = fetch_job(conn,jid,'FOR UPDATE')
if job is None:
raise ValueError('unknown job')
att = conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (job['attachment_id'],)).fetchone()
if job['state'] != 'failed' or att['pipeline_version'] != job['pipeline_version'] or not att['staging_key']:
raise ValueError('only current failed job with retained staging can be replayed')
conn.execute("UPDATE jobs SET state='queued',attempt=0,error_code=NULL,next_at=now(),lease_until=NULL WHERE id=%s",(jid,))
conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=now() WHERE id=%s",(att['id'],))
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
audit(conn,jid,'operator_replay')
def upgrade(store, aid):
# Only trusted CLI; never accept version/storage paths from HTTP clients.
with store.connect() as conn:
# Lock jobs before attachment, same order as the worker.
conn.execute('SELECT id FROM jobs WHERE attachment_id=%s ORDER BY pipeline_version FOR UPDATE',(aid,)).fetchall()
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(aid,)).fetchone()
if att is None or (att['state'] != 'ready' and not att['staging_key']):
raise ValueError('upgrade requires durable source')
version=att['pipeline_version']+1
# A completed immutable original is itself a durable source for upgrade.
conn.execute("UPDATE attachments SET pipeline_version=%s,staging_key=CASE WHEN state='ready' THEN object_key ELSE staging_key END,state='queued',variants='[]',error_code=NULL,updated_at=now() WHERE id=%s",(version,aid))
conn.execute("UPDATE jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",(aid,))
return create_job(conn,aid,version)
+45
View File
@@ -0,0 +1,45 @@
"""Recoverable DB outbox -> persistent Valkey Streams (at least once)."""
import argparse
import json
import signal
import threading
from psycopg.rows import dict_row
from app import store
from pipeline import broker, envelope, STREAM
stop=threading.Event()
def publish_once(client):
with store.connect() as conn:
# Marking published happens AFTER XADD. A crash in between produces a
# duplicate. DB remains authoritative; no acknowledged intent is lost.
rows=conn.cursor(row_factory=dict_row).execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
WHERE j.state IN ('queued','processing') AND j.next_at<=now()
AND (o.published_at IS NULL OR (j.state='processing' AND j.lease_until<now()
AND o.published_at<now()-interval '5 seconds'))
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
for row in rows:
client.xadd(STREAM,{'job':json.dumps(envelope(row))})
conn.execute('UPDATE outbox SET published_at=now(),deliveries=deliveries+1 WHERE job_id=%s',(row['id'],))
return len(rows)
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--once',action='store_true');args=parser.parse_args()
store.pool.open(wait=True,timeout=30)
client=broker()
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:
count=publish_once(client)
if count:print(json.dumps({'published':count}),flush=True)
except Exception as exc:
print(json.dumps({'publisher_error':type(exc).__name__}),flush=True)
if args.once:raise
if args.once:break
stop.wait(.25)
finally:
client.close();store.pool.close()
+2
View File
@@ -0,0 +1,2 @@
-r requirements-dev.txt
matplotlib==3.10.0
+9
View File
@@ -0,0 +1,9 @@
fastapi==0.115.0
uvicorn==0.30.6
psycopg[binary]==3.3.6
psycopg-pool==3.2.6
argon2-cffi==23.1.0
python-dotenv==1.0.1
redis==5.1.0
boto3==1.42.0
Pillow==13.0.0
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
umask 077
python3 - <<'PY' > .env
import secrets
for name in ('POSTGRES_PASSWORD','S3_ROOT_PASSWORD','S3_API_PASSWORD'):
print(name+'='+secrets.token_hex(24))
print('APP_PORT=8085')
PY
fi
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if docker compose version >/dev/null 2>&1; then
exec docker compose "$@"
fi
exec docker-compose "$@"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
"""Sanitized real-container evidence. Never dump environment or labels wholesale."""
import json
from pathlib import Path
import subprocess
ROOT=Path(__file__).resolve().parents[1]
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-a','-q'],cwd=ROOT,text=True).split()
result=[]
for item in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)):
result.append({'name':item['Name'],'id':item['Id'],'hostname':item['Config']['Hostname'],
'service':item['Config']['Labels']['com.docker.compose.service'],
'image_id':item['Image'],'status':item['State']['Status'],
'started_at':item['State']['StartedAt'],
'health':item['State'].get('Health',{}).get('Status'),
'nano_cpus':item['HostConfig']['NanoCpus'],'memory_bytes':item['HostConfig']['Memory'],
'published_ports':item['HostConfig']['PortBindings'],
'volumes':[{'name':m['Name'],'destination':m['Destination']} for m in item['Mounts'] if m['Type']=='volume']})
print(json.dumps(sorted(result,key=lambda c:c['name']),indent=2))
+14
View File
@@ -0,0 +1,14 @@
import json
import sys
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from app import store
from psycopg.rows import dict_row
store.pool.open(wait=True,timeout=20)
try:
with store.connect() as conn:
rows=conn.cursor(row_factory=dict_row).execute('''SELECT j.id AS job_id,j.state,j.attempt,j.lease_until,
a.pipeline_version,o.published_at,o.deliveries FROM jobs j JOIN attachments a ON a.id=j.attachment_id
JOIN outbox o ON o.job_id=j.id WHERE a.id=%s ORDER BY j.pipeline_version''',(sys.argv[1],)).fetchall()
print(json.dumps(rows,default=str))
finally:store.pool.close()
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
trap 'scripts/compose.sh start publisher worker >/dev/null' EXIT
scripts/compose.sh stop worker publisher
scripts/compose.sh run --rm --no-deps ops python tests/pipeline_faults.py > evidence/pipeline-faults.json
scripts/compose.sh run --rm --no-deps ops python tests/cleanup_and_limits.py > evidence/cleanup-and-limits.json
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
scripts/compose.sh restart db valkey broker s3 api lb publisher worker janitor
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?Required API hostname from X-Instance-Id}"
for id in $(scripts/compose.sh ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 3 "$id" >/dev/null
exit 0
fi
done
printf '%s\n' 'Target is not a running API replica' >&2
exit 1
+29
View File
@@ -0,0 +1,29 @@
"""Synchronize operational result tables with real sanitized test artifacts."""
import json
from pathlib import Path
import re
ROOT=Path(__file__).resolve().parents[1];E=ROOT/'evidence'
load=lambda name:json.loads((E/name).read_text())
life=load('lifecycle.json')['rows'];back=load('backpressure.json');cleanup=load('cleanup-and-limits.json')['rows'];variants=load('image-variants.json')['rows']
faults=load('pipeline-faults.json')['rows'];verification=load('persistence-verified.json')
failover=json.loads(next(s for s in (E/'failover.txt').read_text().splitlines() if s.startswith('{')))
retry=next(r for r in faults if r['scenario']=='bounded_retry_jitter')
content='### Восстановление и ограничения\n\n| Воздействие | Реальное наблюдение | Результат |\n| --- | --- | --- |\n'
content+=f"| Остановка worker, SIGKILL API до publish, restart broker | Длина stream {life[0]['stream_length_before']} → {life[0]['stream_length_after']}; ready через {life[0]['ready_ms']/1000:.3f} с после запуска worker | PASS, исходник сохранён |\n"
content+=f"| Смерть worker после PUT, настоящий lease | Ready через {life[1]['ready_ms']/1000:.3f} с; попыток {life[1]['job']['attempt']} | PASS, без ускорения часов |\n"
content+=f"| Остановка broker и заполнение очереди | Принято {back['accepted_while_broker_down']}; следующий HTTP {back['next_upload_status']}; восстановлено {back['all_ready']} за {back['drain_seconds']} с | PASS, потеряно {back['lost_accepted']} |\n"
content+=f"| Перезапуск всего стенда | {verification['running_processes_restarted']} реальных процессов, прежние volumes и сессия | PASS |\n"
content+=f"| Отказ API | {failover['successful_writes']} успешных записей, {failover['failed_attempts']} переходных ошибок; первая успешная запись через {failover['first_recovery_seconds']} с | PASS |\n"
content+=f"| Тайм-аут decode, тестовый бюджет 50 мс | Время сценария {cleanup[0]['elapsed_ms']} мс; retry, затем ready | PASS |\n"
content+=f"| Ограничение адресного пространства, тестовый бюджет 32 MiB | Конечный {cleanup[1]['error_code']}, replay завершился | PASS |\n"
content+=f"| Staging cleanup, часы +2 дня | Кандидатов {cleanup[2]['candidates']}, повторных {cleanup[2]['repeat_candidates']}; активный исходник сохранён | PASS |\n"
content+='\n### Проверка геометрии реальных файлов\n\n| Исходник/сценарий | Thumbnail | Crop | Optimized |\n| --- | --- | --- | --- |\n'
for row in variants:
dims=lambda key:'×'.join(map(str,row['dimensions'][key]))
content+=f"| {row['case']} | {dims('thumbnail')} | {dims('crop')} | {dims('optimized')} |\n"
content+='\n### Наблюдаемые retry с jitter\n\n| После попытки | Сохранённая задержка до следующей, с |\n| --- | ---: |\n'
for i,delay in enumerate(retry['delays_seconds'],1):content+=f'| {i} | {delay:.6f} |\n'
content+=f"\nПосле {retry['max_attempts']}-й временной ошибки зафиксирован `retry_exhausted`; явный replay завершается ready. При проверке расписания next_at искусственно сдвигается, так что таблица показывает записанные интервалы, а не фактически выжданные паузы.\n\n"
p=ROOT/'REPORT.md';s=p.read_text();a=s.index('<!-- BEGIN OPERATIONAL RESULTS -->');b=s.index('<!-- END OPERATIONAL RESULTS -->',a)
p.write_text(s[:a]+'<!-- BEGIN OPERATIONAL RESULTS -->\n\n'+content+s[b:])
print('Synchronized REPORT.md operational, geometry and retry tables')
+80
View File
@@ -0,0 +1,80 @@
"""Generate research tables and static plots only from measured raw evidence."""
import csv
import json
import os
from pathlib import Path
import statistics
os.environ.setdefault('MPLCONFIGDIR','/tmp/mtusi-lab5-matplotlib')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT=Path(__file__).resolve().parents[1]
E=ROOT/'evidence'
data=json.loads((E/'pipeline-faults.json').read_text())
names=['before_effect','after_effect','after_commit','concurrent_duplicates','reorder_stale_version']
labels=['Crash before PUT','Crash after PUT','Crash after COMMIT','8 duplicate jobs','Old job after v2']
summary=[]
for name in names:
rows=[r for r in data['rows'] if r['scenario']==name]
times=[r['recovery_ms'] for r in rows]
summary.append(dict(scenario=name,repeats=len(rows),mean_ms=round(statistics.mean(times),3),
min_ms=min(times),max_ms=max(times),stdev_ms=round(statistics.stdev(times),3),
object_sets=1,objects_per_current_version=4,failures=sum(r['status']!='PASS' for r in rows)))
with (E/'fault-summary.csv').open('w') as out:
w=csv.DictWriter(out,fieldnames=list(summary[0]),lineterminator='\n');w.writeheader();w.writerows(summary)
fig,axes=plt.subplots(1,2,figsize=(13,4.8))
means=[r['mean_ms'] for r in summary]
axes[0].bar(labels,means,color='#2878b5',yerr=[r['stdev_ms'] for r in summary],capsize=4)
axes[0].set_ylabel('Scenario wall time, ms (mean +/- sample SD, n=3)')
axes[0].tick_params(axis='x',rotation=25)
axes[0].set_title('Process startup and accelerated lease recovery included')
axes[1].bar(labels,[r['objects_per_current_version'] for r in summary],color='#32a77b')
axes[1].axhline(4,color='black',linestyle='--',label='1 original + 3 variants')
axes[1].set_ylim(0,5);axes[1].set_ylabel('Objects in current version prefix')
axes[1].tick_params(axis='x',rotation=25);axes[1].set_title('No extra variant sets in tested scenarios');axes[1].legend()
fig.tight_layout();fig.savefig(E/'pipeline-faults.png',dpi=160);fig.savefig(E/'pipeline-faults.svg');plt.close(fig)
svg=E/'pipeline-faults.svg'
svg.write_text('\n'.join(line.rstrip() for line in svg.read_text().splitlines())+'\n')
table='| Сценарий | Повторы | Среднее, мс | min–max, мс | SD, мс | Объектов текущей версии | Ошибки |\n| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n'
translations=['Падение до PUT','Падение после PUT','Падение после COMMIT до ack','8 конкурентных дублей','Старая версия после новой']
for label,r in zip(translations,summary):
table+=f"| {label} | {r['repeats']} | {r['mean_ms']} | {r['min_ms']}–{r['max_ms']} | {r['stdev_ms']} | {r['objects_per_current_version']} | {r['failures']} |\n"
text='''# Исследование: уровень 5, трек 2
## Гипотеза и инварианты
Версионирование результатов, transactional outbox, блокировки PostgreSQL и повторяемые ключи S3 позволяют завершать принятые задачи при at-least-once доставке без дополнительных наборов вариантов и перезаписи нового результата старым.
Инварианты проверяются assertions: каждый проверяемый upload с 202 завершается ready; оригинал побайтно совпадает с принятым; у текущей версии ровно четыре объекта; метаданные и SHA-256 каждого варианта соответствуют скачанным байтам; повтор ready не меняет метаданные; устаревшая job не изменяет новую версию.
## Методика
`make research` останавливает обычные worker/publisher и запускает `tests/pipeline_faults.py` в ops. Создаются реальные HTTP-вложения и настоящие consumer-group доставки Valkey Streams. В отдельном процессе вызывается `os._exit(86)` до PUT, после всех PUT до COMMIT, после COMMIT до XACK. Неподтверждённая доставка затем захватывается другим consumer. Матрица содержит три повтора каждого из пяти основных сценариев, дополнительно проверяет смену уже опубликованной v1 на v2, failed/replay, пять временных ошибок и повтор публикации outbox.
Для ускорения матрицы lease переводится в прошлое SQL-командой; до этого отдельно проверяется отказ захвата действующего lease. Это не измерение реального времени ожидания lease. `tests/lifecycle.py` отдельно проверяет восстановление с настоящими часами, без изменения дедлайна. Конкурентные дубли запускаются восемью потоками, обычный worker последовательный. Время в таблице включает HTTP/БД, запуск Python-процессов и обработку; это время полного тестового сценария, а не latency одного изображения в production.
Один локальный Linux Docker Engine в Colima, arm64, 4 vCPU, около 8 GiB RAM. Лимиты: worker 1 CPU/768 MiB, API 2 реплики по 2 CPU/512 MiB, PostgreSQL 1 CPU/512 MiB, MinIO 0.5 CPU/768 MiB, broker 0.25 CPU/192 MiB; ops 1 CPU/512 MiB. Ограничения CPU — верхние пределы, а не выделенные ядра. Прочие лабораторные могут работать в этой же VM. Файл матрицы — PNG 320×200 из публичных fixtures. Отдельные геометрические тесты используют PNG/JPEG от 80×40 до 1600×800. Нагрузочного прогрева нет: исследуется корректность конкретных interleavings, не максимальный throughput. Точные лимиты и образы сохранены в `evidence/containers.json` и `compose.yaml`.
## Результаты матрицы
'''+table+'''
![Время сценариев и количество объектов](evidence/pipeline-faults.png)
После падения до COMMIT требуются две управляемые попытки. После COMMIT повтор не выполняет обработку: достаточно одной попытки. При восьми конкурентных доставках обрабатывает один поток, остальные видят занятость или завершённый результат. После новой версии старые задания не создают объекты v1 и не изменяют публичные метаданные v2. Отдельно проверено обновление уже существующего результата v1 на v2 с изменённым качеством кодирования.
Raw: [pipeline-faults.json](evidence/pipeline-faults.json), [CSV](evidence/fault-summary.csv), [lifecycle.json](evidence/lifecycle.json), [backpressure.json](evidence/backpressure.json), [cleanup-and-limits.json](evidence/cleanup-and-limits.json).
## Ограничения
Это проверка конечного набора аварийных расписаний, а не математическое доказательство для всех interleavings. Исследование не покрывает потерю диска/volume, несколько узлов, длительные сетевые разделения и повреждение файловой системы. Локальная матрица не подтверждает SLA и не измеряет p95 под производственной нагрузкой; для выбранного трека это не заявляется. CPU/RAM ограничены конфигурацией, пиковый RSS декодера в этом исследовании не измеряется.
Физический PUT может повториться после аварии: одинаковый ключ заменяется теми же данными. Гарантируется один текущий логический набор результатов, а доставка остаётся at least once. Объекты прежних успешно опубликованных версий сохраняются: это история версий, не дубли текущего результата. Изменение алгоритма должно сопровождаться увеличением pipeline_version; старый бинарный worker нужно вывести из эксплуатации перед изменением кода существующей версии.
'''
(ROOT/'RESEARCH.md').write_text(text)
report=ROOT/'REPORT.md'
if report.exists() and '<!-- BEGIN RESEARCH RESULTS -->' in report.read_text():
content=report.read_text();a=content.index('<!-- BEGIN RESEARCH RESULTS -->');b=content.index('<!-- END RESEARCH RESULTS -->',a)
content=content[:a]+'<!-- BEGIN RESEARCH RESULTS -->\n\n'+table+'\n![Исследование pipeline](evidence/pipeline-faults.png)\n\n'+content[b:]
report.write_text(content)
print('Generated RESEARCH.md, CSV, PNG/SVG and synchronized REPORT.md research table')
+15
View File
@@ -0,0 +1,15 @@
"""Check that the persistence hook actually restarted every running service."""
import json
from pathlib import Path
root=Path(__file__).resolve().parents[1]/'evidence'
before={c['name']:c for c in json.loads((root/'persistence-before.json').read_text()) if c['status']=='running'}
after={c['name']:c for c in json.loads((root/'persistence-after.json').read_text())}
rows=[]
for name,old in before.items():
new=after[name]
assert old['id']==new['id'],name+': container unexpectedly replaced'
assert old['started_at']!=new['started_at'],name+': process not restarted'
assert old['volumes']==new['volumes'],name+': volumes changed'
assert new['status']=='running',name+': not running'
rows.append(dict(name=name,service=new['service'],same_container=True,process_restarted=True,same_volumes=True))
print(json.dumps({'status':'PASS','running_processes_restarted':len(rows),'rows':rows},indent=2))
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
"""Wait for actual healthy Compose API containers and their public LB aliases."""
import argparse
import json
from pathlib import Path
import subprocess
import time
import urllib.request
ROOT=Path(__file__).resolve().parents[1]
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',default='http://localhost:8085')
parser.add_argument('--replicas',type=int,default=2)
args=parser.parse_args()
deadline=time.monotonic()+90
while time.monotonic()<deadline:
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-q','api'],cwd=ROOT,text=True).split()
if len(ids)==args.replicas:
state=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
if all(c['State'].get('Health',{}).get('Status')=='healthy' for c in state):
expected={c['Config']['Hostname'] for c in state}
try:
observed=set()
for _ in range(16):
with urllib.request.urlopen(args.base_url+'/health/ready',timeout=2) as response:
observed.add(response.headers.get('X-Instance-Id'))
if observed==expected:
print('PASS: healthy API replicas observed through LB')
break
except OSError:
pass
time.sleep(.5)
else:
raise SystemExit('API readiness/LB discovery exceeded 90 seconds')
+40
View File
@@ -0,0 +1,40 @@
"""Real broker outage and bounded admission, followed by complete recovery."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import time
from lifecycle import compose,probe
from client import Client,check
def main():
c=Client('http://localhost:8085');started=time.monotonic()
try:
compose('stop','worker','publisher','broker')
alice,bob,eve,chat=c.bootstrap()
# Default deployment admits at most 100 pending logical jobs across APIs.
def upload(_):return Client(c.base).attachment(alice,chat)
with ThreadPoolExecutor(max_workers=8) as pool:accepted=list(pool.map(upload,range(100)))
fixture=accepted[0][1]
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=full.png',429,actor=alice,raw=fixture,headers={'Content-Type':'image/png'})
c.message(alice,chat,'messages remain writable while image queue is full')
for att,_ in (accepted[0],accepted[-1]):
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
admitted_seconds=time.monotonic()-started
# Every 202 was accepted while the broker was genuinely stopped.
state=probe(accepted[0][0]['id'])
check(state['published_at'] is None and state['state']=='queued','upload depended on unavailable broker')
compose('start','broker','publisher','worker');recovery=time.monotonic()
for att,data in accepted:
ready=c.wait_ready(bob,att,wait_seconds=120)
check(len(ready['variants'])==3 and ready['sha256']==hashlib.sha256(data).hexdigest(),'recovery lost accepted metadata')
for att,data in (accepted[0],accepted[-1]):
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'recovery lost original bytes')
print(json.dumps({'status':'PASS','accepted_while_broker_down':len(accepted),'next_upload_status':429,
'messages_available':True,'all_ready':len(accepted),'admission_seconds':round(admitted_seconds,3),
'drain_seconds':round(time.monotonic()-recovery,3),'lost_accepted':0,
'profile':'100 PNG 320x200, 8 upload clients, 1 serial worker; closed-loop; not a throughput benchmark'},indent=2))
finally:compose('start','broker','publisher','worker')
if __name__=='__main__':main()
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Research track: cache consistency and stale session protection for Valkey-backed sessions."""
import base64
import json
import os
import statistics
import time
import urllib.error
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080")
CACHER_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0")
def http_json(method, path, *, expected=200, headers=None, body=None):
req_headers = {"Accept": "application/json"}
if headers:
req_headers.update(headers)
data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8")
if data is not None:
req_headers.setdefault("Content-Type", "application/json")
request = urllib.request.Request(BASE_URL + path, data=data, headers=req_headers, method=method)
try:
with urllib.request.urlopen(request, timeout=10) as resp:
payload = resp.read()
if not payload:
return {"status": resp.status, "headers": dict(resp.headers), "body": None}
if resp.status not in (expected if isinstance(expected, (list, tuple, set)) else [expected]):
raise AssertionError(f"Expected {expected}, got {resp.status}")
return {"status": resp.status, "headers": dict(resp.headers), "body": json.loads(payload.decode("utf-8"))}
except urllib.error.HTTPError as exc:
payload = exc.read()
if payload:
try:
payload = json.loads(payload.decode("utf-8"))
except Exception:
payload = payload.decode("utf-8", errors="replace")
if exc.code not in (expected if isinstance(expected, (list, tuple, set)) else [expected]):
raise AssertionError(f"Expected status {expected}, got {exc.code} for {method} {path}: {payload}")
return {"status": exc.code, "headers": dict(exc.headers), "body": payload}
def call_with_latency(method, path, *, expected=200, headers=None, body=None):
start = time.perf_counter()
result = http_json(method, path, expected=expected, headers=headers, body=body)
elapsed_ms = (time.perf_counter() - start) * 1000.0
return result, elapsed_ms
def main():
import redis
r = redis.Redis.from_url(CACHER_URL, decode_responses=True, socket_connect_timeout=2, socket_timeout=2)
username = f"cache_{uuid.uuid4().hex[:12]}"
password = "123456789012"
body = {
"username": username,
"display_name": "Cache Consistency",
"password": password,
}
registration = http_json("POST", "/api/v1/users", expected=201, body=body)
assert registration["status"] == 201, registration
basic = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
session = http_json("POST", "/api/v1/auth/sessions", expected=201, headers={"Authorization": basic})
token = session["body"]["token"]
assert token, session
key = f"session:{token}"
assert r.get(key), "session was not populated in Valkey"
ttl = r.ttl(key)
assert ttl and ttl > 0, f"session TTL must be positive, got {ttl}"
latencies = []
for _ in range(10):
_, elapsed_ms = call_with_latency("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
latencies.append(elapsed_ms)
avg = sum(latencies) / len(latencies)
p95 = sorted(latencies)[int(len(latencies) * 0.95) - 1]
stale_payload = {
"token": token,
"user_id": str(uuid.uuid4()),
"expires_at": "2000-01-01T00:00:00Z",
"revoked_at": None,
"user": {"id": str(uuid.uuid4()), "username": "stale", "display_name": "stale", "created_at": "2020-01-01T00:00:00Z"},
}
r.setex(key, 120, json.dumps(stale_payload, separators=(",", ":"), ensure_ascii=False))
stale_result = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
assert stale_result["status"] == 200, stale_result
assert stale_result["body"]["username"] == username, stale_result
# An unexpired payload must not be able to substitute another identity.
stale_payload["expires_at"] = "2100-01-01T00:00:00Z"
r.setex(key, 120, json.dumps(stale_payload))
forged = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})
assert forged["body"]["id"] == session["body"]["user"]["id"], forged
assert forged["body"]["username"] == username, forged
r.setex(key, 120, json.dumps(stale_payload))
def read_identity(_):
return http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})["body"]["id"]
with ThreadPoolExecutor(max_workers=8) as executor:
identities = list(executor.map(read_identity, range(8)))
assert identities == [session["body"]["user"]["id"]] * 8, identities
# Syntactically valid JSON with missing fields must be treated as a miss.
r.setex(key, 120, "{}")
malformed = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})
assert malformed["body"]["username"] == username, malformed
r.setex(key, 1, json.dumps({
"token": token,
"user_id": str(uuid.uuid4()),
"expires_at": "2100-01-01T00:00:00Z",
"revoked_at": None,
"user": {"id": str(uuid.uuid4()), "username": "ttl-recovery", "display_name": "ttl-recovery", "created_at": "2020-01-01T00:00:00Z"},
}, separators=(",", ":"), ensure_ascii=False))
time.sleep(2)
short_ttl = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token})
assert short_ttl["status"] == 200, short_ttl
assert short_ttl["body"]["username"] == username, short_ttl
revoke = http_json("DELETE", "/api/v1/auth/sessions/current", expected=204, headers={"Authorization": "Bearer " + token})
assert revoke["status"] == 204, revoke
assert not r.get(key), "revoked session key should be deleted from cache"
after_logout = http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token})
assert after_logout["status"] == 401, after_logout
# Simulate a writer repopulating the cache after logout's DELETE.
stale_payload.update(session["body"])
stale_payload["user_id"] = session["body"]["user"]["id"]
stale_payload["revoked_at"] = None
r.setex(key, 120, json.dumps(stale_payload))
http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token})
assert not r.exists(key), "stale key after logout must be removed"
print("cache_key_exists=True")
print(f"ttl_seconds={ttl}")
print(f"avg_latency_ms={avg:.2f}")
print(f"p95_latency_ms={p95:.2f}")
print("stale_cache_rejected=True")
print("forged_identity_rejected=True")
print("concurrent_identity_checks=8/8")
print("malformed_cache_recovers=True")
print("repopulated_revoked_cache_rejected=True")
print("short_ttl_recovers_from_db=True")
print("logout_revokes_cache=True")
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
"""Conservative cleanup and actual subprocess timeout boundary; worker stopped."""
from datetime import datetime,timedelta,timezone
import json
from pathlib import Path
import subprocess
import sys
import time
import unittest.mock as mock
import uuid
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,check
from app import store,object_store
from pipeline import envelope,fetch_job,replay
from worker import handle
from janitor import sweep
def main():
store.pool.open(wait=True,timeout=20)
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap()
def upload():
att,data=c.attachment(alice,chat)
with store.connect() as conn:
jid=conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0]
msg=envelope(fetch_job(conn,jid))
return att,msg
rows=[]
try:
att,msg=upload()
real_run=subprocess.run
def slow_decoder(*args,**kwargs):
return real_run([sys.executable,'-c','import time; time.sleep(2)'],timeout=.05,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
start=time.monotonic()
with mock.patch('worker.subprocess.run',side_effect=slow_decoder):
check(handle(msg)=='retry','decoder timeout not classified as transient')
elapsed=time.monotonic()-start
with store.connect() as conn:
job=fetch_job(conn,msg['job_id']);check(job['state']=='queued' and job['error_code']=='processing_timeout','timeout not persisted')
conn.execute('UPDATE jobs SET next_at=now() WHERE id=%s',(job['id'],))
check(handle(msg)=='ready','timeout recovery failed')
rows.append(dict(case='real_child_timeout_with_test_budget_50ms',status='PASS',elapsed_ms=round(elapsed*1000,3)))
# Validate memory-exit classification without risking an OOM on the host.
att2,msg2=upload()
def exhausted(*args,**kwargs):
return real_run([sys.executable,'-c',"import resource; resource.setrlimit(resource.RLIMIT_AS,(32*1024**2,32*1024**2));\ntry: x=bytearray(128*1024**2)\nexcept MemoryError: raise SystemExit(3)"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
with mock.patch('worker.subprocess.run',side_effect=exhausted):
check(handle(msg2)=='failed','memory failure not terminal')
with store.connect() as conn:check(fetch_job(conn,msg2['job_id'])['error_code']=='resource_limit','memory failure leaked details')
replay(store,msg2['job_id']);check(handle(msg2)=='ready','memory-failed replay did not complete')
rows.append(dict(case='real_child_address_space_limit_32MiB',status='PASS',error_code='resource_limit'))
active,msg3=upload()
orphan='staging/'+str(uuid.uuid4())
object_store.client.put_object(Bucket=object_store.bucket,Key=orphan,Body=b'orphan')
# Advance janitor's clock, leaving actual DB state/worker leases intact.
class Future(datetime):
@classmethod
def now(cls,tz=None):return datetime.now(tz)+timedelta(days=2)
with mock.patch('janitor.datetime',Future):
dry=sweep(False,86400)
check(orphan in dry['staging_candidates'],'orphan not detected')
check('staging/'+active['id'] not in dry['staging_candidates'],'active source scheduled for deletion')
object_store.client.head_object(Bucket=object_store.bucket,Key=orphan)
applied=sweep(True,86400)
check(orphan in applied['staging_candidates'],'orphan not deleted')
repeat=sweep(True,86400)
check(not repeat['staging_candidates'],'cleanup is not idempotent')
check(handle(msg3)=='ready','cleanup lost current job')
rows.append(dict(case='staging_dry_run_apply_repeat',status='PASS',candidates=len(applied['staging_candidates']),
repeat_candidates=0,active_retained=True,clock='advanced_2_days'))
print(json.dumps({'status':'PASS','rows':rows},indent=2))
finally:store.pool.close();object_store.close()
if __name__=='__main__':main()
+35
View File
@@ -0,0 +1,35 @@
"""Isolated crashable consumer using a real Streams consumer-group delivery."""
import argparse
import json
from pathlib import Path
import sys
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from app import store
from pipeline import broker,envelope,fetch_job,STREAM
from worker import handle
import redis
p=argparse.ArgumentParser();p.add_argument('job_id');p.add_argument('--crash',choices=['before_effect','after_effect','after_commit']);p.add_argument('--recover',action='store_true');args=p.parse_args()
store.pool.open(wait=True,timeout=20)
r=broker();group='fault-probes'
try:
try:r.xgroup_create(STREAM,group,id='$',mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):raise
with store.connect() as db:msg=envelope(fetch_job(db,args.job_id))
if args.recover:
entries=r.xautoclaim(STREAM,group,'recovered',min_idle_time=0,start_id='0-0',count=100)[1]
target=next(((mid,fields) for mid,fields in entries if json.loads(fields['job'])['job_id']==args.job_id),None)
if target is None:raise AssertionError('expected unacknowledged real broker delivery')
mid,fields=target;msg=json.loads(fields['job'])
else:
target_id=r.xadd(STREAM,{'job':json.dumps(msg)})
while True:
entries=r.xreadgroup(group,'crashable',{STREAM:'>'},count=100,block=1000)
target=next(((mid,fields) for _,batch in entries for mid,fields in batch if mid==target_id),None)
if target:mid,_=target;break
outcome=handle(msg,crash=args.crash)
if outcome!='defer':r.xack(STREAM,group,mid)
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
finally:
r.close();store.pool.close()
+37
View File
@@ -0,0 +1,37 @@
"""Concurrent shared-key requests through LB, real instance headers and history."""
from concurrent.futures import ThreadPoolExecutor
import json
import threading
import uuid
from client import Client, arguments, check, run
def main():
args = arguments(__doc__)
c = Client(args.base_url)
alice, bob, eve, chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(16)
def worker(_):
local = Client(args.base_url)
barrier.wait()
return local.message(alice, chat, ' concurrent body ', key=key)
with ThreadPoolExecutor(max_workers=16) as pool:
responses = list(pool.map(worker, range(16)))
ids = {r['json']['id'] for r in responses}
instances = {r['headers']['X-Instance-Id'] for r in responses}
check(len(ids) == 1, 'concurrent duplicate messages')
check(len(instances) >= 2, 'concurrent requests did not reach two replicas')
path = '/api/v1/chats/' + chat['id'] + '/messages'
c.call('POST', path, 409, actor=alice, body={'text': 'concurrent body'}, headers={'Idempotency-Key': key})
again = c.call('POST', path, 201, actor=alice, body={'attachment_ids': [], 'text': ' concurrent body '}, headers={'Idempotency-Key': key}, schema='Message')
check(again['json'] == responses[0]['json'], 'semantic replay differs')
page = c.call('GET', path, actor=bob, schema='MessagePage')['json']
check(len(page['items']) == 1, 'history contains duplicate effects')
c.message(bob, chat, 'other user', key=key)
print(json.dumps({'parallel_requests': 16, 'observed_instances': sorted(instances), 'unique_messages': len(ids), 'conflict': 409, 'semantic_replay': True, 'user_scope': True}))
if __name__ == '__main__':
run(main)
+51
View File
@@ -0,0 +1,51 @@
"""Actual variant geometry, orientation, alpha, encoder metadata and hashes."""
import hashlib
import io
import json
from pathlib import Path
import sys
from PIL import Image
sys.path.insert(0,str(Path(__file__).resolve().parent))
from client import Client,check
def main():
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();rows=[]
cases=[]
cases.append(('small_alpha',Image.new('RGBA',(80,40),(0,0,0,0)),'PNG',{},(80,40)))
cases.append(('exif_orientation',Image.new('RGB',(400,100),'red'),'JPEG',{'exif':Image.Exif()},(64,256)))
cases[-1][3]['exif'][274]=6
center=Image.new('RGB',(400,100),'red');center.paste('blue',(150,0,250,100))
cases.append(('center_crop',center,'PNG',{},(256,64)))
cases.append(('large_fit',Image.new('RGB',(1600,800),'green'),'PNG',{},(256,128)))
for name,image,fmt,options,expected in cases:
data=io.BytesIO();image.save(data,fmt,**options);payload=data.getvalue()
mime='image/png' if fmt=='PNG' else 'image/jpeg'
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,
raw=payload,headers={'Content-Type':mime},schema='Attachment')['json']
ready=c.wait_ready(alice,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==payload,'original changed')
dimensions={}
for variant in ready['variants']:
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+variant['name'],actor=bob)
check(hashlib.sha256(result['bytes']).hexdigest()==variant['sha256'],'false checksum')
check(len(result['bytes'])==variant['size_bytes'],'false variant size')
with Image.open(io.BytesIO(result['bytes'])) as output:
check(not output.getexif(),'EXIF leaked')
check(output.size==(variant['width'],variant['height']),'false dimensions')
dimensions[variant['name']]=output.size
check(output.format==('WEBP' if variant['name']=='optimized' else 'JPEG'),'wrong encoder')
if variant['name']=='thumbnail':
check(output.size==expected,'thumbnail orientation/scale incorrect')
if name=='small_alpha':check(min(output.getpixel((10,10)))>245,'alpha not composited on white')
if variant['name']=='crop':
check(output.size==(128,128),'crop dimensions wrong')
if name=='center_crop':check(output.getpixel((64,64))[2]>240,'crop not centered')
if variant['name']=='optimized':
check(max(output.size)<=1280,'optimized too large')
if name=='large_fit':check(output.size==(1280,640),'optimized ratio wrong')
rows.append(dict(case=name,status='PASS',dimensions=dimensions,original_sha256=hashlib.sha256(payload).hexdigest()))
print(json.dumps({'status':'PASS','rows':rows},indent=2))
if __name__=='__main__':main()
+84
View File
@@ -0,0 +1,84 @@
"""Host orchestration: stopped worker, API kill, broker restart, real lease."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parent))
from client import Client,check
ROOT=Path(__file__).resolve().parents[1]
COMPOSE=[str(ROOT/'scripts/compose.sh')]
def compose(*args,allow_failure=False):
p=subprocess.run(COMPOSE+list(args),cwd=ROOT,text=True,capture_output=True)
if p.returncode and not allow_failure:
raise AssertionError('Compose action failed: '+' '.join(args[:3])+': '+p.stderr[-500:])
return p
def probe(aid):
return json.loads(compose('run','--rm','--no-deps','ops','python','scripts/query_pipeline.py',aid).stdout)[-1]
def wait_api(c):
deadline=time.monotonic()+60
while time.monotonic()<deadline:
try:c.call('GET','/health/ready',schema='Health');return
except Exception:time.sleep(.3)
raise AssertionError('API recovery timeout')
def snapshot(services):
ids=compose('ps','-q',*services).stdout.split()
state=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
return {x['Name'].lstrip('/'):{'id':x['Id'][:12],'started_at':x['State']['StartedAt'],
'volumes':sorted(m['Name'] for m in x['Mounts'] if m['Type']=='volume')} for x in state}
def main():
c=Client(os.environ.get('BASE_URL','http://localhost:8085'))
rows=[]
try:
compose('stop','worker','publisher')
alice,bob,eve,chat=c.bootstrap();att,data=c.attachment(alice,chat)
message=c.message(alice,chat,'',attachments=[att['id']])['json']
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
check(probe(att['id'])['published_at'] is None,'publisher unexpectedly active')
before=snapshot(['api','broker'])
compose('kill','-s','SIGKILL','api');compose('start','api');wait_api(c)
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
page=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage')['json']
check(any(m['id']==message['id'] for m in page['items']),'queued message lost')
compose('start','publisher')
deadline=time.monotonic()+20
while probe(att['id'])['published_at'] is None:
check(time.monotonic()<deadline,'outbox not delivered');time.sleep(.2)
length_before=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
compose('restart','broker')
length_after=int(compose('exec','-T','broker','valkey-cli','XLEN','attachments.v1').stdout)
check(length_after>=length_before>0,'broker lost durable messages')
after=snapshot(['api','broker'])
check(all(after[name]['started_at']!=row['started_at'] and after[name]['volumes']==row['volumes'] for name,row in before.items()),'restart did not preserve volumes')
start=time.monotonic();compose('start','worker');c.wait_ready(bob,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'accepted original lost')
rows.append(dict(scenario='stopped_worker_api_kill_outbox_broker_restart',status='PASS',attachment_id=att['id'],
stream_length_before=length_before,stream_length_after=length_after,ready_ms=round((time.monotonic()-start)*1000,3),
containers_before=before,containers_after=after))
compose('stop','worker','publisher')
att,data=c.attachment(alice,chat);jid=probe(att['id'])['job_id']
crash=compose('run','--rm','--no-deps','ops','python','tests/drive_job.py',jid,'--crash','after_effect',allow_failure=True)
check(crash.returncode==86,'worker failpoint did not crash')
check(probe(att['id'])['state']=='processing','processing lease missing')
start=time.monotonic();compose('start','publisher','worker');c.wait_ready(bob,att)
elapsed=time.monotonic()-start
check(elapsed>=20,'real lease expiry not exercised')
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'lease recovery corrupted original')
rows.append(dict(scenario='crash_real_lease_expiry',status='PASS',attachment_id=att['id'],ready_ms=round(elapsed*1000,3),
job=probe(att['id']),lease_clock='real'))
print(json.dumps({'status':'PASS','rows':rows},ensure_ascii=False,indent=2))
finally:compose('start','api','broker','publisher','worker')
if __name__=='__main__':main()
+143
View File
@@ -0,0 +1,143 @@
"""Track 2: real crash processes, broker redelivery, version fencing and retries.
Run only with the normal worker and publisher stopped. Creates synthetic rows.
"""
import concurrent.futures
import hashlib
import io
import json
from pathlib import Path
import subprocess
import sys
import time
from PIL import Image
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,check,validate
from app import store,object_store
from pipeline import broker,envelope,fetch_job,MAX_ATTEMPTS,replay,upgrade,STREAM
from worker import handle
from publisher import publish_once
def main():
store.pool.open(wait=True,timeout=20)
c=Client('http://lb:8080');alice,bob,eve,chat=c.bootstrap();r=broker()
rows=[]
def upload():
att,data=c.attachment(alice,chat)
with store.connect() as conn:
jid=str(conn.execute('SELECT id FROM jobs WHERE attachment_id=%s',(att['id'],)).fetchone()[0])
msg=envelope(fetch_job(conn,jid))
validate(msg,json.loads((Path(__file__).resolve().parents[1]/'contracts/jobs.schema.json').read_text()))
return att,data,jid,msg
def metadata(att):return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
def objects(att,version=1):
return sorted(x['Key'] for x in object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix=f"results/{att['id']}/v{version}/").get('Contents',[]))
def invariant(att,data,version=1):
ready=metadata(att);check(ready['status']=='ready','lost accepted work')
check(len(ready['variants'])==3,'wrong number of variants')
check(len(objects(att,version))==4,'duplicate or missing objects')
for name in ('original','thumbnail','crop','optimized'):
result=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=bob)
if name=='original':check(result['bytes']==data,'original mutated')
else:
v=next(x for x in ready['variants'] if x['name']==name)
check(hashlib.sha256(result['bytes']).hexdigest()==v['sha256'],'variant checksum mismatch')
with Image.open(io.BytesIO(result['bytes'])) as im:
check((im.width,im.height)==(v['width'],v['height']),'false dimensions')
check(not im.getexif(),'EXIF not stripped')
return ready
try:
for iteration in range(3):
for crash in ('before_effect','after_effect','after_commit'):
att,data,jid,msg=upload();start=time.monotonic()
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--crash',crash],capture_output=True,text=True)
check(result.returncode==86,'failpoint did not kill the worker process')
before=objects(att)
check(len(before)==(0 if crash=='before_effect' else 4),'wrong crash window')
with store.connect() as conn:
j=fetch_job(conn,jid)
if crash!='after_commit':
check(j['state']=='processing','lease not persisted')
check(handle(msg)=='defer','live lease was stolen')
# Controlled clock acceleration for matrix speed. Separate
# persistence test exercises real 30-second lease timeout.
conn.execute("UPDATE jobs SET lease_until=now()-interval '1 second' WHERE id=%s",(jid,))
result=subprocess.run([sys.executable,'tests/drive_job.py',jid,'--recover'],capture_output=True,text=True)
check(result.returncode==0,'redelivery process failed')
check(json.loads(result.stdout)['outcome'] in ('ready','noop'),'redelivery did not complete')
ready=invariant(att,data)
# Successful duplicate must not update metadata or re-encode.
check(handle(msg)=='noop','ready duplicate was re-executed')
check(metadata(att)==ready,'ready changed after duplicate')
with store.connect() as conn:attempts=fetch_job(conn,jid)['attempt']
rows.append(dict(iteration=iteration+1,scenario=crash,status='PASS',attempts=attempts,
recovery_ms=round((time.monotonic()-start)*1000,3),objects=4,lease_expiry='accelerated' if crash!='after_commit' else 'not_needed'))
att,data,jid,msg=upload();start=time.monotonic()
outcomes=[]
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
outcomes=list(pool.map(lambda _:handle(msg),range(8)))
invariant(att,data)
check(outcomes.count('ready')==1,'concurrent duplicate produced multiple effects')
rows.append(dict(iteration=iteration+1,scenario='concurrent_duplicates',status='PASS',deliveries=8,
effects=1,recovery_ms=round((time.monotonic()-start)*1000,3),outcomes=outcomes,objects=4))
# New version overtakes an unprocessed old version.
att,data,jid,old=upload();new_jid=upgrade(store,att['id'])
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
start=time.monotonic();check(handle(new)=='ready','new version failed')
newest=invariant(att,data,2)
for _ in range(5):check(handle(old)=='noop','stale job not fenced')
check(metadata(att)==newest and not objects(att,1),'old version overwrote new')
rows.append(dict(iteration=iteration+1,scenario='reorder_stale_version',status='PASS',old_deliveries=5,
current_version=2,old_objects=0,objects=4,recovery_ms=round((time.monotonic()-start)*1000,3)))
# Upgrade an already published result, then redeliver the old version.
att,data,jid,old=upload();check(handle(old)=='ready','v1 failed')
v1=invariant(att,data)
new_jid=upgrade(store,att['id'])
with store.connect() as conn:new=envelope(fetch_job(conn,new_jid))
check(handle(new)=='ready','upgrade failed');v2=invariant(att,data,2)
check(v1['variants']!=v2['variants'],'version settings did not change result')
check(handle(old)=='noop' and metadata(att)==v2,'published old version replaced upgraded result')
rows.append(dict(scenario='published_v1_then_v2_then_stale_v1',status='PASS',current_version=2,objects=4))
# Permanent errors, explicit audited replay, ready replay refused.
att,data,jid,msg=upload();check(handle(msg,force_error='permanent')=='failed','permanent failure retried')
check(metadata(att)['status']=='failed','failed not exposed')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':'failed-test'})
replay(store,jid);check(handle(msg)=='ready','operator replay did not recover')
invariant(att,data)
try:replay(store,jid);raise AssertionError('ready replay allowed')
except ValueError:pass
rows.append(dict(scenario='permanent_failed_replay',status='PASS',objects=4))
# Transient retry scheduling respects backoff and exhausts the bound.
att,data,jid,msg=upload();delays=[]
for attempt in range(MAX_ATTEMPTS):
check(handle(msg,force_error='temporary')=='retry','temporary failure not handled')
with store.connect() as conn:
j=fetch_job(conn,jid)
if attempt<MAX_ATTEMPTS-1:
delay=conn.execute('SELECT extract(epoch FROM next_at-now()) FROM jobs WHERE id=%s',(jid,)).fetchone()[0]
check(delay>0,'no backoff');delays.append(float(delay))
check(handle(msg)=='defer','retry ignored schedule')
conn.execute("UPDATE jobs SET next_at=now()-interval '1 second' WHERE id=%s",(jid,))
check(metadata(att)['error_code']=='retry_exhausted','retry was not bounded')
replay(store,jid);check(handle(msg)=='ready','retry exhausted replay failed');invariant(att,data)
rows.append(dict(scenario='bounded_retry_jitter',status='PASS',max_attempts=MAX_ATTEMPTS,delays_seconds=delays))
# COMMIT -> publish recovery, and a lost publish confirm duplicates safely.
att,data,jid,msg=upload()
with store.connect() as conn:check(conn.execute('SELECT published_at FROM outbox WHERE job_id=%s',(jid,)).fetchone()[0] is None,'outbox already published')
publish_once(r)
with store.connect() as conn:
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
publish_once(r)
check(handle(msg)=='ready' and handle(msg)=='noop','republish was unsafe');invariant(att,data)
rows.append(dict(scenario='outbox_republish',status='PASS',objects=4))
with store.connect() as conn:
accepted=conn.execute('SELECT count(*) FROM attachments').fetchone()[0]
result={'status':'PASS','track':2,'matrix_repeats':3,'rows':rows,'accepted_in_database':accepted,
'invariants':{'accepted_tasks_lost':0,'duplicate_variant_sets':0,'stale_version_overwrites':0},
'limitations':['matrix lease expiry accelerated with SQL','single host, worker processing is serial except intentional duplicate probe','no disk-loss or network partition proof']}
print(json.dumps(result,ensure_ascii=False,indent=2))
finally:r.close();store.pool.close();object_store.close()
if __name__=='__main__':main()
+53
View File
@@ -0,0 +1,53 @@
"""Anonymous access outside API and least-privilege negative checks."""
import json
from pathlib import Path
import sys
import urllib.error
import urllib.request
from botocore.exceptions import ClientError
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client,check,run
def main():
from app import store,object_store
c=Client('http://lb:8080',timeout=15)
alice,bob,eve,chat=c.bootstrap()
att,_=c.attachment(alice,chat)
c.wait_ready(alice,att)
key='results/'+att['id']+'/v1/original'
anonymous=[]
for suffix in ('','/'+key):
try:
urllib.request.urlopen('http://s3:9000/'+object_store.bucket+suffix,timeout=3)
raise AssertionError('private bucket allowed anonymous access')
except urllib.error.HTTPError as exc:
check(exc.code==403,'anonymous access failed for an unexpected reason')
anonymous.append(exc.code)
# MinIO returns a filtered ListBuckets response to a ListBucket principal.
visible=[b['Name'] for b in object_store.client.list_buckets().get('Buckets',[])]
check(set(visible)<={object_store.bucket},'unrelated bucket names exposed')
denied=[]
for name,operation in [('create_bucket',lambda:object_store.client.create_bucket(Bucket='must-not-create-lab4')),
('get_bucket_policy',lambda:object_store.client.get_bucket_policy(Bucket=object_store.bucket)),
('write_outside_prefix',lambda:object_store.client.put_object(Bucket=object_store.bucket,Key='outside-policy',Body=b'no'))]:
try:
operation()
raise AssertionError('application credentials have excessive privileges: '+name)
except ClientError as exc:
check(exc.response['Error']['Code']=='AccessDenied','unexpected permission error')
denied.append(name)
# The console is bound to 127.0.0.1 inside S3 and browser UI is disabled.
import socket
try:
socket.create_connection(('s3',9001),timeout=1).close()
raise AssertionError('S3 console reachable from another container')
except OSError:
pass
print(json.dumps({'status':'PASS','anonymous_bucket_and_object_status':anonymous,
'application_admin_operations_denied':denied,'visible_bucket_names':visible,'console_reachable':False},indent=2))
object_store.close()
if __name__=='__main__':
run(main)
+129
View File
@@ -0,0 +1,129 @@
"""Contract boundaries and object authorization; runs in Compose ops network."""
import hashlib
import http.client
import io
import json
from pathlib import Path
import socket
import struct
import sys
import time
from urllib.parse import quote, urlsplit
import uuid
import zlib
from PIL import Image
sys.path.insert(0,str(Path(__file__).resolve().parent))
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from client import Client, check, run
def main():
c=Client('http://lb:8080',timeout=15)
alice,bob,eve,chat=c.bootstrap()
path='/api/v1/chats/'+chat['id']+'/attachments'
fixture=(Path(__file__).parent/'fixtures/sample.png').read_bytes()
cases=[]
def bad(label,status,data=fixture,mime='image/png',filename='test.png',actor=None):
c.call('POST',path+'?filename='+quote(filename,safe=''),status,actor=actor or alice,raw=data,headers={'Content-Type':mime})
cases.append({'case':label,'status':status})
bad('corrupt_png',415,b'not an image')
bad('truncated_png',415,fixture[:50])
bad('mime_spoof',415,mime='image/jpeg')
bad('unsupported_mime',415,mime='application/octet-stream')
bad('empty_payload',415,b'')
for filename in ('../secret.png','a\\b.png','x\x00.png','x\n.png','x\u202e.png','x'*256):
bad('unsafe_filename',400,filename=filename)
bad('foreign_chat_upload',404,actor=eve)
c.call('POST',path+'?filename=x.png',401,raw=fixture,headers={'Content-Type':'image/png'})
cases.append({'case':'unauthenticated_upload','status':401})
# Oversized dimensions are rejected using headers before a full raster decode.
data=bytearray(fixture)
data[16:24]=struct.pack('>II',10000,4001)
data[29:33]=struct.pack('>I',zlib.crc32(data[12:29])&0xffffffff)
bad('over_40_million_pixels',413,bytes(data))
# No large allocation or body transfer: reject declared size before reading.
parsed=urlsplit(c.base)
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=10)
conn.putrequest('POST',path+'?filename=big.png')
for k,v in {'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png','Content-Length':str(10485761)}.items():
conn.putheader(k,v)
conn.endheaders()
response=conn.getresponse()
check(response.status==413,'oversized content-length accepted')
response.read();conn.close()
cases.append({'case':'oversized_content_length_before_body','status':413})
# True streamed overflow, no Content-Length; response is read after the last
# chunk, even if the peer stopped accepting bytes once it knew the limit.
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=15)
conn.putrequest('POST',path+'?filename=chunked.png')
conn.putheader('Authorization','Bearer '+alice['token'])
conn.putheader('Content-Type','image/png')
conn.putheader('Transfer-Encoding','chunked')
conn.endheaders()
try:
block=b'x'*65536
for _ in range(161):
conn.send(b'10000\r\n'+block+b'\r\n')
conn.send(b'0\r\n\r\n')
except (BrokenPipeError,ConnectionResetError):
pass
response=conn.getresponse()
check(response.status==413,'streamed byte limit not enforced')
response.read();conn.close()
cases.append({'case':'chunked_oversize','status':413})
att,content=c.attachment(alice,chat)
# Another member can reference an attachment; ownership is not required.
msg=c.message(bob,chat,'',attachments=[att['id']])['json']
check(msg['attachment_ids']==[att['id']],'member could not link attachment')
for suffix in ('','/content'):
c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve)
c.wait_ready(alice,att)
c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=unknown',404,actor=alice)
cases.append({'case':'foreign_attachment_metadata_and_bytes','status':404})
other=c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'other','member_ids':[alice['id']]},schema='Chat')['json']
other_path='/api/v1/chats/'+other['id']+'/messages'
for aid in (att['id'],str(uuid.uuid4())):
c.call('POST',other_path,404,actor=alice,body={'text':'','attachment_ids':[aid]},headers={'Idempotency-Key':uuid.uuid4().hex})
cases.append({'case':'cross_chat_and_unknown_attachment','status':404})
message_path='/api/v1/chats/'+chat['id']+'/messages'
for ids in ([att['id'],att['id']],['bad-id'],[str(uuid.uuid4()) for _ in range(11)]):
c.call('POST',message_path,400,actor=alice,body={'text':'','attachment_ids':ids},headers={'Idempotency-Key':uuid.uuid4().hex})
key=uuid.uuid4().hex
first=c.message(alice,chat,'',key=key,attachments=[att['id']])['json']
second=c.message(alice,chat,'',key=key,attachments=[att['id']])['json']
check(first==second,'attachment message replay changed')
c.call('POST',message_path,409,actor=alice,body={'text':'different','attachment_ids':[att['id']]},headers={'Idempotency-Key':key})
cases.append({'case':'attachment_idempotency_and_conflict','status':'201/409'})
# Full JPEG validation, including truncated entropy-coded body.
stream=io.BytesIO()
Image.new('RGB',(320,200),'blue').save(stream,'JPEG')
jpeg=stream.getvalue()
uploaded=c.call('POST',path+'?filename=photo.jpg',202,actor=alice,raw=jpeg,headers={'Content-Type':'image/jpeg'},schema='Attachment')['json']
c.wait_ready(alice,uploaded)
downloaded=c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',actor=bob)
check(downloaded['bytes']==jpeg,'JPEG original changed')
bad('truncated_jpeg',415,jpeg[:-20],mime='image/jpeg')
# A disconnect before receiving all bytes creates neither intent nor object.
from app import store,object_store
store.pool.open(wait=True,timeout=15)
try:
def counts():
with store.connect() as db:
n=db.execute('SELECT count(*) FROM attachments').fetchone()[0]
return n,len(object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='staging/').get('Contents',[]))
before=counts()
conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=5)
conn.putrequest('POST',path+'?filename=interrupted.png')
conn.putheader('Authorization','Bearer '+alice['token'])
conn.putheader('Content-Type','image/png')
conn.putheader('Content-Length',str(len(fixture)))
conn.endheaders();conn.send(fixture[:50]);conn.close()
time.sleep(1)
check(counts()==before,'aborted upload persisted partial data')
cases.append({'case':'interrupted_http_body','metadata_or_object_added':False})
finally:
store.pool.close();object_store.close()
print(json.dumps({'status':'PASS','cases':cases,'count':len(cases)},ensure_ascii=False,indent=2))
if __name__=='__main__':
run(main)
+56
View File
@@ -0,0 +1,56 @@
"""Isolated decoder/encoder. No network or credentials; bounded by parent timeout."""
import hashlib
import json
from pathlib import Path
import resource
import sys
import warnings
from PIL import Image, ImageOps
# Address space bound complements the whole-container cgroup memory cap.
resource.setrlimit(resource.RLIMIT_AS, (640*1024**2,640*1024**2))
Image.MAX_IMAGE_PIXELS = 40000000
warnings.simplefilter('error', Image.DecompressionBombWarning)
def fit(image, cap):
w,h=image.size
ratio=min(1,cap/w,cap/h)
return image.resize((max(1,int(w*ratio)),max(1,int(h*ratio))), Image.Resampling.LANCZOS)
def main():
source,target=Path(sys.argv[1]),Path(sys.argv[2])
version=int(sys.argv[3]) if len(sys.argv)>3 else 1
jpeg_quality,webp_quality=(85,80) if version==1 else (90,85)
with Image.open(source,formats=['PNG','JPEG']) as im:
if im.width*im.height>40000000:
raise ValueError('pixels')
im.load()
image=ImageOps.exif_transpose(im).convert('RGBA')
white=Image.new('RGBA',image.size,'white')
white.alpha_composite(image)
rgb=white.convert('RGB')
variants=[]
for name,content_type,output in (
('thumbnail','image/jpeg',fit(rgb,256)),
('crop','image/jpeg',ImageOps.fit(rgb,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5))),
('optimized','image/webp',fit(rgb,1280))):
path=target/name
# Explicit empty EXIF; encoder gets a fresh image without source metadata.
clean=Image.new('RGB',output.size);clean.paste(output)
clean.save(path,format='WEBP' if name=='optimized' else 'JPEG',quality=webp_quality if name=='optimized' else jpeg_quality,exif=b'')
payload=path.read_bytes()
variants.append(dict(name=name,content_type=content_type,width=output.width,height=output.height,
size_bytes=len(payload),sha256=hashlib.sha256(payload).hexdigest()))
(target/'manifest.json').write_text(json.dumps(variants))
if __name__=='__main__':
try:
main()
except MemoryError:
sys.exit(3)
except Exception:
# Deliberately no decoder traceback or client-controlled payload in logs.
sys.exit(2)
+183
View File
@@ -0,0 +1,183 @@
"""Fenced, serial worker; DB row locks protect all effects of one job/version."""
import hashlib
import json
import os
from pathlib import Path
import random
import signal
import subprocess
import sys
import tempfile
import threading
import time
import uuid
from datetime import datetime, timezone
from botocore.exceptions import BotoCoreError, ClientError
from psycopg.rows import dict_row
import redis
from app import store, object_store
from pipeline import audit, broker, fetch_job, GROUP, LEASE, MAX_ATTEMPTS, STREAM, TIMEOUT
stop=threading.Event()
class Permanent(Exception):
pass
def failpoint(name, selected):
# CLI/in-process test only, never from broker or HTTP.
if name == selected:
os._exit(86)
def valid_message(msg):
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
if not isinstance(msg,dict) or not required<=msg.keys() or set(msg)-required-{'traceparent','tracestate'}:
return False
try:
uuid.UUID(msg['job_id']);uuid.UUID(msg['attachment_id'])
datetime.fromisoformat(msg['created_at'].replace('Z','+00:00'))
return (type(msg['schema_version']) is int and msg['schema_version']==1 and msg['type']=='attachment.process.v1'
and type(msg['attempt']) is int and msg['attempt']>=1
and type(msg['pipeline_version']) is int and msg['pipeline_version']>=1)
except (ValueError,TypeError,AttributeError):return False
def claim(msg):
with store.connect() as conn:
job=fetch_job(conn,msg['job_id'],'FOR UPDATE SKIP LOCKED')
if job is None:return 'defer',None
if str(job['attachment_id'])!=msg['attachment_id'] or job['pipeline_version']!=msg['pipeline_version']:
return 'invalid',None
if job['state'] in ('ready','failed','superseded'):
audit(conn,job['id'],'duplicate_noop')
return 'noop',job
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
if att['pipeline_version']!=job['pipeline_version']:
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],))
audit(conn,job['id'],'stale_noop');return 'noop',job
now=datetime.now(timezone.utc)
if job['next_at']>now or (job['state']=='processing' and job['lease_until'] and job['lease_until']>now):
return 'defer',job
if job['attempt']>=MAX_ATTEMPTS:
conn.execute("UPDATE jobs SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(job['id'],))
conn.execute("UPDATE attachments SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(att['id'],))
audit(conn,job['id'],'retry_exhausted');return 'noop',job
job=conn.cursor(row_factory=dict_row).execute("""UPDATE jobs SET state='processing',attempt=attempt+1,
lease_until=now()+(%s*interval '1 second'),updated_at=now() WHERE id=%s RETURNING *""",(LEASE,job['id'])).fetchone()
conn.execute("UPDATE attachments SET state='processing',updated_at=now() WHERE id=%s",(att['id'],))
audit(conn,job['id'],'claimed')
return 'claimed',job
def record_error(job, code, permanent):
with store.connect() as conn:
current=fetch_job(conn,job['id'],'FOR UPDATE')
if current['state']!='processing' or current['attempt']!=job['attempt']:return
final=permanent or current['attempt']>=MAX_ATTEMPTS
state='failed' if final else 'queued'
code=code if permanent or not final else 'retry_exhausted'
delay=random.uniform(.5,1.5)*min(30,2**(current['attempt']-1))
conn.execute("UPDATE jobs SET state=%s,error_code=%s,next_at=now()+(%s*interval '1 second'),lease_until=NULL,updated_at=now() WHERE id=%s",(state,code,delay,job['id']))
conn.execute("UPDATE attachments SET state=%s,error_code=%s,updated_at=now() WHERE id=%s AND pipeline_version=%s",
(state,code if final else None,job['attachment_id'],job['pipeline_version']))
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],))
audit(conn,job['id'],'failed:'+code if final else 'retry:'+code)
def handle(msg, crash=None, force_error=None):
if not valid_message(msg):return 'invalid'
action,job=claim(msg)
if action!='claimed':return action
try:
with store.connect() as conn:
current=fetch_job(conn,job['id'],'FOR UPDATE')
if current['state']!='processing' or current['attempt']!=job['attempt']:return 'defer'
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
if att['pipeline_version']!=job['pipeline_version']:
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],));return 'noop'
if force_error=='permanent':raise Permanent('invalid_image')
if force_error=='temporary':raise TimeoutError()
failpoint('before_effect',crash)
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
response=object_store.client.get_object(Bucket=object_store.bucket,Key=att['staging_key'])
try:payload=response['Body'].read(10485761)
finally:response['Body'].close()
if len(payload)!=att['size_bytes'] or hashlib.sha256(payload).hexdigest()!=att['sha256']:
raise Permanent('staging_corrupt')
(root/'input').write_bytes(payload)
result=subprocess.run([sys.executable,'transform.py',str(root/'input'),str(root),str(job['pipeline_version'])],timeout=TIMEOUT,
env={'PATH':os.environ.get('PATH','/usr/local/bin:/usr/bin:/bin'),'LANG':'C.UTF-8'},
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False)
if result.returncode:
raise Permanent('resource_limit' if result.returncode in (3,-9) else 'invalid_image')
variants=json.loads((root/'manifest.json').read_text())
prefix=f"results/{att['id']}/v{job['pipeline_version']}/"
key=prefix+'original'
def put(key,data,mime):
object_store.client.put_object(Bucket=object_store.bucket,Key=key,Body=data,
ContentType=mime,Metadata={'sha256':hashlib.sha256(data).hexdigest()})
put(key,payload,att['content_type'])
for variant in variants:
variant['object_key']=prefix+variant['name']
put(variant['object_key'],(root/variant['name']).read_bytes(),variant['content_type'])
failpoint('after_effect',crash)
conn.execute("""UPDATE attachments SET state='ready',error_code=NULL,object_key=%s,
variants=%s::jsonb,updated_at=now() WHERE id=%s AND pipeline_version=%s""",
(key,json.dumps(variants),att['id'],job['pipeline_version']))
conn.execute("UPDATE jobs SET state='ready',error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=%s",(job['id'],))
audit(conn,job['id'],'ready')
failpoint('after_commit',crash)
return 'ready'
except Permanent as exc:
record_error(job,str(exc),True);return 'failed'
except ClientError as exc:
code=exc.response.get('Error',{}).get('Code','')
record_error(job,'staging_missing' if code in ('NoSuchKey','404') else 'storage_unavailable',code in ('NoSuchKey','404'))
return 'retry'
except (BotoCoreError,TimeoutError,subprocess.TimeoutExpired):
record_error(job,'processing_timeout',False);return 'retry'
except Exception:
# Includes a lost/ambiguous DB COMMIT: never delete effects. Lease and
# durable outbox safely recover; a ready duplicate cannot be rerun.
print(json.dumps({'worker_error':'infrastructure','job_id':str(job['id'])}),flush=True)
return 'defer'
def ack(client,mid):
client.xack(STREAM,GROUP,mid)
client.xdel(STREAM,mid)
def main():
store.pool.open(wait=True,timeout=30)
client=broker();consumer=str(uuid.uuid4())
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:
try:client.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):raise
reclaimed=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE*1000,start_id='0-0',count=1)[1]
entries=reclaimed
if not entries:
batches=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
entries=batches[0][1] if batches else []
for mid,fields in entries:
try:msg=json.loads(fields.get('job','null'))
except ValueError:msg=None
outcome=handle(msg)
# A failed attempt has a durable retry intent/failed record.
# Defer leaves the delivery pending for lease-based reclaim.
if outcome!='defer':ack(client,mid)
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
except Exception as exc:
print(json.dumps({'worker_error':type(exc).__name__}),flush=True);stop.wait(1)
finally:
client.close();object_store.close();store.pool.close()
if __name__=='__main__':main()