ЛР5: брокер RabbitMQ, outbox, worker, варианты изображений, отказоустойчивость, исследование очереди

This commit is contained in:
kolya5544
2026-10-03 05:10:45 +03:00
parent 276dbdad8a
commit 518c98fd3e
59 changed files with 4947 additions and 32 deletions
+6
View File
@@ -18,3 +18,9 @@ __pycache__
node_modules
evidence
.DS_Store
research
student-tests
scripts
docs
requirements-research.txt
*.md
+27 -16
View File
@@ -1,25 +1,36 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh)
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
# TTL записи в кэше Valkey (не больше остатка жизни сессии)
SESSION_CACHE_TTL_SECONDS=60
# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL).
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
DB_POOL_MAX=10
# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh)
DATABASE_URL_FILE=/run/secrets/database_url
# --- lab3 ---
# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
API_CPUS=1.0
API_MEM=256m
POSTGRES_CPUS=1.0
LB_CPUS=1.0
# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок)
DRAIN_SECONDS=4
# --- lab4 ---
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
S3_CPUS=1.0
UPLOAD_CONCURRENCY=4
# --- lab5 ---
RABBITMQ_CPUS=1.0
WORKER_CPUS=1.0
WORKER_MEM=768m
WORKER_REPLICAS=1
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
MAX_ATTEMPTS=5
LEASE_SECONDS=60
MAX_QUEUED_JOBS=200
+32
View File
@@ -0,0 +1,32 @@
# syntax=docker/dockerfile:1
# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm).
ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) ---
FROM ${PYTHON_IMAGE} AS builder
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1
WORKDIR /build
COPY requirements.txt .
RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
# --- runtime ---
FROM ${PYTHON_IMAGE}
ENV PATH=/opt/venv/bin:$PATH \
PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations
RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
COPY --from=builder /opt/venv /opt/venv
WORKDIR /srv
COPY src/ /srv/src/
COPY migrations/ /srv/migrations/
# ревизия образа (меняется при rolling update; видна в журнале запуска)
ARG REV=dev
LABEL org.opencontainers.image.revision=$REV
ENV APP_REVISION=$REV
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"]
# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless;
# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py).
CMD ["python", "-m", "app.serve"]
+16
View File
@@ -28,3 +28,19 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- собственные цели студента ---
.PHONY: secrets up down student-test
secrets:
sh scripts/init-secrets.sh
up: secrets
docker compose --env-file .env up -d --build
down:
docker compose --env-file .env down
student-test:
$(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v
$(PYTHON) -m unittest discover -s student-tests -p "test_attachments.py" -v
$(PYTHON) -m unittest discover -s student-tests -p "test_pipeline.py" -v
+94 -16
View File
@@ -1,37 +1,115 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: [ФИО], [группа]
- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Очередь под перегрузкой»
- Commit/PR: локальный коммит ветки `solution/lab5`; PR — после публикации
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, uvicorn, asyncpg, aiobotocore, aio-pika (RabbitMQ), Pillow (обработка изображений); RabbitMQ 4.1.8, SeaweedFS 4.48, PostgreSQL 17, Valkey 8, HAProxy 3.0. Код lab4 перенесён. Использование ИИ: [заполняется автором]
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Compose v2, bash (Git Bash на Windows), Python ≥ 3.9; на Windows `PYTHONUTF8=1`.
Версии: `python:3.12.12-slim-bookworm@sha256:593bd06e…`, `postgres:17.11-alpine@sha256:b0f9560a…`, `valkey/valkey:8.1.10-alpine@sha256:081c2f5c…`, `haproxy:3.0-alpine@sha256:56b887da…`, `chrislusf/seaweedfs:4.48@sha256:4e61d15f…`, `rabbitmq:4.1-alpine@sha256:6a0f7984…` (4.1.8), зависимости — `requirements.txt` (закреплены).
```sh
sh scripts/init-secrets.sh # secrets/*: БД, S3 (admin + ключи API), RabbitMQ (rabbitmq.conf, broker_url)
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build # postgres, valkey, seaweedfs, rabbitmq -> migrate + s3-init -> api x2, worker -> lb
curl http://127.0.0.1:8080/health/ready
python3 tests/smoke.py --base-url http://127.0.0.1:8080 # upload -> 202 -> ready -> thumbnail 256x160
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
sh scripts/restore-replicas.sh
# собственные тесты
python3 -m unittest discover -s student-tests -p "test_api.py" -v # + test_idempotency.py, test_attachments.py
python3 student-tests/test_pipeline.py # Pillow желателен (пиксельные проверки)
python3 student-tests/test_resilience5.py # окна отказа, ~2.5 мин
# операторские команды
docker compose --env-file .env run --rm -T worker python -m app.replay list
docker compose --env-file .env run --rm -T worker python -m app.replay replay <attachment_id>
```
Windows: `scripts/restart.cmd`, `scripts/stop-one.cmd`. Остановка с сохранением данных — `docker compose --env-file .env stop`; **намеренное удаление учебных данных** — `down -v` (volumes `pgdata`, `valkeydata`, `seaweeddata`, `rabbitmqdata`). RabbitMQ 4.x объявил `RABBITMQ_DEFAULT_USER/PASS(_FILE)` устаревшими, поэтому учётные данные задаются конфигурационным файлом-секретом `secrets/rabbitmq.conf`; hostname контейнера фиксирован (`rabbitmq`), иначе очередь не переживает пересоздание контейнера.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
Клиент ─▶ LB ─▶ API ─┬─(1) staging в S3 (приватный prefix staging/)
└─(2) ОДНА транзакция PostgreSQL: attachments(status=queued) + outbox ──▶ 202 queued
outbox ──relay (в API и worker, SKIP LOCKED)──▶ RabbitMQ (quorum, persistent, publisher confirm) ──▶ worker
worker: claim(queued→processing, lease) → staging → thumbnail/crop/WebP → S3 att/<id>/… → ОДНА транзакция: варианты + ready → ack → очистка staging
```
| Роль | Сервис | Health | Volume | Credentials |
| --- | --- | --- | --- | --- |
| Брокер | `rabbitmq` 4.1.8 | `rabbitmq-diagnostics ping` | `rabbitmqdata` | secret `rabbitmq.conf`; клиентам — secret `broker_url` |
| Worker | `worker` (отдельный процесс, тот же образ; 1 CPU / 768 MiB) | heartbeat-файл (<30 с) | — | secrets БД/S3/брокер |
| остальные | как в lab4 | | | |
- **Границы приёма (до 202)**: байты проверены (формат, целостность, размеры) и записаны в **устойчивый staging** (приватный S3-prefix `staging/<id>`, не локальный диск); метаданные и **намерение обработки** записаны одной транзакцией PostgreSQL (таблицы `attachments` и `outbox`). Только после COMMIT клиент получает `202` со снимком `status=queued`. Если брокер недоступен — это не мешает приёму: задание остаётся в outbox и публикуется позже. HTTP-передача байт клиентом по-прежнему синхронна (честная граница: staging уже в S3).
- **Состояния и переходы** (`contracts/attachment-states.md`): `queued → processing → ready | failed`; `processing → queued` при временной ошибке (backoff) и при истечении lease (sweeper); `failed → queued` только явным операторским replay. Повторная доставка готовой задачи — no-op с ack; завершённое состояние не откатывается дубликатом. Условия UPDATE (`status='queued'`, `lease_owner=…`) исключают конкурирующую запись.
- **Задание в брокере** (`jobs.schema.json`): только `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at` — без байт, ключей S3 и credentials; worker берёт доверенные метаданные (ключи, тип) из БД. `job_id` не меняется при redelivery; `attempt` — управляемая попытка.
- **Доставка и подтверждения**: durable quorum-очередь `attachment.process`, persistent-сообщения, **publisher confirm** (подтверждение приёма брокером) → только затем `published_at` в outbox; **ack** consumer — только после COMMIT результата (устойчивый эффект). Это разные границы. Доставка at-least-once; эффект идемпотентен: ключ `(attachment_id, pipeline_version, name)` (PK `attachment_variants`, `ON CONFLICT DO NOTHING`), детерминированные ключи объектов S3.
- **Обработка** (`src/app/pipeline.py`): **thumbnail** — JPEG, вписан в 256×256 без увеличения, EXIF-ориентация применяется, alpha на белый, размеры округляются вниз (320×200 → 256×160), quality 85, метаданные не сохраняются; **crop** — JPEG, центрированный квадрат 128×128 (мелкие допускается увеличивать); **optimized** — WebP, вписан в 1280×1280 без увеличения, `quality=80, method=4`, EXIF/ICC не сохраняются, прозрачность сохраняется. `size_bytes/sha256/content_type` вложения всегда описывают оригинал; у вариантов — свои поля. Готовность (`ready`) ставится после сохранения оригинала и всех вариантов.
- **Лимиты обработки**: API отклоняет файл >10 MiB (413) и >40 млн пикселей (413) до распаковки; worker — таймаут задания 45 с (`JOB_TIMEOUT`), число пикселей, `WORKER_CONCURRENCY` (= prefetch = число потоков), лимит памяти контейнера 768 MiB; JPEG >2560×2560 декодируется сразу в масштабе 1/2.
- **Retry**: ограниченное число попыток `MAX_ATTEMPTS=5`; временные ошибки (S3/БД/таймаут) → `queued` + новая запись outbox с отложенной публикацией; задержка — **экспоненциальный backoff с jitter** (случайное значение в [50 %, 100 %] от `min(60 с, 2 с × 2^(n−1))`); постоянные ошибки (нечитаемое изображение) → сразу `failed` с `error_code` (`invalid_image`, `image_too_large`, `processing_failed`); исчерпание попыток → `failed/retry_exhausted`.
- **Реестр failed и DLQ**: реестр — таблица `attachments` со `status=failed` + `job_events` (журнал событий, без секретов); `python -m app.replay list`. Нечитаемые сообщения брокера (`schema_version`, типы) отправляются в DLQ `attachment.process.dead` через DLX. **Replay** (`python -m app.replay replay <id>`) допускается только для `failed` (иначе `skipped`): новая попытка, новый `job_id`, запись `operator_replay` в `job_events`; повторное исполнение уже успешной работы невозможно.
- **Lease и зависшие задачи**: при захвате задание получает lease (`LEASE_SECONDS=60`); sweeper worker возвращает просроченные `processing` в `queued` (или `failed/retry_exhausted`); публиковать результат может только владелец lease (`FOR UPDATE` + проверка владельца).
- **Outbox** (уровень 4): relay в API и worker читает `FOR UPDATE SKIP LOCKED`; после краха между COMMIT и publish задание публикуется автоматически (проверено). Повторная публикация после краха между confirm и `published_at` даёт дубликат, обрабатываемый как no-op.
- **Очистка staging**: удаляется после COMMIT результата; для `failed` хранится `STAGING_FAILED_RETENTION_SECONDS` (7 суток) для разбора/replay; фоновая очистка не затрагивает текущую работу.
- **Backpressure**: перед приёмом тела API проверяет число активных заданий; при `queued+processing ≥ MAX_QUEUED_JOBS` — `503 unavailable` + `Retry-After` (контракт допускает 503). Readiness API не зависит от брокера и worker (их отказ не должен останавливать приём/чтение).
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| lab4 сохранён с переходом 201→202; smoke (thumbnail 256×160, SHA-256, размеры JPEG) | `tests/smoke.py` | PASS | `evidence/checks.txt` |
| 202 только после устойчивого приёма; остановка API после ответа не теряет работу | `test_resilience5` №1, №4 | worker остановлен → upload 202 → сообщение с вложением → рестарт API → worker поднят → ready | `evidence/checks.txt` |
| Broker и worker в стенде, durable, ack после эффекта; недоступность worker не заставляет API делать resize | описание; №1 | задание ждёт в `queued`, API ничего не обрабатывает | `evidence/checks.txt` |
| Рестарт брокера с volume, принятое не исчезает | №2 | ready после рестарта RabbitMQ | `evidence/checks.txt` |
| Брокер недоступен при приёме → outbox | №3 | 202 при остановленном брокере; в outbox 1 неопубликованная строка; после старта брокера — ready, outbox пуст | `evidence/checks.txt` |
| Crash между COMMIT и publish | №4 (kill всех API при остановленном брокере) | задание доведено без ручного создания | `evidence/checks.txt` |
| Остановка worker после записи в S3 до COMMIT / после COMMIT до ack | №5, №6 (`crash_once:…`) | lease истёк → задание завершено тем же результатом, `lease_expired`+`ready` в журнале; повторная доставка — no-op, вариантов по-прежнему 3 | `evidence/checks.txt` |
| Ограниченный retry, backoff+jitter, failed, DLQ/реестр, replay | №7 | 3 попытки → `failed/retry_exhausted`, 2 события `retry_scheduled`; реестр содержит вложение; replay → ready; повторный replay → `skipped` | `evidence/checks.txt` |
| Варианты по контракту, EXIF, alpha, без увеличения | `test_pipeline.py` (9 тестов) | thumbnail 256×160 / EXIF 100×300 → 85×256 / 50×30 не увеличивается; crop 128×128; WebP; alpha→белый; оригинал побайтно | `evidence/checks.txt` |
| Повторная доставка не размножает варианты | `test_ready_job_redelivery…` | 3 варианта до и после | `evidence/checks.txt` |
| Очистка staging | `test_staging_is_cleaned…` | staging готового удалён, failed — хранится | `evidence/checks.txt` |
| Persistence/failover | `tests/resilience.py` | PASS ×3; `StartedAt` всех 7 сервисов изменился при рестарте | `evidence/checks.txt` |
| **Трек 1** | `research/queue_overload.py` | см. ниже | `evidence/queue_overload.json\|png` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперименты и отказы (трек 1)
## Эксперименты и отказы
**Гипотеза.** Пропускная способность worker с ростом concurrency/prefetch увеличивается только пока есть свободный CPU и не растёт память; на крупных изображениях лишний параллелизм не ускоряет, но кратно увеличивает RSS и p95. Явный лимит принятой работы (backpressure) делает отказы предсказуемыми, а принятая работа не теряется; после разгрузки система принимает новые загрузки.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Методика** (`research/queue_overload.py`; Docker Desktop WSL2; worker **1 CPU / 768 MiB**; один процесс aiohttp, 8 клиентов-загрузчиков). Матрица 3 × 3: `WORKER_CONCURRENCY` ∈ {1, 2, 4} × изображения {small 320×200 PNG 109 КиБ ×80, medium 2400×1600 PNG 1.9 МиБ ×24, large 6000×4000 JPEG 3.0 МиБ ×12}; перед каждой ячейкой worker пересоздаётся (чистый пик RSS = `VmHWM`); очередь и возраст старейшего задания снимаются каждые 0.4 с; время до ready — `ready_at − accepted_at` из БД.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| concurrency | small: throughput (готово/с) / p95 (с) / RSS (МиБ) | medium: throughput / p95 / RSS | large: throughput / p95 / RSS |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| 1 | 28.9 / 2.46 / 75 | 4.21 / 5.05 / 136 | 3.34 / 3.37 / 161 |
| 2 | 43.7 / 1.60 / 76 | **4.71 / 4.69** / 194 | **3.77 / 2.96** / 254 |
| 4 | **51.8 / 1.26** / 80 | 3.97 / 5.40 / 322 | 3.60 / 3.12 / 430 |
Все 116 принятых файлов завершились `ready`, `failed` = 0, OOM-killer не срабатывал (пик 430 МиБ из 768). Пик глубины очереди/возраста старейшей задачи зависит от темпа приёма (≈20–66 заданий, до 5 с).
Выводы: (1) для мелких файлов прирост от concurrency почти линейный (29 → 44 → 52/с: пока есть CPU); (2) для крупных изображений **оптимум — concurrency 2** (лучшие throughput и p95); при 4 процесс упирается в лимит 1 CPU и задержка растёт (medium: p95 4.69 → 5.40 с), а RSS растёт кратно (136 → 194 → 322 МиБ) — компромисс latency/ресурсы; (3) память масштабируется примерно как `concurrency × размер декодируемого изображения`, поэтому лимит памяти worker определяет допустимый prefetch для крупных файлов (рекомендация: `WORKER_CONCURRENCY=2` при 768 МиБ).
**Backpressure и восстановление** (`MAX_QUEUED_JOBS=30`, worker concurrency 1, burst 200 мелких загрузок 32 клиентами за 0.5 с): принято **57**, отклонено **143** ответами `503 + Retry-After`; пик очереди 48 (превышение лимита на число одновременных проверок — проверка лимита выполняется до приёма и не является строгой блокировкой); все 57 принятых заданий завершились `ready` за 2.6 с (`failed` 0, потерь 0, p95 = 1.9 с); после разгрузки 3 новые загрузки приняты (восстановление). График: `evidence/queue_overload_backpressure.png`.
**Ограничения методики.** Один ноутбук; лимит CPU worker 1.0 — часть результата (при большем CPU оптимум concurrency сместится); темп приёма ограничен клиентом (≈8 соединений), поэтому глубина очереди на крупных файлах невелика; повтор каждой ячейки не выполнялся (единичный прогон), различия <10 % (например, medium 2 vs 4) следует считать на границе погрешности; throughput средних/крупных файлов включает время приёма байт через API.
### Окна отказа (`contracts/attachment-states.md`)
| Окно | Воздействие | Результат |
| --- | --- | --- |
| после staging до записи задания | сбой БД после записи в staging (lab4-механизм) | staging удаляется компенсацией; иначе reconcile (lab4) |
| после COMMIT до publish | остановка брокера + kill всех API | outbox довёл задание |
| после publish до confirm | таймаут/разрыв | публикация повторяется, дубликат — no-op |
| после записи S3 до COMMIT результата | `crash_once:after_s3_before_commit` | lease → повтор тем же результатом |
| после COMMIT результата до ack | `crash_once:after_commit_before_ack` | redelivery = no-op + ack |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один узел RabbitMQ, S3, PostgreSQL (SPOF). Отложенные повторы реализованы через outbox (брокерных delayed-exchange нет).
- Один worker-процесс по умолчанию (`WORKER_REPLICAS` можно увеличить); lease и `SKIP LOCKED`/условные UPDATE допускают несколько worker.
- Конкурентные записи старой и новой `pipeline_version` защищены ключом эффекта и условием `pipeline_version <= job.pipeline_version`; версионирование нескольких пайплайнов (трек 2) не реализовано.
- Подтверждено экспериментами: оптимум concurrency для крупных изображений, рост памяти, backpressure и восстановление; гипотеза без доказательства: поведение при >1 worker и при другом лимите CPU.
+239
View File
@@ -0,0 +1,239 @@
# Стенд lab5: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий).
# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md.
name: mtusi-lab5
x-common: &common
restart: unless-stopped
logging:
driver: json-file
options: {max-size: "10m", max-file: "3"}
services:
postgres:
<<: *common
image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql/data
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 12
mem_limit: 512m
cpus: ${POSTGRES_CPUS:-1.0}
networks: [backend]
valkey:
<<: *common
image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec",
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 12
mem_limit: 128m
cpus: 0.5
networks: [backend]
# Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз).
# Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities).
# Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403).
seaweedfs:
<<: *common
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
entrypoint: ["/bin/sh", "-c"]
command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"]
secrets: [seaweed_s3_config]
volumes:
- seaweeddata:/data
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"]
interval: 5s
timeout: 3s
retries: 24
start_period: 10s
mem_limit: 768m
cpus: ${S3_CPUS:-1.0}
networks: [backend]
# Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает.
s3-init:
build: .
image: mtusi-messenger:lab5
command: ["python", "-m", "app.s3_bootstrap"]
environment:
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key]
depends_on:
seaweedfs: {condition: service_healthy}
restart: "no"
networks: [backend]
# Брокер: durable quorum-очередь, persistent-сообщения, publisher confirms; данные на volume rabbitmqdata.
# hostname фиксирован: имя узла RabbitMQ определяет каталог данных, иначе очередь не переживёт пересоздание контейнера.
rabbitmq:
<<: *common
image: rabbitmq:4.1-alpine@sha256:6a0f7984185cd2cdbf78a184b4c17abe88ecbbda6006a7acf17dbc7e4307f4e2
hostname: rabbitmq
# RabbitMQ 4.x: RABBITMQ_DEFAULT_USER/PASS объявлены устаревшими -> учётные данные в конфигурационном файле (secret)
secrets:
- source: rabbitmq_conf
target: /etc/rabbitmq/conf.d/20-credentials.conf
mode: 0444
volumes:
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 8s
retries: 12
mem_limit: 512m
cpus: ${RABBITMQ_CPUS:-1.0}
networks: [backend]
migrate:
build: .
image: mtusi-messenger:lab5
command: ["python", "-m", "app.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres: {condition: service_healthy}
restart: "no"
networks: [backend]
api:
<<: *common
build: .
image: mtusi-messenger:lab5
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL_FILE: /run/secrets/database_url
VALKEY_URL: redis://valkey:6379/0
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
MAX_INFLIGHT: ${MAX_INFLIGHT:-64}
ACCESS_LOG: ${ACCESS_LOG:-1}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
FAULT_INJECTION: ${API_FAULT_INJECTION:-}
BROKER_URL_FILE: /run/secrets/broker_url
MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-200}
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
stop_grace_period: 15s
mem_limit: ${API_MEM:-256m}
cpus: ${API_CPUS:-1.0}
deploy:
replicas: ${API_REPLICAS:-2}
networks: [backend]
# Worker: отдельный процесс (тот же образ). Берёт задания из RabbitMQ, обрабатывает изображения, пишет результат в S3 и БД.
worker:
<<: *common
build: .
image: mtusi-messenger:lab5
command: ["python", "-m", "app.worker"]
environment:
SERVICE_NAME: messenger-worker
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5}
LEASE_SECONDS: ${LEASE_SECONDS:-60}
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2}
FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-}
ACCESS_LOG: "0"
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
healthcheck:
test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker.alive') < 30 else 1)"]
interval: 10s
timeout: 5s
retries: 3
start_period: 20s
stop_grace_period: 30s
mem_limit: ${WORKER_MEM:-768m}
cpus: ${WORKER_CPUS:-1.0}
deploy:
replicas: ${WORKER_REPLICAS:-1}
networks: [backend]
lb:
<<: *common
image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
- api
ports:
- "127.0.0.1:8080:8080"
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"]
interval: 5s
timeout: 3s
retries: 6
mem_limit: 128m
cpus: ${LB_CPUS:-1.0}
stop_grace_period: 10s
networks: [backend]
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
seaweed_s3_config:
file: ./secrets/seaweed_s3.json
s3_admin_access_key:
file: ./secrets/s3_admin_access_key
s3_admin_secret_key:
file: ./secrets/s3_admin_secret_key
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
rabbitmq_conf:
file: ./secrets/rabbitmq.conf
broker_url:
file: ./secrets/broker_url
volumes:
pgdata:
valkeydata:
seaweeddata:
rabbitmqdata:
networks:
backend:
Binary file not shown.
+49
View File
@@ -0,0 +1,49 @@
$ python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ python3 tests/smoke.py (upload -> 202 queued -> ready, thumbnail 256x160)
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
# контейнеры до перезапуска:
/mtusi-lab5-api-1 StartedAt=2026-10-02T23:58:35.341793495Z
/mtusi-lab5-api-2 StartedAt=2026-10-02T23:58:35.523444484Z
/mtusi-lab5-worker-1 StartedAt=2026-10-02T23:58:48.76269165Z
/mtusi-lab5-rabbitmq-1 StartedAt=2026-10-02T23:53:22.014370477Z
/mtusi-lab5-seaweedfs-1 StartedAt=2026-10-02T23:46:39.837097835Z
/mtusi-lab5-postgres-1 StartedAt=2026-10-02T23:46:39.850517882Z
/mtusi-lab5-valkey-1 StartedAt=2026-10-02T23:46:39.827928529Z
$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd)
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
# контейнеры после перезапуска:
/mtusi-lab5-api-1 StartedAt=2026-10-02T23:59:33.742707516Z
/mtusi-lab5-api-2 StartedAt=2026-10-02T23:59:32.383092893Z
/mtusi-lab5-worker-1 StartedAt=2026-10-02T23:59:28.718812327Z
/mtusi-lab5-rabbitmq-1 StartedAt=2026-10-02T23:59:29.238592291Z
/mtusi-lab5-seaweedfs-1 StartedAt=2026-10-02T23:59:29.413843719Z
/mtusi-lab5-postgres-1 StartedAt=2026-10-02T23:59:28.458564542Z
/mtusi-lab5-valkey-1 StartedAt=2026-10-02T23:59:29.099168068Z
$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh (1-я реплика)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 0, "first_recovery_seconds": 2.328}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
(2-я реплика)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 0, "first_recovery_seconds": 2.328}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ python -m unittest test_api.py
Ran 12 tests in 2.390s
OK
$ python -m unittest test_idempotency.py
Ran 9 tests in 1.971s
OK
$ python -m unittest test_attachments.py
Ran 11 tests in 5.686s
OK
$ python student-tests/test_pipeline.py (интерпретатор с Pillow: пиксельные проверки alpha -> белый фон)
Ran 9 tests in 10.648s
OK
$ python student-tests/test_topology.py
Ran 6 tests in 1.514s
OK
$ python student-tests/test_resilience5.py (окна отказа: worker/брокер/outbox/lease/retry/replay)
Ran 7 tests in 139.354s
OK
+254
View File
@@ -0,0 +1,254 @@
{
"date": "2026-10-03 02:55:33",
"args": {
"out": "evidence",
"concurrency": [
1,
2,
4
],
"n_small": 80,
"n_medium": 24,
"n_large": 12,
"limit": 30,
"burst": 200,
"b_medium": false
},
"worker_limits": {
"cpus": 1.0,
"memory": "768m"
},
"image_kib": {
"small": 109,
"medium": 1961,
"large": 3100
},
"matrix": [
{
"concurrency": 1,
"image": "small",
"image_kib": 109,
"uploads": 80,
"accepted": 80,
"accepted_in_s": 0.3,
"all_finished": true,
"peak_queue_depth": 66,
"peak_oldest_age_s": 2.2,
"worker_peak_rss_mib": 74.7,
"worker_oom_killed": "false",
"ready": 80,
"failed": 0,
"p50_s": 1.33,
"p95_s": 2.46,
"max_s": 2.55,
"throughput_per_s": 28.9,
"span_s": 2.8
},
{
"concurrency": 1,
"image": "medium",
"image_kib": 1961,
"uploads": 24,
"accepted": 24,
"accepted_in_s": 0.5,
"all_finished": true,
"peak_queue_depth": 22,
"peak_oldest_age_s": 4.8,
"worker_peak_rss_mib": 135.6,
"worker_oom_killed": "false",
"ready": 24,
"failed": 0,
"p50_s": 2.9,
"p95_s": 5.05,
"max_s": 5.29,
"throughput_per_s": 4.21,
"span_s": 5.7
},
{
"concurrency": 1,
"image": "large",
"image_kib": 3100,
"uploads": 12,
"accepted": 12,
"accepted_in_s": 0.4,
"all_finished": true,
"peak_queue_depth": 11,
"peak_oldest_age_s": 3.3,
"worker_peak_rss_mib": 161.2,
"worker_oom_killed": "false",
"ready": 12,
"failed": 0,
"p50_s": 2.03,
"p95_s": 3.37,
"max_s": 3.37,
"throughput_per_s": 3.34,
"span_s": 3.6
},
{
"concurrency": 2,
"image": "small",
"image_kib": 109,
"uploads": 80,
"accepted": 80,
"accepted_in_s": 0.2,
"all_finished": true,
"peak_queue_depth": 53,
"peak_oldest_age_s": 1.2,
"worker_peak_rss_mib": 76.3,
"worker_oom_killed": "false",
"ready": 80,
"failed": 0,
"p50_s": 0.86,
"p95_s": 1.6,
"max_s": 1.64,
"throughput_per_s": 43.67,
"span_s": 1.8
},
{
"concurrency": 2,
"image": "medium",
"image_kib": 1961,
"uploads": 24,
"accepted": 24,
"accepted_in_s": 0.5,
"all_finished": true,
"peak_queue_depth": 22,
"peak_oldest_age_s": 4.2,
"worker_peak_rss_mib": 194.1,
"worker_oom_killed": "false",
"ready": 24,
"failed": 0,
"p50_s": 2.8,
"p95_s": 4.69,
"max_s": 4.7,
"throughput_per_s": 4.71,
"span_s": 5.1
},
{
"concurrency": 2,
"image": "large",
"image_kib": 3100,
"uploads": 12,
"accepted": 12,
"accepted_in_s": 0.4,
"all_finished": true,
"peak_queue_depth": 12,
"peak_oldest_age_s": 2.7,
"worker_peak_rss_mib": 253.8,
"worker_oom_killed": "false",
"ready": 12,
"failed": 0,
"p50_s": 2.03,
"p95_s": 2.96,
"max_s": 2.96,
"throughput_per_s": 3.77,
"span_s": 3.2
},
{
"concurrency": 4,
"image": "small",
"image_kib": 109,
"uploads": 80,
"accepted": 80,
"accepted_in_s": 0.3,
"all_finished": true,
"peak_queue_depth": 52,
"peak_oldest_age_s": 1.1,
"worker_peak_rss_mib": 80.3,
"worker_oom_killed": "false",
"ready": 80,
"failed": 0,
"p50_s": 0.77,
"p95_s": 1.26,
"max_s": 1.3,
"throughput_per_s": 51.79,
"span_s": 1.5
},
{
"concurrency": 4,
"image": "medium",
"image_kib": 1961,
"uploads": 24,
"accepted": 24,
"accepted_in_s": 0.8,
"all_finished": true,
"peak_queue_depth": 24,
"peak_oldest_age_s": 4.9,
"worker_peak_rss_mib": 321.7,
"worker_oom_killed": "false",
"ready": 24,
"failed": 0,
"p50_s": 3.81,
"p95_s": 5.4,
"max_s": 5.4,
"throughput_per_s": 3.97,
"span_s": 6.1
},
{
"concurrency": 4,
"image": "large",
"image_kib": 3100,
"uploads": 12,
"accepted": 12,
"accepted_in_s": 0.4,
"all_finished": true,
"peak_queue_depth": 12,
"peak_oldest_age_s": 2.7,
"worker_peak_rss_mib": 430.2,
"worker_oom_killed": "false",
"ready": 12,
"failed": 0,
"p50_s": 2.11,
"p95_s": 3.12,
"max_s": 3.12,
"throughput_per_s": 3.6,
"span_s": 3.3
}
],
"backpressure": {
"MAX_QUEUED_JOBS": 30,
"burst": 200,
"attempt_statuses": {
"202": 57,
"503": 143
},
"accepted": 57,
"peak_queue_depth": 48,
"all_accepted_finished": true,
"accepted_in_s": 0.5,
"drained_after_s": 2.6,
"retry_after_header_seen": true,
"recovered_new_uploads_accepted": true,
"no_accepted_job_lost": true,
"ready": 57,
"failed": 0,
"p95_s": 1.89
},
"backpressure_queue_samples": [
[
0.2,
0,
0.0
],
[
0.7,
48,
0.5
],
[
1.3,
31,
1.0
],
[
1.9,
16,
1.6
],
[
2.4,
0,
0.0
]
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

+50
View File
@@ -0,0 +1,50 @@
# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы.
global
log stdout format raw local0 notice
maxconn 4096
defaults
mode http
log global
option dontlognull
option forwardfor
# connect короткий: недоступная реплика быстро заменяется другой (redispatch)
timeout connect 1s
timeout client 30s
timeout server 30s
timeout http-request 10s
timeout http-keep-alive 5s
timeout queue 5s
retries 3
option redispatch 1
retry-on conn-failure empty-response response-timeout 0rtt-rejected
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
hold other 2s
hold obsolete 2s
frontend public
bind :8080
default_backend api
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
frontend lb_health
bind :8081
monitor-uri /healthz
backend api
balance roundrobin
# Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён.
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-8 api:8080
+52
View File
@@ -0,0 +1,52 @@
-- 0001: пользователи, чаты, участники, сообщения, сессии.
-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции.
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении.
CREATE UNIQUE INDEX users_username_key ON users (username);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort.
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
chat_created_at timestamptz NOT NULL,
ord smallint NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users (id),
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000)
);
-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n
-- обслуживается этим индексом без сортировки и без сканирования всей истории.
CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id);
-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены.
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT now(),
expires_at timestamptz NOT NULL,
revoked_at timestamptz
);
CREATE INDEX sessions_expires_at ON sessions (expires_at);
+17
View File
@@ -0,0 +1,17 @@
-- 0002: общая идемпотентность отправки сообщений (lab3).
-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages).
-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
idem_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idem_key),
CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
-- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT
CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED
);
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
+35
View File
@@ -0,0 +1,35 @@
-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос).
-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения):
-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения.
-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс.
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision)
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
request_hash bytea)
LANGUAGE plpgsql AS $$
DECLARE
v_new uuid := gen_random_uuid();
v_ins uuid;
v_created timestamptz;
BEGIN
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea;
RETURN;
END IF;
DELETE FROM idempotency_keys k
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
AND k.created_at < now() - make_interval(secs => p_ttl);
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, v_new)
ON CONFLICT DO NOTHING
RETURNING message_id INTO v_ins;
IF v_ins IS NOT NULL THEN
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
RETURNING created_at INTO v_created;
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash;
ELSE
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
END IF;
END;
$$;
+90
View File
@@ -0,0 +1,90 @@
-- 0004 (lab4): метаданные вложений в PostgreSQL, байты — в приватном S3-бакете.
-- Ключ объекта назначает сервер (att/<id>/original); filename — только отображаемое имя.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
status text NOT NULL DEFAULT 'ready',
error_code text,
s3_key text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255)
);
CREATE UNIQUE INDEX attachments_s3_key_key ON attachments (s3_key);
CREATE INDEX attachments_chat ON attachments (chat_id);
-- Связь сообщение <-> вложения (порядок сохраняется). Вложение может быть привязано к нескольким сообщениям чата.
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id),
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
-- Отправка сообщения с вложениями: заменяет функцию из 0003 (добавлен параметр p_atts и результат msg_atts).
DROP FUNCTION send_message(uuid, uuid, text, bytea, text, double precision);
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision,
p_atts uuid[])
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
request_hash bytea, msg_atts uuid[])
LANGUAGE plpgsql AS $$
DECLARE
v_new uuid := gen_random_uuid();
v_ins uuid;
v_created timestamptz;
v_found int;
v_failed int;
BEGIN
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz,
NULL::bytea, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_atts) > 0 THEN
SELECT count(*), count(*) FILTER (WHERE a.status = 'failed') INTO v_found, v_failed
FROM attachments a WHERE a.id = ANY (p_atts) AND a.chat_id = p_chat;
IF v_found <> cardinality(p_atts) THEN
RETURN QUERY SELECT 'attachment_not_found'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
NULL::timestamptz, NULL::bytea, NULL::uuid[];
RETURN;
END IF;
IF v_failed > 0 THEN
RETURN QUERY SELECT 'attachment_failed'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
NULL::timestamptz, NULL::bytea, NULL::uuid[];
RETURN;
END IF;
END IF;
DELETE FROM idempotency_keys k
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
AND k.created_at < now() - make_interval(secs => p_ttl);
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, v_new)
ON CONFLICT DO NOTHING
RETURNING message_id INTO v_ins;
IF v_ins IS NOT NULL THEN
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
RETURNING created_at INTO v_created;
INSERT INTO message_attachments (message_id, attachment_id, position)
SELECT v_new, a, (o - 1)::smallint FROM unnest(p_atts) WITH ORDINALITY AS t(a, o);
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash, p_atts;
ELSE
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash,
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
END IF;
END;
$$;
+60
View File
@@ -0,0 +1,60 @@
-- 0005 (lab5): асинхронная обработка вложений — состояния, варианты, outbox, журнал событий.
-- Расширение обратно совместимо с lab4: старые строки (ready) остаются валидными.
ALTER TABLE attachments
ADD COLUMN staging_key text,
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN attempt integer NOT NULL DEFAULT 0,
ADD COLUMN job_id uuid,
ADD COLUMN lease_owner text,
ADD COLUMN lease_expires_at timestamptz,
ADD COLUMN processing_started_at timestamptz,
ADD COLUMN accepted_at timestamptz NOT NULL DEFAULT clock_timestamp(),
ADD COLUMN ready_at timestamptz;
-- очередь/backlog и поиск зависших processing без полного скана
CREATE INDEX attachments_active ON attachments (status, accepted_at) WHERE status IN ('queued', 'processing');
CREATE INDEX attachments_lease ON attachments (lease_expires_at) WHERE status = 'processing';
-- Результаты обработки. Ключ эффекта (attachment_id, pipeline_version, name): повторная запись не создаёт второй набор вариантов.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
name text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
s3_key text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT variants_content_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp'))
);
-- Transactional outbox: намерение «обработать файл» фиксируется В ТОЙ ЖЕ транзакции, что и строка вложения.
-- Relay публикует строки в брокер (publisher confirm) и только затем ставит published_at. available_at — отложенная
-- публикация (backoff между попытками). Доставка at-least-once; обработка идемпотентна.
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL DEFAULT 1,
attempt integer NOT NULL,
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX outbox_pending ON outbox (available_at) WHERE published_at IS NULL;
-- Журнал событий пайплайна (аудит операторского replay, причины отказов; без секретов).
CREATE TABLE job_events (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL,
job_id uuid,
event text NOT NULL,
attempt integer,
detail text,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX job_events_attachment ON job_events (attachment_id, id);
+33
View File
@@ -0,0 +1,33 @@
aio-pika==10.1.0
aiobotocore==3.9.2
aiohappyeyeballs==2.7.1
aiohttp==3.14.3
aioitertools==0.13.0
aiormq==7.2.1
aiosignal==1.4.0
argon2-cffi-bindings==26.1.0
argon2-cffi==23.1.0
asyncpg==0.30.0
attrs==26.1.0
botocore==1.43.106
cffi==2.1.1
click==8.5.0
frozenlist==1.8.0
h11==0.16.0
httptools==0.8.0
idna==3.20
jmespath==1.1.0
multidict==6.9.1
pamqp==4.0.1
pillow==12.3.0
propcache==0.5.4
pycparser==3.0
python-dateutil==2.9.0.post0
redis==5.2.1
six==1.17.0
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.0
uvloop==0.21.0; sys_platform != "win32"
wrapt==2.5.0
yarl==1.25.1
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
t=25 восстановить (scripts/restore-replicas.sh)
t=40 остановить реплику B
t=55 восстановить
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
python research/failover_sequence.py --out evidence
"""
import argparse
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
def sh(*args):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
def api_ids():
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab5",
"--filter", "label=com.docker.compose.service=api").split()
def scenario(rec, plan):
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
for at, action in plan:
time.sleep(max(0, rec.t0 + at - time.time()))
t = time.time() - rec.t0
if action == "stop":
target = api_ids()[0]
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
else:
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "restore"})
async def main(args):
stand = loadgen.Stand(BASE)
await stand.setup()
rec = loadgen.Recorder()
rec.marks = []
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
await asyncio.sleep(2)
check = await loadgen.verify(stand, rec)
stops = [m for m in rec.marks if m["action"] == "stop"]
restores = [m for m in rec.marks if m["action"] == "restore"]
windows = []
for i, stop in enumerate(stops):
t0 = stop["t"]
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
last_err = max((e[0] for e in errs), default=t0)
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
"failed_attempts_within_10s": len(errs),
"error_statuses": sorted({str(e[2]) for e in errs}),
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
series = loadgen.per_second(rec)
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
"total": loadgen.summarize(rec), "verification": check,
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
for t, b in sorted(series.items())]}
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "failover_sequence.png"))
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
tl = report["timeline"]
t = [x["t"] for x in tl]
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
for a in ax:
for m in report["marks"]:
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
fig.tight_layout(); fig.savefig(path, dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--seconds", type=int, default=70)
ap.add_argument("--concurrency", type=int, default=16)
asyncio.run(main(ap.parse_args()))
+172
View File
@@ -0,0 +1,172 @@
"""Генератор нагрузки для экспериментов lab5+ (asyncio + aiohttp, closed-loop).
Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %).
При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно
повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых
сообщений (для сверки с историей), число повторов.
"""
import asyncio
import base64
import random
import time
import uuid
import aiohttp
RETRIES = 3
class Stand:
"""Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента."""
def __init__(self, base, users=20, chats=20):
self.base, self.n_users, self.n_chats = base, users, chats
self.users, self.chats = [], []
async def setup(self):
timeout = aiohttp.ClientTimeout(total=30)
async with aiohttp.ClientSession(timeout=timeout) as s:
for i in range(self.n_users):
name = "ld_%s" % uuid.uuid4().hex[:14]
pw = "load-test-password-" + uuid.uuid4().hex[:8]
async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r:
user = await r.json()
assert r.status == 201, user
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
sess = await r.json()
assert r.status == 201, sess
self.users.append({"id": user["id"], "token": sess["token"]})
for i in range(self.n_chats):
members = random.sample(self.users, 3)
owner = members[0]
async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]},
headers={"Authorization": "Bearer " + owner["token"]}) as r:
chat = await r.json()
assert r.status == 201, chat
self.chats.append({"id": chat["id"], "members": members})
class Recorder:
def __init__(self):
self.t0 = time.time()
self.events = [] # (t, kind, status, latency_ms, instance)
self.confirmed = [] # (chat_id, message_id, text)
self.retries = 0
self.final_failures = 0
self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!)
async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0):
connector = aiohttp.TCPConnector(limit=2)
timeout = aiohttp.ClientTimeout(total=5)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s:
n = 0
while not deadline_fn():
chat = random.choice(stand.chats)
user = random.choice(chat["members"])
hdr = {"Authorization": "Bearer " + user["token"]}
if random.random() < read_ratio:
t = time.perf_counter()
try:
async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r:
await r.read()
rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id")))
except Exception as exc:
rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None))
continue
key = uuid.uuid4().hex
text = "m-%s-%d" % (key[:8], n) + "я" * text_size
n += 1
for attempt in range(RETRIES + 1):
t = time.perf_counter()
try:
async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text},
headers=dict(hdr, **{"Idempotency-Key": key})) as r:
body = await r.json(content_type=None)
status, inst = r.status, r.headers.get("X-Instance-Id")
except Exception as exc:
status, inst, body = type(exc).__name__, None, None
rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst))
if status == 201:
rec.confirmed.append((chat["id"], body["id"], text))
break
if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429):
rec.retries += 1
await asyncio.sleep(0.05 * (attempt + 1))
continue
rec.final_failures += 1
rec.unconfirmed.append((chat["id"], key, text))
break
if think:
await asyncio.sleep(think)
async def run_load(stand, rec, seconds, concurrency, **kw):
end = time.time() + seconds
await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency)))
async def verify(stand, rec):
"""Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза."""
timeout = aiohttp.ClientTimeout(total=30)
stored = {}
async with aiohttp.ClientSession(timeout=timeout) as s:
for chat in stand.chats:
hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]}
cursor, items = None, []
while True:
url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "")
async with s.get(url, headers=hdr) as r:
page = await r.json()
assert r.status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
stored[chat["id"]] = items
ids = {m["id"] for items in stored.values() for m in items}
texts = [m["text"] for items in stored.values() for m in items]
confirmed_missing = [c for c in rec.confirmed if c[1] not in ids]
dup_texts = len(texts) - len(set(texts))
unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts))
return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing),
"duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed),
"unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures}
def per_second(rec, kind="POST"):
buckets = {}
for t, k, status, lat, inst in rec.events:
if k != kind:
continue
b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()})
if status == 201 or (kind == "GET" and status == 200):
b["ok"] += 1
b["lat"].append(lat)
if inst:
b["instances"].add(inst)
else:
b["err"] += 1
return buckets
def pct(values, p):
if not values:
return None
values = sorted(values)
return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2)
def summarize(rec, t_from=None, t_to=None, kind="POST"):
rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)]
if not rows:
return {}
dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows))
ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)]
errs = {}
for e in rows:
if not (e[2] == 201 or (kind == "GET" and e[2] == 200)):
errs[str(e[2])] = errs.get(str(e[2]), 0) + 1
return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None,
"error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)}
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
"""Трек 1 (очередь под перегрузкой), lab5.
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready);
после разгрузки новая загрузка снова принимается.
python research/queue_overload.py --out evidence
"""
import argparse
import asyncio
import base64
import io
import json
import os
import statistics
import subprocess
import threading
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROJECT = "mtusi-lab5"
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
def psql(sql):
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
def healthy(service, expected=1):
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
def recreate(service, env, wait_seconds=90):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
deadline = time.time() + wait_seconds
while time.time() < deadline:
if healthy(service, 1 if service == "worker" else 2):
break
time.sleep(1)
time.sleep(6 if service == "api" else 3)
def make_images(root):
import random
random.seed(11)
out = {}
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
def noisy(w, h):
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
return Image.blend(base, grad, 0.5)
buf = io.BytesIO()
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
out["medium"] = buf.getvalue()
buf = io.BytesIO()
noisy(6000, 4000).save(buf, "JPEG", quality=88)
out["large"] = buf.getvalue()
return out
async def register(s):
name = "qo_%s" % uuid.uuid4().hex[:12]
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
class QueueSampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.samples, self.stop = [], threading.Event()
self.t0 = time.time()
def run(self):
while not self.stop.is_set():
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
try:
n, age = out.split("|")
self.samples.append((time.time() - self.t0, int(n), float(age)))
except ValueError:
pass
self.stop.wait(0.4)
def worker_peak_rss():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
try:
return round(int(out.split()[1]) / 1024, 1)
except (IndexError, ValueError):
return None
async def burst(s, token, chat, data, n, clients, retry_503=False):
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
async def one(i):
async with sem:
for attempt in range(60):
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
body = await r.json(content_type=None)
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
if r.status == 202:
accepted.append(body["id"])
return
if r.status in (503, 429) and retry_503:
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
continue
return
except aiohttp.ClientError:
await asyncio.sleep(0.2)
await asyncio.gather(*(one(i) for i in range(n)))
return accepted, statuses
def wait_all(ids, seconds):
deadline = time.time() + seconds
while time.time() < deadline:
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
if left == "0":
return True
time.sleep(0.5)
return False
def stats_for(ids):
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
% ",".join("'%s'" % i for i in ids)).splitlines()
ttr, ready, failed, first, last = [], 0, 0, None, None
for line in rows:
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
if status == "ready" and t:
ready += 1
ttr.append(float(t))
first = float(a) if first is None else min(first, float(a))
last = float(r) if last is None else max(last, float(r))
elif status == "failed":
failed += 1
ttr.sort()
pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731
span = (last - first) if first is not None else None
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
async def part_a(args, images):
cells = []
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
for conc in args.concurrency:
for size, n in sizes.items():
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images[size], n, 8)
accepted_in = time.time() - t0
done = wait_all(ids, 400)
sampler.stop.set()
sampler.join()
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
cell.update(stats_for(ids))
print(json.dumps(cell, ensure_ascii=False))
cells.append(cell)
return cells
async def part_b(args, images):
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
recreate("worker", {"WORKER_CONCURRENCY": "1"})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
accepted_in = time.time() - t0
done = wait_all(ids, 300)
drained_after = time.time() - t0
sampler.stop.set()
sampler.join()
# восстановление: после разгрузки новая загрузка принимается
extra, st2 = await burst(s, token, chat, images["small"], 3, 1)
wait_all(extra, 60)
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
"retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3,
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
print(json.dumps(res, ensure_ascii=False))
recreate("api", {})
recreate("worker", {})
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
async def main(args):
images = make_images(ROOT)
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
report["matrix"] = await part_a(args, images)
recreate("worker", {})
report["backpressure"], samples = await part_b(args, images)
report["backpressure_queue_samples"] = samples
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "queue_overload.png"))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
sizes = ["small", "medium", "large"]
concs = sorted({c["concurrency"] for c in report["matrix"]})
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
for i, c in enumerate(concs):
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
fig.tight_layout(); fig.savefig(path, dpi=110)
s = report["backpressure_queue_samples"]
fig, ax = plt.subplots(figsize=(8, 4))
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
ap.add_argument("--n-small", type=int, default=80)
ap.add_argument("--n-medium", type=int, default=24)
ap.add_argument("--n-large", type=int, default=12)
ap.add_argument("--limit", type=int, default=30)
ap.add_argument("--burst", type=int, default=200)
ap.add_argument("--b-medium", action="store_true")
asyncio.run(main(ap.parse_args()))
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh.
Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id."""
import argparse
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
async def main(args):
stand = loadgen.Stand("http://127.0.0.1:8080")
await stand.setup()
rec = loadgen.Recorder()
info = {}
def update():
time.sleep(max(0, rec.t0 + 10 - time.time()))
info["start"] = round(time.time() - rec.t0, 2)
out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True)
info["end"] = round(time.time() - rec.t0, 2)
info["script_exit"] = out.returncode
info["script_output"] = out.stdout.strip().splitlines()[-6:]
threading.Thread(target=update, daemon=True).start()
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
await asyncio.sleep(1)
check = await loadgen.verify(stand, rec)
instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]}
instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]}
instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]}
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision,
"instances_before": sorted(instances_before), "instances_during": sorted(instances_during),
"instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after),
"before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]),
"after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check,
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0}
with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--seconds", type=int, default=60)
ap.add_argument("--concurrency", type=int, default=16)
ap.add_argument("--revision", default="rolling-%d" % int(time.time()))
asyncio.run(main(ap.parse_args()))
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются.
set -eu
cd "$(dirname "$0")/.."
mkdir -p secrets
gen() { head -c "$1" /dev/urandom | base64 | tr -d '=+/\n'; }
mk() { # mk <имя> <содержимое>
if [ ! -s "secrets/$1" ]; then printf '%s' "$2" > "secrets/$1"; echo "создан secrets/$1"; fi
}
mk postgres_password "$(gen 24)"
mk database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
# S3 (SeaweedFS): администратор (только bootstrap) и отдельные ключи API с правами на ОДИН bucket
mk s3_admin_access_key "adm$(gen 8)"
mk s3_admin_secret_key "$(gen 30)"
mk s3_access_key "api$(gen 8)"
mk s3_secret_key "$(gen 30)"
BUCKET="${S3_BUCKET:-messenger-attachments}"
if [ ! -s secrets/seaweed_s3.json ]; then
cat > secrets/seaweed_s3.json <<EOF
{
"identities": [
{"name": "admin", "credentials": [{"accessKey": "$(cat secrets/s3_admin_access_key)", "secretKey": "$(cat secrets/s3_admin_secret_key)"}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "api", "credentials": [{"accessKey": "$(cat secrets/s3_access_key)", "secretKey": "$(cat secrets/s3_secret_key)"}],
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]}
]
}
EOF
echo "создан secrets/seaweed_s3.json"
fi
mk rabbitmq_user "mq$(gen 6)"
mk rabbitmq_password "$(gen 24)"
mk rabbitmq.conf "default_user = $(cat secrets/rabbitmq_user)
default_pass = $(cat secrets/rabbitmq_password)
"
mk broker_url "amqp://$(cat secrets/rabbitmq_user):$(cat secrets/rabbitmq_password)@rabbitmq:5672/"
chmod 644 secrets/* 2>/dev/null || true # читает непривилегированный пользователь контейнера (uid 10001)
+7
View File
@@ -0,0 +1,7 @@
@echo off
rem Windows analogue of restart.sh: restart the whole stand, volumes are kept; wait until healthy.
cd /d "%~dp0.."
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb >NUL 2>&1
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb >NUL 2>&1
exit /b %errorlevel%
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey, S3. Данные не удаляются.
# После рестарта ждём healthy всех сервисов (`up --wait`), чтобы проверка сразу видела готовый стенд (S3 стартует ~10 с).
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb >/dev/null 2>&1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb >/dev/null 2>&1
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2):
# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются.
set -eu
cd "$(dirname "$0")/.."
for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do
docker rm "$cid" >/dev/null
done
N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}"
docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; }
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
# Rolling update API без простоя: реплики заменяются по одной.
# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией);
# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB ->
# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить.
# Использование: scripts/rolling-update.sh [REVISION]
set -eu
cd "$(dirname "$0")/.."
REV="${1:-$(date +%s)}"
COMPOSE="docker compose --env-file .env"
API_FILTER="--filter label=com.docker.compose.project=mtusi-lab5 --filter label=com.docker.compose.service=api"
$COMPOSE build --build-arg REV="$REV" api >/dev/null
$COMPOSE run --rm migrate >/dev/null
OLD=$(docker ps -q $API_FILTER)
N=$(echo "$OLD" | wc -w | tr -d ' ')
echo "revision=$REV replicas=$N"
for old in $OLD; do
BEFORE_FILE=$(mktemp)
docker ps -q $API_FILTER | sort > "$BEFORE_FILE"
$COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1
NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -)
rm -f "$BEFORE_FILE"
for i in $(seq 1 60); do
[ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break
sleep 1
done
sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации
docker stop --time 15 "$old" >/dev/null
docker rm "$old" >/dev/null
echo "заменена реплика $old -> $NEW"
done
+8
View File
@@ -0,0 +1,8 @@
@echo off
rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped.
set "cid="
echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2
for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab5 --filter label^=com.docker.compose.service^=api') do set "cid=%%i"
if "%cid%"=="" exit /b 3
docker stop --time 0 %cid% >NUL 2>&1
exit /b %errorlevel%
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и
# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с).
# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart
# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=mtusi-lab5" --filter "label=com.docker.compose.service=api")
[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; }
docker stop --time 0 "$cid" >/dev/null
+206
View File
@@ -0,0 +1,206 @@
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
import asyncio
import hashlib
import tempfile
import unicodedata
import uuid
from . import config, images
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
ALLOWED_TYPES = ("image/png", "image/jpeg")
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
"a.s3_key, a.created_at, a.updated_at, a.pipeline_version")
VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2}
def attachment_json(row, variants=()):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
"updated_at": format_dt(row["updated_at"])}
def valid_filename(query):
values = query.get("filename")
if not values or len(values) != 1:
raise bad("Требуется параметр filename")
name = values[0]
if not 1 <= len(name) <= 255 or not name.strip():
raise bad("filename: длина 1–255 символов")
try:
name.encode("utf-8")
except UnicodeEncodeError:
raise bad("filename содержит некорректные символы") from None
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
return name
class StreamResponse:
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
def __init__(self, content_type, length, stream):
self.content_type, self.length, self.stream = content_type, length, stream
async def send(self, send, request_id, instance_id):
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
(b"cache-control", b"private, no-store")]
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
await send({"type": "http.response.start", "status": 200, "headers": headers})
try:
while True:
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
if not chunk:
break
await send({"type": "http.response.body", "body": chunk, "more_body": True})
finally:
self.stream.close()
await send({"type": "http.response.body", "body": b""})
class AttachmentMixin:
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
def _init_attachments(self):
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
self._upload_waiting = 0
async def _receive_upload(self, req):
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
declared = req.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_UPLOAD_BYTES:
raise too_large
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
digest, size = hashlib.sha256(), 0
try:
while True:
event = await req.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
digest.update(chunk)
spool.write(chunk)
if not event.get("more_body"):
break
except BaseException:
spool.close()
raise
if size == 0:
spool.close()
raise bad("Пустой файл")
return spool, digest.hexdigest(), size
async def upload_attachment(self, req, params):
user_id = await self.authenticate(req)
chat_id = await self._require_member(user_id, params)
filename = valid_filename(req.query)
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
if media not in ALLOWED_TYPES:
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")
if backlog >= config.MAX_QUEUED_JOBS:
raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже",
{"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)})
if self._upload_waiting >= config.UPLOAD_QUEUE:
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self._upload_waiting += 1
try:
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
finally:
self._upload_waiting -= 1
try:
spool, sha256, size = await self._receive_upload(req)
try:
try:
loop = asyncio.get_running_loop()
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
except images.ImageRejected as exc:
raise ApiError(exc.status, exc.code, exc.message) from None
att_id = uuid.uuid4()
staging_key = "staging/%s" % att_id
final_key = "att/%s/original" % att_id
spool.seek(0)
await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix)
try: # 2) метаданные + задание в outbox: ОДНА транзакция
if "db_after_put" in config.FAULT_INJECTION:
raise RuntimeError("injected: сбой БД после записи в staging")
async with self.db.conn() as c:
async with c.transaction():
row = await c.fetchrow(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
"height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
"s3_key, created_at, updated_at",
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
final_key, staging_key)
await c.execute(
"INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, 1, 1)",
uuid.uuid4(), att_id)
except BaseException:
try:
if "s3_delete_fails" in config.FAULT_INJECTION:
raise OSError("injected: компенсирующее удаление не удалось")
await self.s3.delete(staging_key)
except Exception as exc:
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
raise
finally:
spool.close()
finally:
self._upload_slots.release()
self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox
return 202, attachment_json(row)
async def _attachment_for(self, req, params):
user_id = await self.authenticate(req)
att_id = parse_uuid(params["attachment_id"], "attachment_id")
row = await self.db.fetchrow(
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
"WHERE a.id = $1", att_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
async def _variants(self, row):
if row["status"] != "ready":
return []
rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
"WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"])
rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]])
return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"],
"height": r["height"], "sha256": r["sha256"]} for r in rows]
async def get_attachment(self, req, params):
row = await self._attachment_for(req, params)
return 200, attachment_json(row, await self._variants(row))
async def download_attachment(self, req, params):
row = await self._attachment_for(req, params)
variant = "original"
if "variant" in req.query:
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
raise bad("variant: original | thumbnail | crop | optimized")
variant = req.query["variant"][0]
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась")
key, content_type = row["s3_key"], row["content_type"]
if variant != "original":
v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 "
"AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant)
if v is None:
raise ApiError(404, "not_found", "Вариант отсутствует")
key, content_type = v["s3_key"], v["content_type"]
stream, length = await self.s3.open_stream(key)
return 200, StreamResponse(content_type, length, stream)
+67
View File
@@ -0,0 +1,67 @@
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
"""
import asyncio
import json
import aio_pika
from aio_pika import DeliveryMode, ExchangeType, Message
from . import config
from .web import log
QUEUE = "attachment.process"
DEAD_QUEUE = "attachment.process.dead"
DEAD_EXCHANGE = "attachment.dlx"
class Broker:
def __init__(self):
self.connection = None
self.channel = None
self.queue = None
self._lock = asyncio.Lock()
async def start(self, prefetch=None):
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
client_properties={"connection_name": config.SERVICE_NAME})
self.channel = await self.connection.channel(publisher_confirms=True)
if prefetch:
await self.channel.set_qos(prefetch_count=prefetch)
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
await dead.bind(dlx, routing_key="dead")
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
async def ensure(self, prefetch=None):
async with self._lock:
if self.channel is None or self.channel.is_closed:
await self.start(prefetch)
async def stop(self):
if self.connection:
try:
await asyncio.wait_for(self.connection.close(), 3)
except Exception:
pass
async def publish(self, job, headers=None):
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
await self.ensure()
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
message_id=job["job_id"], headers=headers or {})
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
config.BROKER_TIMEOUT)
async def queue_depth(self):
await self.ensure()
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
return q.declaration_result.message_count
async def ping(self):
return self.channel is not None and not self.channel.is_closed
+92
View File
@@ -0,0 +1,92 @@
"""Конфигурация из переменных окружения (см. .env.example)."""
import os
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
if os.environ.get(name):
return os.environ[name]
path = os.environ.get(name + "_FILE")
if path and os.path.exists(path):
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
return default
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = _int("APP_PORT", 8080)
APP_ENV = os.environ.get("APP_ENV", "lab")
DATABASE_URL = _secret("DATABASE_URL")
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
# --- S3 / вложения (lab4) ---
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
S3_POOL = _int("S3_POOL", 20)
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
# --- lab5: брокер, outbox, worker ---
SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api")
BROKER_URL = _secret("BROKER_URL")
BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0)
RELAY_BATCH = _int("RELAY_BATCH", 50)
RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0)
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки
LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь
JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease)
MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5)
RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0)
RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0)
SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0)
STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600)
MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After
BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5)
+71
View File
@@ -0,0 +1,71 @@
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
import asyncio
from contextlib import asynccontextmanager
import asyncpg
from . import config
from .web import log, unavailable
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
class Database:
def __init__(self):
self.pool = None
async def start(self):
if not config.DATABASE_URL:
log("error", "database_url_missing")
return
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
self.pool = await asyncpg.create_pool(
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
max_inactive_connection_lifetime=60,
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
async def stop(self):
if self.pool:
try:
await asyncio.wait_for(self.pool.close(), 3)
except Exception:
self.pool.terminate()
@asynccontextmanager
async def conn(self):
if self.pool is None:
raise unavailable("База данных не настроена")
try:
async with self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) as connection:
yield connection
except DEPENDENCY_ERRORS as exc:
log("warning", "db_unavailable", error_type=type(exc).__name__)
raise unavailable() from exc
async def _run(self, method, sql, args, retry):
for attempt in (0, 1):
try:
async with self.conn() as c:
return await getattr(c, method)(sql, *args)
except Exception as exc:
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
raise
async def fetch(self, sql, *args):
return await self._run("fetch", sql, args, True)
async def fetchrow(self, sql, *args, retry=True):
return await self._run("fetchrow", sql, args, retry)
async def fetchval(self, sql, *args):
return await self._run("fetchval", sql, args, True)
async def execute(self, sql, *args):
return await self._run("execute", sql, args, False)
async def ping(self):
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
+102
View File
@@ -0,0 +1,102 @@
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
"""
import struct
import zlib
from PIL import Image, UnidentifiedImageError
from . import config
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
class ImageRejected(Exception):
def __init__(self, status, code, message):
super().__init__(message)
self.status, self.code, self.message = status, code, message
def sniff(head):
if head.startswith(PNG_SIGNATURE):
return "image/png"
if head[:3] == b"\xff\xd8\xff":
return "image/jpeg"
return None
def _check_png_stream(fileobj, width, height):
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
fileobj.seek(8)
inflater = zlib.decompressobj()
total, channels, bit_depth, interlaced = 0, 0, 0, 0
while True:
head = fileobj.read(8)
if len(head) < 8:
raise ValueError("truncated")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise ValueError("chunk")
if kind == b"IHDR":
data = fileobj.read(length)
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
fileobj.read(4)
elif kind == b"IDAT":
remaining = length
while remaining:
piece = fileobj.read(min(remaining, 65536))
if not piece:
raise ValueError("truncated")
remaining -= len(piece)
pending = piece
while pending:
total += len(inflater.decompress(pending, 1 << 20))
pending = inflater.unconsumed_tail
fileobj.read(4)
elif kind == b"IEND":
break
else:
fileobj.seek(length + 4, 1)
if not inflater.eof:
raise ValueError("zlib stream incomplete")
if not interlaced:
row = (width * channels * bit_depth + 7) // 8
if total != height * (row + 1):
raise ValueError("unexpected image data size")
def validate(fileobj, declared_type):
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
fileobj.seek(0)
detected = sniff(fileobj.read(16))
if detected is None or detected != declared_type:
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
fileobj.seek(0)
try:
with Image.open(fileobj) as img:
width, height = img.size
fmt = img.format
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
if width * height > config.MAX_IMAGE_PIXELS:
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
fileobj.seek(0)
try:
img = Image.open(fileobj)
if fmt == "PNG":
img.verify()
_check_png_stream(fileobj, width, height)
else:
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
img.load()
img.close()
except Exception: # любая ошибка декодера = повреждённый файл
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
return detected, width, height
+417
View File
@@ -0,0 +1,417 @@
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
import base64
import binascii
import hashlib
import json
import os
import re
import time
import uuid
import asyncpg
from . import config, passwords
from .attachments import AttachmentMixin, StreamResponse
from .db import DEPENDENCY_ERRORS, Database
from .broker import Broker
from .outbox import Relay
from .s3 import S3
from .sessions import SessionCache, Sessions
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
ROUTES = [
("health_live", re.compile(r"/health/live"), {"GET"}),
("health_ready", re.compile(r"/health/ready"), {"GET"}),
("users", re.compile(r"/api/v1/users"), {"POST"}),
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
]
ROUTE_LABELS = {
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
}
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
BEARER = 'Bearer realm="messenger"'
BASIC = 'Basic realm="messenger", charset="UTF-8"'
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
def user_json(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
"created_at": format_dt(row["created_at"])}
def chat_json(row):
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
"created_at": format_dt(row["created_at"])}
def message_json(row):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
"text": row["text"], "created_at": format_dt(row["created_at"]),
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
class Messenger(AttachmentMixin):
def __init__(self):
self.db = Database()
self.s3 = S3()
self.broker = Broker()
self.relay = Relay(self.db, self.broker)
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
self._init_attachments()
self.cache = SessionCache()
self.sessions = Sessions(self.db, self.cache)
self.inflight = 0
self.draining = False
# --- аутентификация ------------------------------------------------------
@staticmethod
def bearer_token(req):
header = req.headers.get("authorization", "")
scheme, _, token = header.partition(" ")
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
raise unauthorized()
return token.strip()
async def authenticate(self, req):
token = self.bearer_token(req)
user_id = await self.sessions.resolve(token)
if user_id is None:
raise unauthorized()
req.user_id, req.token = user_id, token
return user_id
# --- обработчики -----------------------------------------------------------
async def health_live(self, req, params):
return 200, {"status": "ok"}
async def health_ready(self, req, params):
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
if self.draining:
raise unavailable("Сервис останавливается")
try:
await self.db.ping()
except ApiError:
raise
except Exception:
raise unavailable() from None
return 200, {"status": "ok"}
async def users(self, req, params):
req.require_json_media_type()
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
{"username", "display_name", "password"})
username = data["username"]
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
display_name = valid_text(data["display_name"], "display_name", 100)
password = valid_text(data["password"], "password", 128, allow_empty=True)
if len(password) < 12:
raise bad("password должен содержать от 12 до 128 символов")
pw_hash = await passwords.hash_password(password)
row = await self.db.fetchrow(
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
uuid.uuid4(), username, display_name, pw_hash)
if row is None:
raise ApiError(409, "conflict", "username уже занят")
return 201, user_json(row)
async def users_me(self, req, params):
user_id = await self.authenticate(req)
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
if row is None:
raise unauthorized()
return 200, user_json(row)
async def create_session(self, req, params):
header = req.headers.get("authorization", "")
scheme, _, value = header.partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized(BASIC, "Неверные учётные данные")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
username, sep, password = decoded.partition(":")
if not sep:
raise ValueError
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized(BASIC, "Неверные учётные данные") from None
row = None
if USERNAME_RE.fullmatch(username):
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
"WHERE username = $1", username)
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
if not ok:
raise unauthorized(BASIC, "Неверные учётные данные")
token, expires_at = await self.sessions.create(row["id"])
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
async def session_current(self, req, params):
await self.authenticate(req)
await self.sessions.revoke(req.token)
return 204, None
async def chats(self, req, params):
user_id = await self.authenticate(req)
if req.method == "GET":
limit, after = page_params(req.query, "c", user_id)
if after is None:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
else:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
req.require_json_media_type()
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
title = valid_text(data["title"], "title", 100)
members = data["member_ids"]
if not isinstance(members, list) or not 1 <= len(members) <= 100:
raise bad("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(m, "member_id") for m in members]
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user_id not in ids:
raise bad("Создатель должен быть в member_ids")
chat_id = uuid.uuid4()
uuids = [uuid.UUID(i) for i in ids]
try:
async with self.db.conn() as c:
async with c.transaction(): # чат и участники создаются атомарно
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
chat_id, title)
await c.execute(
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
chat_id, created, uuids)
except asyncpg.ForeignKeyViolationError:
raise bad("Неизвестный участник") from None
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
async def _require_member(self, user_id, params):
chat_id = parse_uuid(params["chat_id"], "chat_id")
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
if not ok:
raise ApiError(404, "not_found", "Чат не найден")
return chat_id
async def chat(self, req, params):
user_id = await self.authenticate(req)
chat_id = parse_uuid(params["chat_id"], "chat_id")
row = await self.db.fetchrow(
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
chat_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Чат не найден")
return 200, chat_json(row)
async def messages(self, req, params):
user_id = await self.authenticate(req)
if req.method == "POST":
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
return 201, message_json(row)
chat_id = await self._require_member(user_id, params)
if req.method == "GET":
limit, after = page_params(req.query, "m", chat_id)
if after is None:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
else:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
async def _post_message(self, req, chat_id, user_id):
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
try:
key = req.headers.get("idempotency-key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
req.require_json_media_type()
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
raw_atts = data.get("attachment_ids", [])
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
raise bad("attachment_ids: массив до 10 элементов")
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
if len(set(atts)) != len(atts):
raise bad("attachment_ids содержит дубликаты")
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
except ApiError:
await self._require_member(user_id, {"chat_id": chat_id})
raise
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
ensure_ascii=False).encode("utf-8")).digest()
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
[uuid.UUID(a) for a in atts], retry=False)
if row["outcome"] == "not_member":
raise ApiError(404, "not_found", "Чат не найден")
if row["outcome"] == "attachment_not_found":
raise ApiError(404, "not_found", "Вложение не найдено")
if row["outcome"] == "attachment_failed":
raise ApiError(409, "conflict", "Вложение не удалось обработать")
if bytes(row["request_hash"]) != request_hash:
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
# --- ASGI ----------------------------------------------------------------------
async def __call__(self, scope, receive, send):
if scope["type"] == "lifespan":
return await self.lifespan(receive, send)
if scope["type"] != "http":
return
started = time.perf_counter()
request_id = new_request_id()
route_name, status, headers = "unknown", 500, {}
try:
req = Request(scope, receive, request_id)
handler, params, route_name = self.route(req.method, req.path)
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self.inflight += 1
try:
await asyncio.sleep(0)
status, body = await handler(req, params)
finally:
self.inflight -= 1
except ApiError as exc:
status, headers = exc.status, exc.headers
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
except DEPENDENCY_ERRORS as exc:
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
except Exception as exc:
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
status = 500
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
if isinstance(body, StreamResponse):
try:
await body.send(send, request_id, INSTANCE_ID)
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
else:
await respond(send, status, body, request_id, headers, INSTANCE_ID)
if os.environ.get("ACCESS_LOG", "1") != "0":
log("info", "request", request_id=request_id, method=scope.get("method"),
route=ROUTE_LABELS.get(route_name, "unmatched"), status=status,
duration_ms=round((time.perf_counter() - started) * 1000, 3))
async def janitor(self):
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
import random
await asyncio.sleep(random.uniform(5, 30))
while True:
try:
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
except Exception as exc:
log("warning", "janitor_failed", error_type=type(exc).__name__)
await asyncio.sleep(3600)
def route(self, method, path):
for name, pattern, methods in ROUTES:
m = pattern.fullmatch(path)
if m:
if method not in methods:
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
return getattr(self, name), m.groupdict(), name
raise ApiError(404, "not_found", "Ресурс не найден")
async def lifespan(self, receive, send):
while True:
event = await receive()
if event["type"] == "lifespan.startup":
await self.db.start()
await self.cache.start()
await self.s3.start()
self._relay_task = asyncio.create_task(self.relay.run())
self._janitor = asyncio.create_task(self.janitor())
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
await send({"type": "lifespan.startup.complete"})
elif event["type"] == "lifespan.shutdown":
self.draining = True
self._janitor.cancel()
await self.cache.stop()
self.relay.stop()
self._relay_task.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "shutdown")
await send({"type": "lifespan.shutdown.complete"})
return
async def overloaded_app(scope, receive, send):
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
if scope["type"] != "http":
return
request_id = new_request_id()
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
"request_id": request_id}}
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
def _install_overload_response():
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
try:
module = __import__(name, fromlist=["service_unavailable"])
except ImportError:
continue
module.service_unavailable = overloaded_app
def create_app():
_install_overload_response()
return Messenger()
app = create_app()
+64
View File
@@ -0,0 +1,64 @@
"""Применение версионируемых SQL-миграций: python -m app.migrate
Свойства:
* повторный запуск ничего не делает (таблица schema_migrations);
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
* каждая миграция — одна транзакция вместе с записью о применении;
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
"""
import asyncio
import hashlib
import os
import sys
import time
import asyncpg
from . import config
from .web import log
LOCK_ID = 7352001
async def connect(deadline):
while True:
try:
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
if time.monotonic() > deadline:
raise
log("info", "waiting_for_database", error_type=type(exc).__name__)
await asyncio.sleep(1)
async def main():
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
conn = await connect(time.monotonic() + wait)
try:
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
directory = os.path.abspath(config.MIGRATIONS_DIR)
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
count = 0
for name in files:
sql = open(os.path.join(directory, name), encoding="utf-8").read()
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
if name in applied:
if applied[name] != checksum:
log("error", "migration_modified", version=name)
sys.exit(2)
continue
async with conn.transaction():
await conn.execute(sql)
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
count += 1
log("info", "migration_applied", version=name)
log("info", "migrations_done", applied=count, total=len(files))
finally:
await conn.close()
if __name__ == "__main__":
asyncio.run(main())
+74
View File
@@ -0,0 +1,74 @@
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
"""
import asyncio
from datetime import timezone
from . import config
from .web import format_dt, log
JOB_TYPE = "attachment.process.v1"
def job_message(row):
return {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
class Relay:
def __init__(self, db, broker):
self.db, self.broker = db, broker
self._kick = asyncio.Event()
self._stop = False
self.last_error = None
def kick(self):
self._kick.set()
def stop(self):
self._stop = True
self._kick.set()
async def relay_once(self):
published = 0
async with self.db.conn() as c:
async with c.transaction():
rows = await c.fetch(
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at FROM outbox "
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
config.RELAY_BATCH)
for row in rows:
await self.broker.publish(job_message(row), headers=self.headers_for(row))
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
published += 1
return published
def headers_for(self, row):
return {}
async def run(self):
delay = config.RELAY_INTERVAL
while not self._stop:
try:
n = await self.relay_once()
self.last_error = None
delay = config.RELAY_INTERVAL
if n >= config.RELAY_BATCH:
continue # возможно, есть ещё
except asyncio.CancelledError:
raise
except Exception as exc:
if self.last_error != type(exc).__name__:
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
self.last_error = type(exc).__name__
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
try:
await asyncio.wait_for(self._kick.wait(), delay)
except asyncio.TimeoutError:
pass
self._kick.clear()
+31
View File
@@ -0,0 +1,31 @@
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from . import config
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
parallelism=config.ARGON2_PARALLELISM)
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
async def hash_password(password):
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
def _verify(stored, password):
try:
return _hasher.verify(stored, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored, password):
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
return ok and stored is not None
+97
View File
@@ -0,0 +1,97 @@
"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized.
Правила (contracts/README.md):
* thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый
фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются;
* crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85;
* optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена).
Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS.
"""
import hashlib
import io
import math
from PIL import Image, ImageOps
from . import config
Image.MAX_IMAGE_PIXELS = None
PIPELINE_VERSION = 1
JPEG_QUALITY = 85
WEBP_QUALITY = 80
class PermanentError(Exception):
"""Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code)."""
def __init__(self, code):
super().__init__(code)
self.code = code
def _flatten(img):
"""RGBA/LA/P с прозрачностью -> RGB на белом фоне."""
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return img.convert("RGB")
def _fit_size(width, height, box, upscale=False):
scale = min(box / width, box / height)
if not upscale:
scale = min(scale, 1.0)
return max(1, math.floor(width * scale)), max(1, math.floor(height * scale))
def _encode(img, fmt, **params):
buf = io.BytesIO()
img.save(buf, fmt, **params)
data = buf.getvalue()
return data, hashlib.sha256(data).hexdigest()
def process(fileobj):
"""Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError."""
try:
fileobj.seek(0)
src = Image.open(fileobj)
fmt = src.format
width, height = src.size
if fmt not in ("PNG", "JPEG"):
raise PermanentError("invalid_image")
if width * height > config.MAX_IMAGE_PIXELS:
raise PermanentError("image_too_large")
if fmt == "JPEG" and width > 2560 and height > 2560:
# экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280)
src.draft("RGB", (width // 2, height // 2))
src.load()
img = ImageOps.exif_transpose(src)
except PermanentError:
raise
except MemoryError:
raise
except Exception:
raise PermanentError("invalid_image") from None
base = _flatten(img)
out = []
# thumbnail
tw, th = _fit_size(base.width, base.height, 256)
thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base
data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th})
# crop 128x128 (центрированный квадрат)
side = min(base.width, base.height)
left, top = (base.width - side) // 2, (base.height - side) // 2
crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS)
data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128})
# optimized WebP (прозрачность сохраняется)
opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base
ow, oh = _fit_size(opt_src.width, opt_src.height, 1280)
opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src
data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4)
out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh})
return out
+91
View File
@@ -0,0 +1,91 @@
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
загрузкой, метаданные которой ещё не зафиксированы;
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
Выход: JSON-отчёт со счётчиками и временем обхода.
"""
import argparse
import asyncio
import json
import time
from datetime import datetime, timedelta, timezone
import asyncpg
from . import config
from .s3 import S3
async def reconcile(apply, grace, prefix):
started = time.perf_counter()
s3 = S3()
await s3.start()
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
t0 = time.perf_counter()
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
db_ms = (time.perf_counter() - t0) * 1000
by_key = {r["s3_key"]: r for r in rows}
t0 = time.perf_counter()
objects = {}
async for key, size, modified in s3.list_keys(prefix):
objects[key] = (size, modified)
list_ms = (time.perf_counter() - t0) * 1000
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace)
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
for key, (size, modified) in objects.items():
row = by_key.get(key)
if row is None:
if modified < cutoff:
report["orphan_objects"].append(key)
else:
report["orphans_skipped_fresh"] += 1
elif row["status"] == "ready" and row["size_bytes"] != size:
report["size_mismatch"].append(str(row["id"]))
for key, row in by_key.items():
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
report["missing_objects"].append(str(row["id"]))
if apply:
for key in report["orphan_objects"]:
# повторная проверка перед удалением: за время обхода строка могла появиться
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
await s3.delete(key)
report["fixed"]["deleted_orphans"] += 1
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
for att_id in ids:
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
"WHERE id = $1 AND status = 'ready'", att_id, code)
report["fixed"]["marked_failed"] += int(n.split()[-1])
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
"total": round((time.perf_counter() - started) * 1000, 1)}
return report
finally:
await conn.close()
await s3.stop()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
ap.add_argument("--prefix", default="att/")
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
args = ap.parse_args()
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
report[k + "_count"] = len(report[k])
report[k] = report[k][:args.max_listed]
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+70
View File
@@ -0,0 +1,70 @@
"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay <attachment_id> | replay-all]
list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging;
replay <attachment_id> — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id,
запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный
эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется.
Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов.
"""
import argparse
import asyncio
import json
import sys
import uuid
import asyncpg
from . import config
async def list_failed(conn):
rows = await conn.fetch(
"SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments "
"WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200")
return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"],
"staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows]
async def replay(conn, att_id):
async with conn.transaction():
row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id)
if row is None:
return {"attachment_id": str(att_id), "result": "not_found"}
if row["status"] != "failed":
return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]}
if not row["staging_key"]:
return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"}
job_id = uuid.uuid4()
await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, "
"lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id)
await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)",
job_id, att_id, row["pipeline_version"])
await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')",
att_id, job_id)
return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)}
async def main_async(args):
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
if args.command == "list":
return await list_failed(conn)
if args.command == "replay":
return await replay(conn, uuid.UUID(args.attachment_id))
return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)]
finally:
await conn.close()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="command", required=True)
sub.add_parser("list")
r = sub.add_parser("replay")
r.add_argument("attachment_id")
sub.add_parser("replay-all")
print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2))
if __name__ == "__main__":
sys.exit(main())
+89
View File
@@ -0,0 +1,89 @@
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
import asyncio
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from . import config
from .web import log, unavailable
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
class S3:
def __init__(self):
self._ctx = None
self.client = None
async def start(self):
if not config.S3_ENDPOINT:
log("error", "s3_endpoint_missing")
return
session = get_session()
self._ctx = session.create_client(
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
s3={"addressing_style": "path"}))
self.client = await self._ctx.__aenter__()
async def stop(self):
if self._ctx:
await self._ctx.__aexit__(None, None, None)
def _check(self):
if self.client is None:
raise unavailable("Хранилище файлов не настроено")
async def put(self, key, fileobj, content_type, size):
self._check()
try:
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
ContentLength=size)
except S3_ERRORS as exc:
log("warning", "s3_put_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
async def put_bytes(self, key, data, content_type):
self._check()
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type)
async def open_stream(self, key):
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
self._check()
try:
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except S3_ERRORS as exc:
log("warning", "s3_get_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
return resp["Body"], resp["ContentLength"]
async def delete(self, key):
self._check()
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
async def head(self, key):
"""Метаданные объекта или None, если его нет."""
self._check()
try:
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
return None
raise unavailable("Хранилище файлов недоступно") from exc
except S3_ERRORS as exc:
raise unavailable("Хранилище файлов недоступно") from exc
async def ping(self):
self._check()
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
async def list_keys(self, prefix):
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
self._check()
paginator = self.client.get_paginator("list_objects_v2")
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
for obj in page.get("Contents", []):
yield obj["Key"], obj["Size"], obj["LastModified"]
+85
View File
@@ -0,0 +1,85 @@
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
2) проверяется, что анонимный запрос к bucket отклоняется (private);
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
"""
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import ClientError
from . import config
def secret(name):
path = os.environ.get(name + "_FILE")
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
def client(session, access, secret_key):
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
aws_secret_access_key=secret_key,
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
s3={"addressing_style": "path"}))
async def main():
bucket = config.S3_BUCKET
session = get_session()
deadline = time.time() + 60
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
while True:
try:
await admin.list_buckets()
break
except Exception as exc:
if time.time() > deadline:
print("S3 недоступен:", type(exc).__name__)
return 1
await asyncio.sleep(1)
try:
await admin.head_bucket(Bucket=bucket)
print("bucket %s уже существует" % bucket)
except ClientError:
await admin.create_bucket(Bucket=bucket)
print("bucket %s создан" % bucket)
# приватность: запрос без подписи отклоняется
try:
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
print("ОШИБКА: bucket доступен анонимно")
return 2
except urllib.error.HTTPError as exc:
if exc.code not in (401, 403):
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
return 2
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
# ключ API: рабочие права на bucket, нет прав администратора
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
key = "bootstrap-check/%d" % int(time.time())
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
await api.delete_object(Bucket=bucket, Key=key)
if got != b"ok":
print("ОШИБКА: чтение/запись ключом API")
return 3
try:
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
return 4
except ClientError as exc:
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
print("bootstrap завершён")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+40
View File
@@ -0,0 +1,40 @@
"""Точка входа: uvicorn с управляемым завершением (drain).
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
Так плановая остановка/rolling update не даёт ошибок клиентам.
"""
import signal
import threading
import uvicorn
from . import config
from .main import app
from .web import log
class DrainingServer(uvicorn.Server):
_draining = False
def handle_exit(self, sig, frame):
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
self._draining = True
app.draining = True
log("info", "draining", seconds=config.DRAIN_SECONDS)
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
timer.daemon = True
timer.start()
return
super().handle_exit(sig, frame)
def main():
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
DrainingServer(cfg).run()
if __name__ == "__main__":
main()
+131
View File
@@ -0,0 +1,131 @@
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
Политики:
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
circuit breaker не даёт каждому запросу ждать таймаут;
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
"""
import hashlib
import secrets
import time
from . import config
from .web import log
try:
import redis.asyncio as aioredis
except ImportError: # кэш опционален
aioredis = None
def token_hash(token):
return hashlib.sha256(token.encode("ascii")).digest()
class SessionCache:
def __init__(self):
self.client = None
self.down_until = 0.0
self.stats = {"hit": 0, "miss": 0, "error": 0}
async def start(self):
if config.VALKEY_URL and aioredis:
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
socket_connect_timeout=config.VALKEY_TIMEOUT)
async def stop(self):
if self.client:
await self.client.aclose()
def _usable(self):
return self.client is not None and time.monotonic() >= self.down_until
def _fail(self, exc):
self.stats["error"] += 1
if time.monotonic() >= self.down_until:
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
async def get(self, key):
if not self._usable():
return None
try:
value = await self.client.get(b"s:" + key)
except Exception as exc:
self._fail(exc)
return None
self.stats["hit" if value is not None else "miss"] += 1
return value
async def put(self, key, value, ttl):
if not self._usable() or ttl <= 0:
return
try:
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
except Exception as exc:
self._fail(exc)
async def delete(self, key):
if not self._usable():
return
try:
await self.client.delete(b"s:" + key)
except Exception as exc:
self._fail(exc)
async def ping(self):
if self.client is None:
return None
try:
return bool(await self.client.ping())
except Exception:
return False
class Sessions:
def __init__(self, db, cache):
self.db, self.cache = db, cache
@staticmethod
def _cache_value(user_id, expires_at):
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
async def create(self, user_id):
token = secrets.token_urlsafe(32) # 256 бит энтропии
digest = token_hash(token)
row = await self.db.fetchrow(
"INSERT INTO sessions (token_hash, user_id, expires_at) "
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
digest, user_id, float(config.SESSION_TTL_SECONDS))
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return token, row["expires_at"]
async def resolve(self, token):
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
digest = token_hash(token)
cached = await self.cache.get(digest)
if cached is not None:
user_id, _, expires_ms = cached.decode().partition("|")
if int(expires_ms) > time.time() * 1000:
return user_id
await self.cache.delete(digest)
row = await self.db.fetchrow(
"SELECT user_id, expires_at FROM sessions "
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
if row is None:
return None
user_id = str(row["user_id"])
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return user_id
async def revoke(self, token):
digest = token_hash(token)
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
await self.cache.delete(digest)
+187
View File
@@ -0,0 +1,187 @@
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
import base64
import json
import re
import sys
import time
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qsl
from . import config
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
LIMIT_RE = re.compile(r"[0-9]{1,9}")
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message):
return ApiError(400, "invalid_request", message)
def unavailable(message="Зависимость временно недоступна"):
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
def log(level, msg, **fields):
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "msg": msg}
record.update(fields)
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
sys.stdout.flush()
# --- время -------------------------------------------------------------------
def format_dt(value):
value = value.astimezone(timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
def dt_to_us(value):
return (value - EPOCH) // timedelta(microseconds=1)
def us_to_dt(us):
return EPOCH + timedelta(microseconds=us)
# --- валидация ---------------------------------------------------------------
def parse_uuid(value, what="id"):
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
raise bad("Некорректный %s" % what)
return value.lower()
def valid_text(value, field, maximum, allow_empty=False):
if not isinstance(value, str):
raise bad("Поле %s должно быть строкой" % field)
try:
value.encode("utf-8")
except UnicodeEncodeError:
raise bad("Поле %s содержит некорректные символы" % field) from None
if "\x00" in value:
raise bad("Поле %s содержит NUL" % field)
if len(value) > maximum:
raise bad("Поле %s слишком длинное" % field)
if not allow_empty and not value.strip():
raise bad("Поле %s не может быть пустым" % field)
return value
def _reject_constant(name):
raise ValueError(name)
def parse_json_object(raw, required, allowed):
try:
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Тело запроса — некорректный JSON") from None
if not isinstance(data, dict):
raise bad("Ожидается JSON-объект")
if set(data) - allowed:
raise bad("Лишние поля в запросе")
if any(key not in data for key in required):
raise bad("Не хватает обязательных полей")
return data
def encode_cursor(kind, resource, key):
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
def decode_cursor(text, kind, resource):
try:
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
or not -62135596800000000 < data["t"] < 253402300799000000):
raise ValueError
return data["t"], data["i"]
except (ValueError, UnicodeError):
raise bad("Некорректный cursor") from None
def page_params(query, kind, resource):
limit, cursor = 50, None
if "limit" in query:
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
raise bad("limit должен быть целым числом 1–100")
limit = int(query["limit"][0])
if not 1 <= limit <= 100:
raise bad("limit должен быть целым числом 1–100")
if "cursor" in query:
if len(query["cursor"]) != 1 or not query["cursor"][0]:
raise bad("Некорректный cursor")
cursor = decode_cursor(query["cursor"][0], kind, resource)
return limit, cursor
def parse_query(raw):
result = {}
if raw:
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
result.setdefault(key, []).append(value)
return result
class Request:
def __init__(self, scope, receive, request_id):
self.method = scope["method"]
self.path = scope["path"]
self.query = parse_query(scope.get("query_string", b""))
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
self.receive = receive
self.request_id = request_id
self.user_id = None
async def body(self):
declared = self.headers.get("content-length")
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks, size = [], 0
while True:
event = await self.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks.append(chunk)
if not event.get("more_body"):
return b"".join(chunks)
def require_json_media_type(self):
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
if media != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
async def respond(send, status, body, request_id, extra=None, instance_id=None):
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
headers = [(b"x-request-id", request_id.encode())]
if status != 204:
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
else:
headers.append((b"content-length", b"0"))
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
await send({"type": "http.response.start", "status": status, "headers": headers})
await send({"type": "http.response.body", "body": payload})
def new_request_id():
return uuid.uuid4().hex
+323
View File
@@ -0,0 +1,323 @@
"""Worker вложений: python -m app.worker
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
"""
import asyncio
import json
import os
import random
import signal
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import aio_pika
from . import config, pipeline
from .broker import QUEUE, Broker
from .db import Database
from .outbox import Relay
from .s3 import S3
from .web import log
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
def backoff_seconds(attempt):
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
return random.uniform(ceiling * 0.5, ceiling)
class Worker:
def __init__(self):
self.db, self.s3, self.broker = Database(), S3(), Broker()
self.relay = Relay(self.db, self.broker)
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
self.busy = 0
self.stopping = False
self.tasks = []
# --- claim / finalize ---------------------------------------------------------
async def claim(self, job):
return await self.db.fetchrow(
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
async def fail(self, att_id, job, code, event="failed"):
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
async def reschedule(self, att_id, job, reason):
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
nxt = job["attempt"] + 1
if nxt > config.MAX_ATTEMPTS:
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
return
delay = backoff_seconds(job["attempt"])
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at) "
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5) FROM u RETURNING 1) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason)
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
async def publish_result(self, row, job, variants):
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
async with self.db.conn() as c:
async with c.transaction():
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
"FOR UPDATE", row["id"], WORKER_ID)
if owner is None:
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
for v in variants:
await c.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
return True
# --- обработка ----------------------------------------------------------------
async def process(self, job):
att_id = uuid.UUID(job["attachment_id"])
row = await self.claim(job)
if row is None:
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
started = time.perf_counter()
try:
await self.fault("after_claim")
stream, _ = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
finally:
stream.close()
loop = asyncio.get_running_loop()
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
finally:
spool.close()
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
await self.copy_original(row)
for v in variants:
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
await self.fault("after_s3_before_commit")
if not await self.publish_result(row, job, variants):
log("warning", "lease_lost", attachment_id=job["attachment_id"])
return
await self.fault("after_commit_before_ack")
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
duration_ms=round((time.perf_counter() - started) * 1000, 1))
try:
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
except Exception as exc:
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
except pipeline.PermanentError as exc:
await self.fail(att_id, job, exc.code)
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
except MemoryError:
await self.fail(att_id, job, "processing_failed")
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
except asyncio.CancelledError:
raise
except InjectedCrash:
raise
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
await self.reschedule(att_id, job, type(exc).__name__)
async def copy_original(self, row):
stream, size = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
stream.close()
spool.seek(0)
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
finally:
spool.close()
async def fault(self, point):
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
if "crash_once:" + point in config.FAULT_INJECTION:
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
if not os.path.exists(marker):
open(marker, "w").close()
log("error", "fault_crash_once", point=point)
os._exit(137)
if "crash:" + point in config.FAULT_INJECTION:
log("error", "fault_crash", point=point)
os._exit(137)
if "fail:" + point in config.FAULT_INJECTION:
raise OSError("injected failure at " + point)
async def on_message(self, message):
try:
job = json.loads(message.body)
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
raise ValueError("bad job")
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
except Exception:
log("error", "job_unreadable")
await message.reject(requeue=False) # -> DLQ
return
self.busy += 1
try:
await self.process(job)
await message.ack()
except InjectedCrash:
raise
except Exception as exc:
log("error", "job_unhandled", error_type=type(exc).__name__)
await message.nack(requeue=True)
finally:
self.busy -= 1
# --- sweeper -------------------------------------------------------------------
async def sweep_once(self):
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
rows = await self.db.fetch(
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) SELECT COALESCE(job_id, gen_random_uuid()), id, "
"pipeline_version, attempt + 1 FROM u WHERE status = 'queued' RETURNING 1), "
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
for r in rows:
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
if rows:
self.relay.kick()
async def sweeper(self):
while not self.stopping:
try:
await self.sweep_once()
except Exception as exc:
log("warning", "sweeper_failed", error_type=type(exc).__name__)
await asyncio.sleep(config.SWEEP_INTERVAL)
async def staging_janitor(self):
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
while not self.stopping:
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
try:
rows = await self.db.fetch(
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
float(config.STAGING_FAILED_RETENTION_SECONDS))
for r in rows:
await self.s3.delete(r["staging_key"])
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
except Exception as exc:
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
# --- жизненный цикл ------------------------------------------------------------
async def connect_broker(self):
while not self.stopping:
try:
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
return
except Exception as exc:
log("warning", "broker_unavailable", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def consume(self):
while not self.stopping:
await self.connect_broker()
try:
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
tag = await queue.consume(self.on_message)
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
while not self.stopping and not self.broker.channel.is_closed:
await asyncio.sleep(1)
if self.stopping:
await queue.cancel(tag)
return
except Exception as exc:
log("warning", "consume_failed", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def run(self):
await self.db.start()
await self.s3.start()
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
max_attempts=config.MAX_ATTEMPTS)
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, self.request_stop)
except NotImplementedError:
signal.signal(sig, lambda *_: self.request_stop())
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
beat = 0.0
while not self.stopping:
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
beat = time.monotonic()
try:
with open("/tmp/worker.alive", "w") as fh:
fh.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(0.5)
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
while self.busy and time.monotonic() < deadline:
await asyncio.sleep(0.2)
self.relay.stop()
for t in self.tasks:
t.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "worker_stopped", unfinished=self.busy)
def request_stop(self):
log("info", "worker_stopping")
self.stopping = True
class InjectedCrash(Exception):
pass
def main():
asyncio.run(Worker().run())
if __name__ == "__main__":
main()
Binary file not shown.

After

Width:  |  Height:  |  Size: 344 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 109 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 132 B

+208
View File
@@ -0,0 +1,208 @@
"""Собственные тесты lab2 против запущенного стенда (docker compose up -d).
python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080
Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности
зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами.
"""
import base64
import json
import os
import subprocess
import unittest
import urllib.error
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
ROOT = os.path.join(os.path.dirname(__file__), "..")
def call(method, path, body=None, token=None, headers=None, raw=None):
h = dict(headers or {})
if method == "POST" and path.endswith("/messages") and headers is None:
h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
if token:
h["Authorization"] = "Bearer " + token
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
status, hdrs, payload = resp.status, resp.headers, resp.read()
except urllib.error.HTTPError as err:
status, hdrs, payload = err.code, err.headers, err.read()
return status, hdrs, (json.loads(payload) if payload else None)
def basic(username, password):
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
def register(prefix="u"):
name = "%s_%s" % (prefix, uuid.uuid4().hex[:12])
password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо
status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password})
assert status == 201, user
status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password))
assert status == 201, session
return {"user": user, "password": password, "token": session["token"], "session": session}
def psql(sql):
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger",
"-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True)
return out.stdout.strip()
class AuthTest(unittest.TestCase):
def assertError(self, result, status, code):
self.assertEqual(result[0], status, result[2])
self.assertEqual(result[2]["error"]["code"], code)
self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"])
def test_register_validation(self):
base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12}
for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1},
{"username": "BAD"}, {"display_name": " "}):
body = dict(base, **patch)
self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request")
missing = {k: v for k, v in base.items() if k != "password"}
self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request")
self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points
def test_password_is_not_exposed_or_stored_plain(self):
actor = register()
self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"})
stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"])
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
self.assertNotIn(actor["password"], stored)
def test_wrong_password_and_unknown_user_are_indistinguishable(self):
actor = register()
wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123"))
unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123"))
for r in (wrong, unknown):
self.assertError(r, 401, "unauthorized")
self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"')
self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"])
for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()},
{"Authorization": "Bearer abc"}):
self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401)
def test_bearer_rules(self):
actor = register()
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"])
for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]},
{"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}):
r = call("GET", "/api/v1/users/me", headers=headers)
self.assertError(r, 401, "unauthorized")
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
other = register()
me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2]
self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется
def test_session_shape_entropy_and_logout(self):
actor = register()
s = actor["session"]
self.assertEqual(s["token_type"], "Bearer")
self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит
self.assertNotIn(actor["user"]["id"], s["token"])
self.assertNotEqual(register()["token"], s["token"])
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204)
self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized")
self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized")
def test_logout_revokes_in_database_and_cache(self):
actor = register()
call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш
call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])
revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"])
self.assertEqual(revoked, "1")
def test_token_only_stored_as_hash(self):
actor = register()
count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"])
self.assertEqual(count, "0")
def test_concurrent_registration_of_same_username(self):
name = "race_" + uuid.uuid4().hex[:10]
body = {"username": name, "display_name": "x", "password": "racing-password-1"}
with ThreadPoolExecutor(12) as pool:
codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12)))
self.assertEqual(codes, [201] + [409] * 11)
self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1")
class DataTest(unittest.TestCase):
def test_chat_creation_is_atomic(self):
a = register()
before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
ghost = str(uuid.uuid4())
r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"])
self.assertEqual(r[0], 400)
after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников")
ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"])
self.assertEqual(ok[0], 201)
def test_foreign_chat_empty_message_bad_json(self):
a, b, eve = register(), register(), register()
chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404)
self.assertEqual(call("GET", path, token=eve["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400)
self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400)
sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2]
self.assertEqual(sent["sender_id"], a["user"]["id"])
def test_multi_page_history_and_cursor_binding(self):
a, b = register(), register()
chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)]
walked, cursor, pages = [], None, 0
while True:
q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "")
page = call("GET", path + q, token=b["token"])[2]
walked += page["items"]
pages += 1
cursor = page["next_cursor"]
if not cursor:
break
self.assertEqual(walked, sent)
self.assertGreaterEqual(pages, 3)
first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"]
r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"])
self.assertEqual(r[0], 400)
def test_parallel_writes_from_many_clients(self):
users = [register() for _ in range(5)]
chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
def client(u):
return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)]
with ThreadPoolExecutor(5) as pool:
codes = [c for part in pool.map(client, users) for c in part]
self.assertEqual(codes, [201] * 100)
total, cursor = [], None
while True:
page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2]
total += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
self.assertEqual(len({m["id"] for m in total}), 100)
if __name__ == "__main__":
unittest.main()
+205
View File
@@ -0,0 +1,205 @@
"""Собственные тесты вложений lab5 (негативные сценарии и права). Запуск:
python -m unittest discover -s student-tests -p "test_attachments.py" -v (BASE_URL по умолчанию http://127.0.0.1:8080)
"""
import hashlib
import json
import os
import socket
import struct
import sys
import time
import unittest
import urllib.error
import urllib.parse
import urllib.request
import uuid
import zlib
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, ROOT, call, psql, register # noqa: E402
FIX = os.path.join(os.path.dirname(__file__), "fixtures")
PNG = open(os.path.join(FIX, "sample.png"), "rb").read()
JPG = open(os.path.join(FIX, "sample.jpg"), "rb").read()
def upload(token, chat_id, data, ctype="image/png", filename="sample.png", raw_filename=None):
qs = raw_filename if raw_filename is not None else "filename=" + filename
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?%s" % (BASE, chat_id, qs), data=data, method="POST",
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, json.loads(r.read())
except urllib.error.HTTPError as err:
body = err.read()
return err.code, err.headers, (json.loads(body) if body else None)
def download(token, att_id, suffix="/content"):
req = urllib.request.Request("%s/api/v1/attachments/%s%s" % (BASE, att_id, suffix),
headers={"Authorization": "Bearer " + token if token else ""})
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, r.read()
except urllib.error.HTTPError as err:
return err.code, err.headers, err.read()
def wait_ready(token, att_id, seconds=60):
"""lab5: загрузка возвращает 202 queued; готовность определяется опросом метаданных."""
deadline = time.time() + seconds
while time.time() < deadline:
code, _, body = download(token, att_id, "")
att = json.loads(body)
if att["status"] == "ready":
return att
assert att["status"] != "failed", att
time.sleep(0.3)
raise AssertionError("not ready")
def png_chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
def bomb_png(width=20000, height=20000):
"""Корректный по структуре PNG, заявляющий 400 млн пикселей, без данных изображения (отклоняется по заголовку)."""
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 16)) + png_chunk(b"IEND", b"")
class AttachmentTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register("att"), register("att"), register("att")
self.chat = call("POST", "/api/v1/chats", {"title": "файлы", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
self.other = call("POST", "/api/v1/chats", {"title": "другой", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
def assertError(self, result, status, code=None):
self.assertEqual(result[0], status, result[2])
if code:
self.assertEqual(result[2]["error"]["code"], code)
def test_png_and_jpeg_roundtrip_with_checksum(self):
for data, ctype, name in ((PNG, "image/png", "фото.png"), (JPG, "image/jpeg", "photo.jpg")):
status, headers, att = upload(self.a["token"], self.chat["id"], data, ctype, urllib.parse.quote(name))
self.assertEqual(status, 202, att)
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
wait_ready(self.b["token"], att["id"])
self.assertEqual((att["size_bytes"], att["sha256"], att["content_type"]), (len(data), hashlib.sha256(data).hexdigest(), ctype))
self.assertEqual((att["chat_id"], att["owner_id"], att["filename"]), (self.chat["id"], self.a["user"]["id"], name))
code, hdr, body = download(self.b["token"], att["id"]) # другой участник чата
self.assertEqual((code, body, hdr.get_content_type()), (200, data, ctype))
self.assertEqual(download(self.b["token"], att["id"], "")[0], 200)
def test_too_large_is_413_and_nothing_is_stored(self):
before = psql("SELECT count(*) FROM attachments")
r = upload(self.a["token"], self.chat["id"], PNG[:8] + b"\x00" * (10 * 1024 * 1024 + 1))
self.assertError(r, 413, "payload_too_large")
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
def test_corrupted_and_spoofed_files_are_415_not_500(self):
cases = {
"truncated png": (PNG[: len(PNG) // 2], "image/png"),
"garbage png": (b"\x89PNG\r\n\x1a\n" + b"\x00" * 200, "image/png"),
"random bytes": (os.urandom(500), "image/png"),
"truncated jpeg": (JPG[: len(JPG) // 2], "image/jpeg"),
"garbage jpeg": (b"\xff\xd8\xff\xe0" + b"\x00" * 200, "image/jpeg"),
"png as jpeg": (PNG, "image/jpeg"),
"jpeg as png": (JPG, "image/png"),
"text as png": (b"hello, i am not an image", "image/png"),
"gif": (b"GIF89a" + b"\x00" * 50, "image/png"),
}
for name, (data, ctype) in cases.items():
with self.subTest(name):
self.assertError(upload(self.a["token"], self.chat["id"], data, ctype), 415, "unsupported_media_type")
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "text/plain"), 415)
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "multipart/form-data; boundary=x"), 415)
def test_decompression_bomb_header_is_413(self):
self.assertError(upload(self.a["token"], self.chat["id"], bomb_png()), 413, "payload_too_large")
# 6000x6600 = 39.6 млн пикселей < лимита: по размерам допустимо, но данных изображения нет -> файл повреждён (415)
self.assertEqual(upload(self.a["token"], self.chat["id"], bomb_png(6000, 6600))[0], 415)
def test_dangerous_filenames_are_400(self):
bad = ["a%2Fb.png", "..%5Cx.png", "x%00.png", "x%0A.png", "x%7F.png", "", "%20%20", "n" * 256]
for name in bad:
with self.subTest(name):
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + name), 400, "invalid_request")
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename=""), 400)
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + "n" * 255)[0], 202)
def test_empty_body_is_400(self):
self.assertError(upload(self.a["token"], self.chat["id"], b""), 400)
def test_foreign_user_cannot_upload_read_or_download(self):
self.assertError(upload(self.eve["token"], self.chat["id"], PNG), 404)
att = upload(self.a["token"], self.chat["id"], PNG)[2]
wait_ready(self.a["token"], att["id"])
for suffix in ("", "/content"):
self.assertEqual(download(self.eve["token"], att["id"], suffix)[0], 404)
self.assertEqual(download(None, att["id"], suffix)[0], 401)
self.assertEqual(download(self.a["token"], str(uuid.uuid4()))[0], 404)
self.assertEqual(download(self.a["token"], "not-a-uuid")[0], 400)
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=thumbnail")[0], 200) # после ready вариант доступен
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=bogus")[0], 400)
def test_message_with_attachments_and_chat_binding(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
foreign_chat_att = upload(self.a["token"], self.other["id"], JPG, "image/jpeg")[2]
path = "/api/v1/chats/%s/messages" % self.chat["id"]
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
self.assertEqual(sent[0], 201, sent[2])
self.assertEqual(sent[2]["attachment_ids"], [att["id"]])
self.assertEqual(call("POST", path, {"text": "ещё раз то же вложение", "attachment_ids": [att["id"]]}, self.b["token"])[0], 201)
listed = call("GET", path + "?limit=10", token=self.b["token"])[2]["items"]
self.assertEqual([m["attachment_ids"] for m in listed], [[att["id"]], [att["id"]]])
# вложение другого чата / несуществующее -> 404; дубликаты, >10, пустое сообщение без вложений -> 400
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [foreign_chat_att["id"]]}, self.a["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, self.a["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"], att["id"]]}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": " ", "attachment_ids": []}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": "nope"}, self.a["token"])[0], 400)
# чужой пользователь не может сослаться на вложение
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"]]}, self.eve["token"])[0], 404)
def test_idempotent_replay_with_attachments(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
path = "/api/v1/chats/%s/messages" % self.chat["id"]
key = uuid.uuid4().hex
h = {"Idempotency-Key": key}
first = call("POST", path, {"text": "t", "attachment_ids": [att["id"]]}, self.a["token"], headers=h)
again = call("POST", path, {"attachment_ids": [att["id"]], "text": "t"}, self.a["token"], headers=h) # другой порядок ключей
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
self.assertEqual(call("POST", path, {"text": "t"}, self.a["token"], headers=h)[0], 409) # другое тело
text_key = uuid.uuid4().hex
m1 = call("POST", path, {"text": "без вложений"}, self.a["token"], headers={"Idempotency-Key": text_key})
m2 = call("POST", path, {"text": "без вложений", "attachment_ids": []}, self.a["token"], headers={"Idempotency-Key": text_key})
self.assertEqual(m1[2], m2[2]) # отсутствие == []
self.assertEqual(m1[2]["attachment_ids"], [])
def test_aborted_upload_leaves_nothing(self):
before = (psql("SELECT count(*) FROM attachments"))
host, port = BASE.split("//")[1].split(":")
with socket.create_connection((host, int(port))) as s:
s.sendall(("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
"Content-Type: image/png\r\nContent-Length: 200000\r\n\r\n" % (self.chat["id"], self.a["token"])).encode() + PNG[:5000])
import time
time.sleep(1)
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
def test_bucket_is_private_outside_the_api(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
wait_ready(self.a["token"], att["id"])
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
import subprocess
code = ("import urllib.request,urllib.error,sys\n"
"try:\n urllib.request.urlopen('http://seaweedfs:8333/messenger-attachments/%s', timeout=5); print('OPEN')\n"
"except urllib.error.HTTPError as e: print(e.code)\n" % key)
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
self.assertEqual(out, "403", "анонимный GET объекта напрямую из S3 должен быть запрещён")
if __name__ == "__main__":
unittest.main()
+121
View File
@@ -0,0 +1,121 @@
"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск:
python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB)
"""
import os
import sys
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
sys.path.insert(0, os.path.dirname(__file__))
from test_api import call, psql, register # noqa: E402
def chat_with(owner, *others):
ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others]
return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2]
def history(token, chat_id):
items, cursor = [], None
while True:
page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2]
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
return items
class IdempotencyTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register(), register(), register()
self.chat = chat_with(self.a, self.b)
self.path = "/api/v1/chats/%s/messages" % self.chat["id"]
def post(self, actor, body, key, path=None):
return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key})
def test_key_validation(self):
for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"},
{"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}):
h = dict(headers)
h.setdefault("Content-Type", "application/json")
r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h)
self.assertEqual(r[0], 400, headers)
self.assertEqual(r[2]["error"]["code"], "invalid_request")
self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201)
def test_replay_returns_same_message_and_creates_no_duplicate(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "один раз"}, key)
second = self.post(self.a, {"text": "один раз"}, key)
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[2], second[2])
self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1)
def test_different_body_same_key_is_409(self):
key = uuid.uuid4().hex
self.post(self.a, {"text": "первое"}, key)
r = self.post(self.a, {"text": "другое"}, key)
self.assertEqual(r[0], 409)
self.assertEqual(r[2]["error"]["code"], "conflict")
def test_scope_is_user_and_chat(self):
key = uuid.uuid4().hex
m1 = self.post(self.a, {"text": "общий ключ"}, key)[2]
m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь
other = chat_with(self.a)
m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат
self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3)
def test_replay_rechecks_session_and_membership(self):
key = uuid.uuid4().hex
self.post(self.a, {"text": "защищено"}, key)
self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний
r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key})
self.assertEqual(r[0], 401) # недействительный токен
logout = register()
chat = chat_with(logout)
path = "/api/v1/chats/%s/messages" % chat["id"]
self.post(logout, {"text": "x"}, key, path)
call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"])
self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401)
def test_semantically_equal_body_is_replayed(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "эквивалент"}, key)
raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }',
token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"})
self.assertEqual((raw[0], raw[2]), (201, first[2]))
def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self):
key = uuid.uuid4().hex
with ThreadPoolExecutor(24) as pool:
results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24)))
self.assertEqual({r[0] for r in results}, {201})
self.assertEqual(len({r[2]["id"] for r in results}), 1)
instances = {r[1]["X-Instance-Id"] for r in results}
self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances)
found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"]
self.assertEqual(len(found), 1)
def test_expired_key_is_reusable_after_ttl(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "ttl"}, key)[2]
psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key)
again = self.post(self.a, {"text": "ttl"}, key)
self.assertEqual(again[0], 201)
self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым")
def test_session_is_shared_between_replicas_without_relogin(self):
seen = set()
for _ in range(30):
r = call("GET", "/api/v1/users/me", token=self.a["token"])
self.assertEqual(r[0], 200)
seen.add(r[1]["X-Instance-Id"])
self.assertGreaterEqual(len(seen), 2, seen)
if __name__ == "__main__":
unittest.main()
+182
View File
@@ -0,0 +1,182 @@
"""Тесты асинхронного конвейера вложений lab5 (контракт вариантов, состояния, права). Запуск:
python -m unittest discover -s student-tests -p "test_pipeline.py" -v
Пиксельные проверки выполняются, если установлен Pillow (иначе пропускаются); размеры проверяются по заголовкам файлов.
"""
import hashlib
import io
import os
import struct
import sys
import time
import unittest
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import call, register # noqa: E402
from test_attachments import FIX, JPG, PNG, download, upload # noqa: E402
try:
from PIL import Image
except ImportError: # pragma: no cover
Image = None
def fixture(name):
return open(os.path.join(FIX, name), "rb").read()
def jpeg_size(data):
i = 2
while i < len(data):
while data[i] == 0xFF:
i += 1
marker = data[i]
i += 1
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
continue
length = int.from_bytes(data[i:i + 2], "big")
if marker in (0xC0, 0xC1, 0xC2):
return int.from_bytes(data[i + 5:i + 7], "big"), int.from_bytes(data[i + 3:i + 5], "big")
i += length
raise AssertionError("JPEG dimensions not found")
def webp_size(data):
assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not WebP"
kind = data[12:16]
if kind == b"VP8X":
return 1 + int.from_bytes(data[24:27], "little"), 1 + int.from_bytes(data[27:30], "little")
if kind == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
if kind == b"VP8 ":
return struct.unpack("<HH", data[26:30])[0] & 0x3FFF, struct.unpack("<HH", data[26:30])[1] & 0x3FFF
raise AssertionError("unknown WebP chunk")
def wait_status(token, att_id, wanted=("ready",), seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
code, _, body = download(token, att_id, "")
att = __import__("json").loads(body)
if att["status"] in wanted:
return att
time.sleep(0.3)
raise AssertionError("вложение не достигло %s за %d с: %s" % (wanted, seconds, att["status"]))
class PipelineTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register("pipe"), register("pipe"), register("pipe")
self.chat = call("POST", "/api/v1/chats", {"title": "конвейер", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
def ready(self, data, ctype="image/png", name="x.png"):
status, _, att = upload(self.a["token"], self.chat["id"], data, ctype, name)
self.assertEqual(status, 202, att)
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
return wait_status(self.b["token"], att["id"])
def variants(self, att):
return {v["name"]: v for v in att["variants"]}
def fetch(self, att, variant):
code, hdr, body = download(self.b["token"], att["id"], "/content?variant=" + variant)
self.assertEqual(code, 200)
return hdr.get_content_type(), body
def test_sample_png_produces_all_variants_per_contract(self):
att = self.ready(PNG)
v = self.variants(att)
self.assertEqual(set(v), {"thumbnail", "crop", "optimized"})
self.assertEqual((att["sha256"], att["size_bytes"]), (hashlib.sha256(PNG).hexdigest(), len(PNG))) # описывают оригинал
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]), (256, 160, "image/jpeg"))
self.assertEqual((v["crop"]["width"], v["crop"]["height"], v["crop"]["content_type"]), (128, 128, "image/jpeg"))
self.assertEqual(v["optimized"]["content_type"], "image/webp")
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (320, 200)) # без увеличения
ctype, thumb = self.fetch(att, "thumbnail")
self.assertEqual((ctype, jpeg_size(thumb)), ("image/jpeg", (256, 160)))
self.assertEqual((v["thumbnail"]["size_bytes"], v["thumbnail"]["sha256"]), (len(thumb), hashlib.sha256(thumb).hexdigest()))
_, crop = self.fetch(att, "crop")
self.assertEqual(jpeg_size(crop), (128, 128))
ctype, opt = self.fetch(att, "optimized")
self.assertEqual((ctype, webp_size(opt)), ("image/webp", (320, 200)))
code, hdr, original = download(self.b["token"], att["id"])
self.assertEqual((code, original, hdr.get_content_type()), (200, PNG, "image/png")) # оригинал неизменен
def test_exif_orientation_is_applied(self):
att = self.ready(fixture("exif_rot6.jpg"), "image/jpeg", "rot.jpg")
v = self.variants(att)
# 300x100 с Orientation=6 -> 100x300 -> вписывается в 256: 85x256 (округление вниз)
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (85, 256))
self.assertEqual(jpeg_size(self.fetch(att, "thumbnail")[1]), (85, 256))
def test_small_image_is_not_upscaled_but_crop_is_128(self):
att = self.ready(fixture("small.png"), "image/png", "small.png")
v = self.variants(att)
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (50, 30))
self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128))
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (50, 30))
@unittest.skipIf(Image is None, "нужен Pillow")
def test_alpha_is_composited_on_white(self):
att = self.ready(fixture("alpha.png"), "image/png", "alpha.png")
thumb = Image.open(io.BytesIO(self.fetch(att, "thumbnail")[1])).convert("RGB")
left, right = thumb.getpixel((10, thumb.height // 2)), thumb.getpixel((thumb.width - 10, thumb.height // 2))
self.assertTrue(all(c > 235 for c in left), "прозрачная область должна стать белой: %s" % (left,))
self.assertTrue(right[0] > 200 and right[1] < 60, "непрозрачная красная область сохранена: %s" % (right,))
def test_message_can_reference_attachment_before_ready_and_failed_cannot(self):
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
path = "/api/v1/chats/%s/messages" % self.chat["id"]
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
self.assertEqual((status, sent[0], sent[2]["attachment_ids"]), (202, 201, [att["id"]]))
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))
self.assertEqual(poison[0], 202, poison[2])
failed = wait_status(self.b["token"], poison[2]["id"], ("failed",))
self.assertEqual((failed["status"], failed["error_code"], failed["variants"]), ("failed", "invalid_image", []))
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [failed["id"]]}, self.a["token"])[0], 409)
self.assertEqual(download(self.b["token"], failed["id"], "/content")[0], 409)
self.assertEqual(download(self.b["token"], failed["id"], "/content?variant=thumbnail")[0], 409)
def test_content_is_409_while_not_ready_and_original_is_untouched(self):
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
code = download(self.b["token"], att["id"], "/content")[0]
self.assertIn(code, (200, 409)) # быстрый worker мог успеть: 200 допустим, но не 404/500
done = wait_status(self.b["token"], att["id"])
self.assertEqual(download(self.b["token"], done["id"], "/content")[2], PNG)
def test_other_variants_visible_only_to_chat_members(self):
att = self.ready(PNG)
for variant in ("thumbnail", "crop", "optimized", "original"):
self.assertEqual(download(self.eve["token"], att["id"], "/content?variant=" + variant)[0], 404)
self.assertEqual(download(self.b["token"], att["id"], "/content?variant=bogus")[0], 400)
def test_ready_job_redelivery_does_not_duplicate_variants(self):
from test_api import psql
att = self.ready(PNG)
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"])
self.assertEqual(before, "3")
psql("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES (gen_random_uuid(), '%s', 1, 2)" % att["id"])
time.sleep(4)
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3")
self.assertEqual(wait_status(self.b["token"], att["id"])["status"], "ready")
def test_staging_is_cleaned_after_ready_and_kept_for_failed(self):
import subprocess
from test_api import ROOT, psql
att = self.ready(PNG)
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))[2]
wait_status(self.b["token"], poison["id"], ("failed",))
def exists(att_id):
code = "\n".join(["import asyncio", "from app.s3 import S3", "async def m():",
" s = S3(); await s.start(); print((await s.head('staging/%s')) is not None); await s.stop()" % att_id,
"asyncio.run(m())"])
return subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
self.assertEqual(exists(att["id"]), "False", "staging готового вложения должен быть удалён после COMMIT результата")
self.assertEqual(exists(poison["id"]), "True", "staging неудавшегося вложения хранится для разбора/replay (retention)")
if __name__ == "__main__":
unittest.main()
+154
View File
@@ -0,0 +1,154 @@
"""Отказы конвейера lab5 (окна отказа из contracts/attachment-states.md). Тесты управляют контейнерами (volumes сохраняются):
python student-tests/test_resilience5.py # ~5-7 минут
Окна: (1) worker остановлен -> upload -> рестарт API -> worker поднят; (2) рестарт брокера с volume; (3) брокер недоступен
при приёме (outbox); (4) crash API между COMMIT и publish; (5) crash worker после записи в S3 до COMMIT результата;
(6) crash после COMMIT до ack; (7) исчерпание попыток и операторский replay.
"""
import json
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, call, psql, register # noqa: E402
from test_attachments import PNG, download, upload # noqa: E402
from test_pipeline import wait_status # noqa: E402
def compose(*args, env=None, check=True):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True,
env=dict(os.environ, **(env or {})), check=check).stdout
def wait(predicate, seconds=90, message="условие"):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if predicate():
return
except Exception:
pass
time.sleep(1)
raise AssertionError("%s не выполнено за %d с" % (message, seconds))
def healthy(service):
ids = compose("ps", "-q", service).split()
return bool(ids) and all(subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", i], capture_output=True, text=True).stdout.strip() == "healthy" for i in ids)
def api_ready():
return call("GET", "/health/ready")[0] == 200
def recreate_worker(env=None):
compose("up", "-d", "--no-deps", "--force-recreate", "worker", env=env)
wait(lambda: healthy("worker"), 90, "worker healthy")
class ResilienceTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
recreate_worker()
compose("up", "-d", "--no-deps", "rabbitmq", "api")
def setUp(self):
compose("start", "rabbitmq", "postgres", "minio", "valkey", check=False)
wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy")
wait(api_ready, 60, "api ready")
self.a = register("res")
self.chat = call("POST", "/api/v1/chats", {"title": "отказы", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
def accept(self, data=PNG):
status, _, att = upload(self.a["token"], self.chat["id"], data)
self.assertEqual(status, 202, att)
return att
def done(self, att, seconds=90):
ready = wait_status(self.a["token"], att["id"], ("ready",), seconds)
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3") # без дублей
code, _, body = download(self.a["token"], att["id"])
self.assertEqual((code, body), (200, PNG))
return ready
def test_1_worker_down_upload_api_restart_worker_up(self):
compose("stop", "worker")
att = self.accept()
sent = call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
self.assertEqual(sent[0], 201)
time.sleep(3)
self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued") # не обрабатывается без worker
compose("restart", "api")
wait(api_ready, 60, "api ready")
compose("start", "worker")
self.done(att)
def test_2_broker_restart_keeps_accepted_jobs(self):
compose("stop", "worker")
att = self.accept()
time.sleep(2)
compose("restart", "rabbitmq")
wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy")
compose("start", "worker")
self.done(att)
def test_3_broker_down_during_acceptance_outbox_delivers_later(self):
compose("stop", "rabbitmq")
att = self.accept() # 202 даже без брокера: намерение зафиксировано в outbox
self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "1")
time.sleep(3)
self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued")
compose("start", "rabbitmq")
self.done(att)
self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "0")
def test_4_api_crash_between_commit_and_publish(self):
compose("stop", "rabbitmq")
att = self.accept() # COMMIT выполнен, publish невозможен
for cid in compose("ps", "-q", "api").split():
subprocess.run(["docker", "kill", cid], capture_output=True) # внезапная смерть всех реплик API
compose("start", "rabbitmq")
wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy")
compose("up", "-d", "--no-deps", "api")
wait(api_ready, 90, "api ready")
self.done(att) # relay (в worker/новом API) довёл задание без ручного вмешательства
def test_5_worker_crash_after_s3_before_commit_lease_recovery(self):
recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_s3_before_commit", "LEASE_SECONDS": "8"})
att = self.accept()
ready = self.done(att, 120)
events = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"])
self.assertIn("lease_expired", events)
self.assertIn("ready", events)
self.assertEqual(ready["status"], "ready")
def test_6_worker_crash_after_commit_before_ack_redelivery_is_noop(self):
recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_commit_before_ack"})
att = self.accept()
self.done(att, 120)
time.sleep(5) # повторная доставка: no-op с ack, вариантов по-прежнему 3
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3")
self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready")
def test_7_retry_exhaustion_registry_and_operator_replay(self):
recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"})
att = self.accept()
failed = wait_status(self.a["token"], att["id"], ("failed",), 60)
self.assertEqual((failed["error_code"], failed["variants"]), ("retry_exhausted", []))
events = psql("SELECT string_agg(event || ':' || coalesce(attempt::text,''), ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"])
self.assertEqual(events.count("retry_scheduled"), 2, events)
registry = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "list"))
self.assertIn(att["id"], [r["attachment_id"] for r in registry])
recreate_worker() # причина устранена
result = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"]))
self.assertEqual(result["result"], "requeued")
self.done(att)
again = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"]))
self.assertEqual(again["result"], "skipped") # replay не исполняет дубль уже успешной работы
if __name__ == "__main__":
unittest.main()
+58
View File
@@ -0,0 +1,58 @@
"""Проверка топологии стенда (уровень 4): число реплик, отсутствие публикации портов API, лимиты ресурсов, healthcheck,
отсутствие container_name, единая публичная точка входа. Запуск: python student-tests/test_topology.py"""
import json
import subprocess
import unittest
PROJECT = "mtusi-lab5"
def sh(*args):
return subprocess.run(args, capture_output=True, text=True, encoding="utf-8").stdout.strip()
def containers(service):
ids = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service).split()
return [json.loads(sh("docker", "inspect", i))[0] for i in ids]
class TopologyTest(unittest.TestCase):
def test_at_least_two_api_replicas_without_published_ports(self):
api = containers("api")
self.assertGreaterEqual(len(api), 2)
for c in api:
self.assertFalse([p for p in c["HostConfig"]["PortBindings"] or {}], "реплика API публикует порт в обход LB")
self.assertEqual(c["State"]["Health"]["Status"], "healthy")
def test_only_lb_is_published_and_only_on_localhost(self):
published = {}
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
for c in containers(service):
for port, binds in (c["HostConfig"]["PortBindings"] or {}).items():
published.setdefault(service, []).extend((port, b["HostIp"], b["HostPort"]) for b in binds)
self.assertEqual(list(published), ["lb"], published)
self.assertEqual(published["lb"], [("8080/tcp", "127.0.0.1", "8080")])
def test_resource_limits_are_set(self):
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
for c in containers(service):
self.assertGreater(c["HostConfig"]["Memory"], 0, service)
self.assertGreater(c["HostConfig"]["NanoCpus"], 0, service)
def test_no_fixed_container_names(self):
for c in containers("api"):
labels = c["Config"]["Labels"]
self.assertEqual(c["Name"].lstrip("/"), "%s-api-%s" % (PROJECT, labels["com.docker.compose.container-number"]))
def test_api_runs_as_non_root(self):
for c in containers("api"):
self.assertEqual(c["Config"]["User"], "10001:10001")
def test_stop_grace_period_covers_drain(self):
for c in containers("api"):
self.assertGreaterEqual(c["Config"].get("StopTimeout", 10), 15)
if __name__ == "__main__":
unittest.main()