ЛР 5 - БВТ2304 - Санжаровский - на 4 #55
+29
-10
@@ -1,20 +1,39 @@
|
||||
# В образ попадает только то, что нужно рантайму. Всё лишнее исключаем:
|
||||
# уменьшает контекст сборки, ускоряет build и не даёт утечь .env/секретам.
|
||||
.git
|
||||
.github
|
||||
.gitignore
|
||||
.dockerignore
|
||||
Dockerfile
|
||||
Dockerfile.example
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
tests/
|
||||
student-tests/
|
||||
scripts/
|
||||
contracts/
|
||||
evidence/
|
||||
*.md
|
||||
.github/
|
||||
lab.json
|
||||
requirements-dev.txt
|
||||
compose.yaml
|
||||
compose.example.yaml
|
||||
examples/
|
||||
Makefile
|
||||
MAINTAINERS.md
|
||||
infra/
|
||||
secrets/
|
||||
certs/
|
||||
volumes/
|
||||
data/
|
||||
backups/
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
+51
-10
@@ -1,25 +1,66 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Скопируйте в .env для ЛОКАЛЬНОГО учебного стенда; .env никогда не коммитится.
|
||||
# Пароли и ключи ниже — синтетические примеры, задайте свои.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
|
||||
# PostgreSQL — источник истины (пользователи, чаты, сообщения, сессии, ключи идемпотентности,
|
||||
# метаданные вложений). Compose собирает из них DATABASE_URL. Вне compose: DATABASE_URL или DATABASE_URL_FILE.
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-local-only
|
||||
|
||||
# Сессии в PostgreSQL, Redis — общий кэш сессий для всех реплик.
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS=3600
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
# CACHE_URL (или CACHE_URL_FILE) задаёт compose: redis://redis:6379/0.
|
||||
|
||||
# lab3: число реплик API за балансировщиком и срок хранения Idempotency-Key.
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# lab4: MinIO. ROOT — только для сервисов s3 и s3-init (bootstrap), в API не передаётся.
|
||||
MINIO_ROOT_USER=minio-root
|
||||
MINIO_ROOT_PASSWORD=change-me-root-local-only
|
||||
# Отдельный пользователь API с правами только на свой bucket (создаёт s3-init).
|
||||
# Вне compose можно передать файлами: S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE.
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=change-me-api-local-only
|
||||
S3_CONNECT_TIMEOUT=2
|
||||
S3_READ_TIMEOUT=10
|
||||
|
||||
# Лимиты загрузки (контракт): 10 MiB и 40 млн пикселей.
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
UPLOAD_BODY_TIMEOUT_SECONDS=60
|
||||
UPLOAD_SPOOL_MEMORY_BYTES=262144
|
||||
UPLOAD_MAX_CONCURRENT=16
|
||||
IMAGE_CHECK_CONCURRENCY=2
|
||||
# Очистка сирот: журнал загрузки старше grace period считается брошенным.
|
||||
UPLOAD_GRACE_SECONDS=900
|
||||
JANITOR_INTERVAL_SECONDS=60
|
||||
|
||||
# Пул соединений и таймауты
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
REDIS_TIMEOUT=0.5
|
||||
SHUTDOWN_TIMEOUT=10
|
||||
|
||||
LOG_LEVEL=info
|
||||
|
||||
# lab5: брокер и worker. BROKER_URL задаёт compose (redis://broker:6379/0 — отдельный Redis с AOF).
|
||||
WORKER_REPLICAS=1
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=20
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
JOB_BACKOFF_MAX_SECONDS=60
|
||||
CLAIM_IDLE_SECONDS=90
|
||||
IMAGE_MEMORY_MB=768
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# Скрипты, которые выполняются ВНУТРИ Linux-контейнеров, обязаны иметь LF,
|
||||
# даже если репозиторий клонирован на Windows с core.autocrlf=true.
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.yaml text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# ---- Stage 1: builder: зависимости ставим отдельно, в рантайм несём только результат
|
||||
FROM python:3.12.7-slim-bookworm AS builder
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --prefix=/install -r requirements.txt
|
||||
|
||||
# ---- Stage 2: runtime
|
||||
FROM python:3.12.7-slim-bookworm AS runtime
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 APP_PORT=8080 SHUTDOWN_TIMEOUT=10
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
WORKDIR /app
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
USER appuser
|
||||
EXPOSE 8080
|
||||
# /health/ready проверяет PostgreSQL: без БД контейнер станет unhealthy.
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD python -c "import os,urllib.request,sys; \
|
||||
u='http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready'; \
|
||||
sys.exit(0 if urllib.request.urlopen(u,timeout=2).status==200 else 1)" || exit 1
|
||||
# exec-форма: uvicorn — PID 1 и сам получает SIGTERM от docker stop.
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -1,37 +1,184 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Санжаровский, БВТ2304
|
||||
- Целевая оценка: **4** (трек 5 не заявлен)
|
||||
- Commit/PR: ветка `solution/bvt2304-sanzharovskiy` форка `Anatoly01/mtusi2026lab5`, PR в `mtusi2026/mtusi2026lab5`
|
||||
- Источники и библиотеки: всё из lab4 (FastAPI, uvicorn, psycopg 3 + psycopg-pool, argon2-cffi, redis-py 8.1, boto3 1.38, Pillow 11.2, HAProxy 3.0, MinIO от Pigsty). Брокер — **Redis Streams** (тот же образ `redis:7.4.1-alpine`, отдельный экземпляр с AOF). Документация: Redis Streams (consumer groups, PEL, `XAUTOCLAIM`), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (для сравнения границ confirm/ack), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), Python `resource` (RLIMIT_AS/RLIMIT_CPU), `multiprocessing` forkserver; Transactional outbox (microservices.io). Использование ИИ: ИИ-ассистент помогал с кодом, тестами и текстом отчёта; решения разобраны и проверены прогоном.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужно: Docker Desktop (Compose v2), Python 3.9+ для тестов (только stdlib). Образы закреплены: `python:3.12.7-slim-bookworm`, `postgres:16.4-alpine`, `redis:7.4.1-alpine` (кэш сессий **и** брокер), `haproxy:3.0.5-alpine`, `pgsty/minio:RELEASE.2026-08-04T00-00-00Z`, `pgsty/mc:RELEASE.2026-09-16T00-00-00Z`. Новых образов по сравнению с lab4 нет.
|
||||
|
||||
```sh
|
||||
cp .env.example .env # Windows: copy .env.example .env — задать свои пароли/ключи
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# порядок: postgres(healthy) → migrate (exit 0, миграции 0001–0004)
|
||||
# s3(healthy) → s3-init (bootstrap, exit 0); broker(healthy)
|
||||
# → api ×2 → worker → lb
|
||||
curl -i http://localhost:8080/health/ready # 200 + X-Instance-Id
|
||||
make check && make test # публичные проверки (Windows: set PYTHONUTF8=1)
|
||||
python student-tests/collect_evidence.py # ВСЕ проверки → evidence/*.txt (+ сводка)
|
||||
```
|
||||
|
||||
Демонстрация уровня 3 вручную: `scripts/stop-worker.(sh|cmd)` → upload (202 queued, content 409) → `docker compose restart api broker` → `scripts/start-worker.(sh|cmd)` → ready. Всё это автоматически делает `python student-tests/jobs.py --scenario worker-down`.
|
||||
|
||||
Отдельные проверки:
|
||||
|
||||
```sh
|
||||
python student-tests/jobs.py --base-url http://localhost:8080 # все 8 сценариев отказов lab5
|
||||
python student-tests/jobs.py --scenario crash-s3 # один сценарий
|
||||
docker compose --env-file .env exec -T api python -m app.tools limits-selftest # лимиты worker
|
||||
docker compose --env-file .env exec -T api python -m app.tools queue # stream, pending, DLQ, outbox
|
||||
docker compose --env-file .env exec -T api python -m app.tools job <attachment_id>
|
||||
docker compose --env-file .env exec -T api python -m app.replay --list # реестр failed
|
||||
docker compose --env-file .env exec -T api python -m app.replay --attachment <id> # явный replay
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh && scripts/restore-api.sh
|
||||
```
|
||||
|
||||
Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных (БД, bucket **и** очередь): `docker compose --env-file .env down -v`. Имя проекта `mtusi-lab5`: volumes lab4 не затрагиваются.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ─▶ lb (HAProxy, 127.0.0.1:8080) ─▶ api ×2 ──① PUT staging/<id> ──────────────▶ s3 (MinIO, private)
|
||||
│ ② BEGIN; attachments(queued) + ▲ staging/<id>
|
||||
│ attachment_jobs + job_outbox; COMMIT │ originals/<id>
|
||||
│ ③ XADD → broker (ответ = «confirm») │ variants/<id>/v1/*
|
||||
│ ④ 202 queued │
|
||||
▼ │
|
||||
postgres ◀──────── ⑦ COMMIT ready ──── worker ───────┘ ⑥ copy + PUT
|
||||
(истина: вложения, ⑤ XREADGROUP ◀── broker (Redis Streams, AOF always,
|
||||
jobs, outbox, volume brokerdata)
|
||||
variants, events) ⑧ XACK ─▶ broker ; ⑨ DELETE staging/<id>
|
||||
```
|
||||
|
||||
| Роль | Service | Версия | Health/readiness | Volume | Порт |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| load_balancer | `lb` | `haproxy:3.0.5-alpine` | `/healthz`; backend `/health/ready` | — | `127.0.0.1:8080` (+8081 ttl, +8082 fault — тестовые профили) |
|
||||
| api ×2 | `api` | `Dockerfile` | `/health/ready` (PostgreSQL) | нет | `expose 8080` |
|
||||
| **worker** | `worker` | тот же образ, `python -m app.worker` | heartbeat-файл обновляется раз в 1 c | нет | нет |
|
||||
| **broker** | `broker` | `redis:7.4.1-alpine` + `--appendonly yes --appendfsync always --maxmemory-policy noeviction` | `redis-cli ping` | `brokerdata` | нет |
|
||||
| postgres, redis, s3, s3-init, migrate | как в lab4 | | | `pgdata`, `s3data` | нет |
|
||||
| тестовые | `api-fault`, `worker-fault` (профиль `fault`), `api-ttl` (профиль `ttl`) | тот же образ | | нет | только через lb |
|
||||
|
||||
**Что перенесено из lab4** без изменений: LB + 2 реплики API без состояния, сессии (PostgreSQL + Redis-кэш), идемпотентность сообщений, проверка изображения по содержимому в API (413/415 синхронно, как в lab4), приватный bucket и отдельный пользователь MinIO, журнал `upload_intents` + janitor, лимиты, хуки. **Переход** по контракту: upload `201 ready` → `202 queued`.
|
||||
|
||||
**Что добавлено в lab5:**
|
||||
|
||||
- **Брокер — Redis Streams на отдельном экземпляре Redis с persistence.** Обычный Pub/Sub не хранит сообщения; Stream хранит, а consumer group даёт at-least-once: выданное worker сообщение лежит в PEL (pending entries list), пока не будет `XACK`. `appendfsync always` — ответ на `XADD` отправляется только после fsync журнала AOF, это наш аналог publisher confirm. `noeviction` — сообщения никогда не вытесняются. Кэш сессий — другой экземпляр Redis без persistence (lab2).
|
||||
- **Сообщение** (`app/broker.py`, строго по `contracts/jobs.schema.json`): `schema_version, job_id, type=attachment.process.v1, attachment_id, pipeline_version=1, attempt, created_at`. Ни байтов, ни URL, ни ключей bucket, ни credentials. Worker берёт ключи объектов и sha256 только из БД. Сообщение, не прошедшее схему или с неизвестным job, уходит в DLQ-stream `attachments:jobs:dlq`.
|
||||
- **Модель** (`migrations/0004_jobs.sql`, только добавления — lab4 с новой схемой работает): `attachments.staging_key`; `attachment_jobs` (job_id, attempt, max_attempts, `lease_token/lease_owner/lease_until`, error_code, replays, `UNIQUE(attachment_id, pipeline_version)`); `job_outbox` (payload, `available_at`, `published_at`); `attachment_variants` с **ключом эффекта** `PRIMARY KEY (attachment_id, pipeline_version, variant)`; `job_events` — журнал (реестр для оператора и audit replay).
|
||||
- **Варианты** (`app/imaging.py`): **thumbnail** — JPEG q85, вписан в 256×256, без увеличения, округление вниз, минимум 1 px (320×200 → 256×160); **crop** — JPEG q85, центрированный квадрат 128×128 (маленькие увеличиваются); **optimized** — WebP q80, `method=4`, вписан в 1280×1280 без увеличения, альфа сохраняется. EXIF orientation применяется до расчёта размеров; EXIF/XMP/ICC в результат не пишутся (проверено: у всех трёх вариантов `exif=False, icc=False`). Прозрачность в JPEG компонуется на белый. JPEG декодируется сразу в уменьшенном масштабе (`draft`), не меньше нужного самому большому варианту.
|
||||
- **Ключи S3 детерминированы**: `originals/<id>` (серверный `CopyObject` из `staging/<id>`), `variants/<id>/v1/<variant>.<ext>`. Повтор job перезаписывает те же объекты теми же байтами; в БД — `ON CONFLICT DO NOTHING`. Второго набора вариантов не бывает.
|
||||
- **Лимиты worker** (`app/worker.py` → дочерний процесс `forkserver`): байты — не больше `MAX_UPLOAD_BYTES` при скачивании из staging; пиксели — по заголовку до распаковки (`MAX_IMAGE_PIXELS`); память — `RLIMIT_AS = IMAGE_MEMORY_MB` (768 МиБ) у ребёнка, нехватка памяти даёт MemoryError в ребёнке, а не OOM всего worker; время — `JOB_TIMEOUT_SECONDS` (60 c) wall-clock, по истечении ребёнок убивается, плюс `RLIMIT_CPU`. Конкуренция — `WORKER_CONCURRENCY`=2 потока, каждый берёт по одному сообщению (`COUNT 1`, аналог prefetch=1). Контейнер worker: 1 CPU / 1 ГиБ.
|
||||
|
||||
### Состояния и транзакционные границы
|
||||
|
||||
| Переход | Где | Что устойчиво сохранено |
|
||||
| --- | --- | --- |
|
||||
| нет → queued | API: журнал → PUT `staging/<id>` → **одна транзакция**: снять журнал, `attachments(queued)`, `attachment_jobs(queued, attempt 1)`, `job_outbox` → COMMIT → `XADD` → **202** | полный проверенный оригинал в staging, метаданные, job и сообщение в outbox |
|
||||
| queued → processing | worker, `SELECT … FOR UPDATE`: новый `lease_token`, `lease_until = now()+20 c` | попытка, аренда, владелец |
|
||||
| processing → queued | временная ошибка, попытки есть: `attempt+1`, строка outbox с `available_at = now()+backoff` | причина (`last_error`), время следующей попытки |
|
||||
| processing → ready | worker: S3 (copy + 3 PUT) → **одна транзакция**: варианты, attachment и job → ready (только если `lease_token` всё ещё наш) → `XACK` → удалить staging | оригинал, три варианта, их sha256/размеры |
|
||||
| processing → failed | постоянная ошибка или `attempt = max_attempts` | `error_code` (`invalid_image`, `processing_failed`, `retry_exhausted`), событие, копия в DLQ-stream |
|
||||
| processing → queued (lease) | reaper: аренда истекла ещё на одну аренду назад | новая попытка через outbox |
|
||||
| failed → queued | `python -m app.replay` (только failed, только если staging ещё есть) | событие `replay` (кто, когда), тот же job_id, `attempt+1`, бюджет ещё 5 попыток |
|
||||
|
||||
Ready никогда не откатывается: повторная доставка ready-задачи — no-op с ack.
|
||||
|
||||
### Окна отказа
|
||||
|
||||
| Окно | Что происходит | Кто восстанавливает | Проверка |
|
||||
| --- | --- | --- | --- |
|
||||
| после staging, до записи job | объект в staging + строка `upload_intents` | API компенсирует сразу; иначе janitor после grace (как lab4) | `orphans.py` |
|
||||
| после COMMIT, до publish | job и outbox в БД, сообщения в брокере нет | **outbox relay** (поток maintenance worker, раз в 1 c, строки старше 2 c, `FOR UPDATE SKIP LOCKED`) | `jobs.py outbox` |
|
||||
| брокер недоступен при upload | `XADD` не прошёл за 2 c — 202 всё равно отдаётся | relay, когда брокер вернётся | `jobs.py broker-down` |
|
||||
| после publish, до confirm | брокер мог принять сообщение, а API не узнал | relay опубликует ещё раз → дубль → no-op | `jobs.py redelivery` |
|
||||
| после записи S3, до COMMIT результата | job в processing с арендой умершего worker, сообщение в PEL | перезапущенный worker перечитывает свой PEL; после истечения аренды — **takeover той же попытки**; объекты S3 те же | `jobs.py crash-s3` |
|
||||
| после COMMIT результата, до ack | job ready, сообщение в PEL | redelivery → `already_ready` → ack | `jobs.py crash-ack` |
|
||||
| worker завис (аренда не продлевается) | processing, lease истекает | reaper → attempt+1 → другой worker; проснувшийся не может зафиксировать (fencing по `lease_token`) | `jobs.py lease` |
|
||||
|
||||
### Retry, DLQ, replay
|
||||
|
||||
- **Постоянные** ошибки (без повторов): изображение не декодируется / превышает пиксели / не хватило памяти → `invalid_image`; sha256 staging не совпал с БД или исходника нет → `processing_failed`.
|
||||
- **Временные**: S3 или БД недоступны, таймаут обработки, падение ребёнка. Повтор через outbox с **экспоненциальным backoff и jitter** («equal jitter»: `d = min(60, base·2^(n−1))`, задержка ∈ [d/2, d]). `JOB_MAX_ATTEMPTS`=5, затем `failed retry_exhausted`. Бесконечного немого retry нет.
|
||||
- **Реестр failed** — таблица `attachment_jobs` (status=failed, error_code, last_error) + `job_events`; копия для оператора — DLQ-stream. `python -m app.replay --list` / `--dlq`.
|
||||
- **Replay** (`app/replay.py`) безопасен: меняет только failed (`FOR UPDATE`), ready/queued/processing отклоняет — успешную задачу второй раз не исполнить. job_id тот же, attempt растёт, событие с actor. Если исходник уже удалён retention — отказ.
|
||||
|
||||
### Аренда: долгая или зависшая задача
|
||||
|
||||
Живой worker продлевает аренду каждые `lease/3` ≈ 6,7 c (отдельный поток heartbeat), пока ребёнок обрабатывает картинку. Поэтому **долгая** задача (в тесте 30 c при аренде 20 c) не перехватывается: аренда всё время свежая. Жёсткий предел — `JOB_TIMEOUT_SECONDS`. У **мёртвого или замёрзшего** worker аренда истекает. Тогда:
|
||||
|
||||
1. сообщение из его PEL заберёт он сам после рестарта или другой worker (`XAUTOCLAIM`, 90 c); обработку получит тот, кто первым увидит истёкшую аренду;
|
||||
2. если за ещё одну аренду никто не перехватил, reaper переводит job в queued с новой попыткой.
|
||||
|
||||
Два неконтролируемых дубля невозможны: захват — `SELECT … FOR UPDATE`, фиксация результата сверяет `lease_token`. Старый worker после пробуждения получает `LeaseLost` и ничего не пишет в БД, а его объекты S3 совпадают байт в байт.
|
||||
|
||||
### Очистка staging
|
||||
|
||||
- ready: worker удаляет `staging/<id>` **после** COMMIT ready и ack; если не успел, подчищает maintenance через 30 c;
|
||||
- failed: исходник хранится `STAGING_RETENTION_SECONDS` (24 ч) для replay, потом удаляется;
|
||||
- брошенные загрузки (нет job): janitor по `upload_intents`, обход bucket проверяет и `originals/`, и `staging/`.
|
||||
|
||||
Текущую работу cleanup не трогает: удаляются только staging у ready и у failed старше retention.
|
||||
|
||||
**Соответствие `.env.example`:** `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` используются как есть; добавлены `WORKER_REPLICAS`, `JOB_LEASE_SECONDS`, `JOB_BACKOFF_BASE/MAX_SECONDS`, `CLAIM_IDLE_SECONDS`, `IMAGE_MEMORY_MB`. `BROKER_URL` задаёт compose (`redis://broker:6379/0`), поддерживается и `BROKER_URL_FILE`. `SESSION_STORE` — сессии всегда в PostgreSQL.
|
||||
|
||||
**Readiness:** `/health/ready` проверяет только PostgreSQL. Брокер не критичен для приёма — намерение лежит в outbox (проверено: брокер выключен → `/health/ready` 200, upload 202). S3 критичен только для вложений (как в lab4).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Основной прогон — **Docker на ПК** (Windows, Docker Desktop, стек из `compose.yaml` как есть): `python student-tests/collect_evidence.py`, все шаги PASS, выводы в `evidence/<шаг>.txt`. До этого код отлажен на локальном стенде без Docker (PostgreSQL 16, два процесса API за HAProxy, Redis-брокер с AOF, worker, S3-эмулятор moto) — `evidence/local/*.txt`, там те же результаты.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | `make test` | `PASS: lab5 public API scenario`: 202 → ready, thumbnail 256×160 JPEG, sha256 | `evidence/smoke.txt` |
|
||||
| Брокер с persistence | `redis-cli CONFIG GET` | `appendonly yes`, `appendfsync always`, `maxmemory-policy noeviction` | `evidence/broker-config.txt` |
|
||||
| **3:** 202 только после устойчивого приёма; брокер и worker отдельно; API не делает resize | `jobs.py worker-down` | worker остановлен: 202 queued за 0,06 c; через 3 c всё ещё queued; content и thumbnail → 409; сообщение с queued-вложением → 201; задание лежит в stream (len=1) | `evidence/jobs-worker-down.txt` |
|
||||
| **3:** рестарт API и брокера с volumes → worker → ready | `jobs.py worker-down` | после `restart broker` + `restart api` сообщение на месте (len=1), тот же токен; worker запущен → ready за 2,5 c, оригинал побайтно, thumbnail 256×160, staging удалён | `evidence/jobs-worker-down.txt` |
|
||||
| **3:** at-least-once, ack после эффекта, повтор не создаёт второй набор | `jobs.py redelivery`, `crash-ack` | 2 дубля ready-задачи → no-op: 3 строки вариантов, 3 объекта S3, `ready` записан один раз, pending 0; crash после COMMIT → redelivery → no-op | `evidence/jobs-redelivery.txt`, `jobs-crash-ack.txt` |
|
||||
| **3:** ограниченный retry, финальный failed | `jobs.py retry-replay` | 5 попыток → `failed retry_exhausted` за 22 c; content → 409; прикрепить к сообщению → 409 | `evidence/jobs-retry-replay.txt` |
|
||||
| **4:** crop 128×128 и WebP | smoke, `jobs.py` | у каждого ready-вложения варианты `thumbnail`, `crop`, `optimized` | `evidence/jobs-*.txt` |
|
||||
| **4:** лимиты байт/пикселей/времени/памяти worker | `app.tools limits-selftest` | 5/5: 36 Мп при лимите 10 Мп → `invalid_image: pixel limit`; RLIMIT_AS 120 МиБ → `memory limit`; битый PNG → `invalid_image`; зависание 10 c убито через 2 c | `evidence/worker-limits.txt` |
|
||||
| **4:** outbox: crash между COMMIT и publish | `jobs.py outbox` (api-fault) | клиент получил 502; вложение и job закоммичены; relay опубликовал строку outbox → ready без нового upload | `evidence/jobs-outbox.txt` |
|
||||
| **4:** брокер недоступен при приёме | `jobs.py broker-down` | `/health/ready` 200; upload 202 за 2,1 c (таймаут publish 2 c); строка outbox не опубликована → брокер вернулся → relay → ready | `evidence/jobs-broker-down.txt` |
|
||||
| **4:** backoff с jitter, постоянные/временные, DLQ/реестр, безопасный replay | `jobs.py retry-replay`, `app.replay` | задержки 0,7 / 1,1 / 3,6 / 6,3 c (база 1 c: d = 1, 2, 4, 8, каждая в [d/2, d]); задача в `replay --list`, копия в DLQ с причиной `retry_exhausted (injected_transient)`; replay → ready, тот же job_id, attempt 6, replays 1; повторный replay ready-задачи → `refused` | `evidence/jobs-retry-replay.txt`, `jobs-registry.txt` |
|
||||
| **4:** worker остановлен после записи объекта, до ack | `jobs.py crash-s3` | `os._exit(1)` после записи S3; контейнер перезапустился, `lease_takeover` той же попытки (attempt 1) → ready за 26,9 c; по-прежнему ровно 3 варианта | `evidence/jobs-crash-s3.txt` |
|
||||
| **4:** lease/timeout зависшего processing | `jobs.py lease` | долгая задача 32 c при аренде 20 c — без перехвата (heartbeat); замёрзший worker → reaper → attempt 2 → ready за 37,3 c; проснувшийся старый worker не смог зафиксировать (fencing), результат не изменился | `evidence/jobs-lease.txt` |
|
||||
| **4:** ограниченная конкуренция, очередь под нагрузкой, очистка staging | `upload-memory` + `queue-drain`; `worker-down` | 90 загрузок по 9,6 МиБ → 90 × 202; worker (2 потока, 1 CPU) разобрал очередь за 103 c после окончания загрузок; staging удаляется после ready | `evidence/upload-memory.txt`, `queue-drain.txt` |
|
||||
| Возможности lab2–4 | `run.py`, `idempotency.py`, `attachments.py`, `orphans.py`, `s3_outage.py`, `upload_memory.py`, persistence, failover ×2 | 30/30, 16/16, 91/91, orphans PASS, S3 outage 12/12, память процессов API +25–29 МиБ при 10–20 параллельных загрузках по 9,6 МиБ; persistence PASS; failover PASS дважды (0 неудачных попыток, восстановление 1,5 и 2,0 c) | `evidence/*.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Что ожидаем | Что наблюдали (Docker) | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| worker остановлен, upload | 202 queued, обработки нет | 202 за 0,06 c, 3 c queued | работа ждёт в stream; 0 потерь |
|
||||
| рестарт брокера (AOF) с необработанным заданием | сообщение сохранено | len=1 до и после | ready через 2,5 c после старта worker |
|
||||
| брокер остановлен при upload | 202 без брокера | 202 за 2,1 c, outbox не опубликован | relay опубликовал после старта брокера, сценарий 14 c |
|
||||
| api убит между COMMIT и publish | задача не теряется | 502 клиенту, job в БД | ready без нового upload (сценарий 20 c) |
|
||||
| worker убит после S3, до COMMIT | та же попытка доходит до ready | `lease_takeover`, attempt 1 | 26,9 c (рестарт + истечение аренды 20 c) |
|
||||
| worker убит после COMMIT, до ack | дубль = no-op | redelivery → no-op, pending 0 | 0 c для пользователя |
|
||||
| worker «завис» | после истечения аренды задачу берёт другой | reaper → attempt 2 → ready | 37,3 c (аренда 20 c + ещё одна) |
|
||||
| 5 временных ошибок подряд | failed `retry_exhausted` | 22 c, задержки растут | replay → ready |
|
||||
| 90 больших загрузок | приём не ждёт обработку | 90 × 202, p50 4,4–7,2 c (сама загрузка 9,6 МиБ) | очередь разобрана за 103 c |
|
||||
|
||||
Время восстановления после смерти worker определяется арендой: `JOB_LEASE_SECONDS`=20 c. Короче — быстрее восстановление, но выше риск перехватить живую задачу при паузе GC/диска. Heartbeat каждые 6,7 c даёт запас в 3 продления.
|
||||
|
||||
**Порядок шагов в `collect_evidence.py`.** Первый полный прогон уронил 4 унаследованных теста lab4: они скачивали файл сразу после ответа (теперь это 202, а не 201 ready), а persistence/failover стояли в очереди за 90 большими картинками от `upload-memory`. Тесты исправлены: ждут ready. Добавлен шаг `queue-drain`: ждёт пустой очереди перед persistence. В `upload_memory.py` критерий памяти теперь — память процессов (`anon`), без файлового кэша ядра. `usage` по-прежнему печатается: +68 МиБ, тоже меньше 100 МиБ.
|
||||
|
||||
## Ответы на вопросы защиты
|
||||
|
||||
- **Publisher confirm vs consumer ack.** Confirm — граница «брокер принял и сохранил»: у нас ответ `XADD` после fsync AOF. Ack (`XACK`) — граница «потребитель устойчиво выполнил работу»: у нас после COMMIT ready в PostgreSQL. Между ними сообщение лежит в PEL.
|
||||
- **Что сохранено в момент 202.** Полный проверенный оригинал в приватном `staging/<id>` (S3), строка `attachments` (queued), `attachment_jobs` и `job_outbox` — в одной транзакции. Сообщение в брокере — почти всегда, но не обязательно: если нет, его опубликует relay.
|
||||
- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: сообщение повторяется, если ack не дошёл. Ключ эффекта — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` + детерминированные ключи S3; логическая работа — `job_id` (UNIQUE по `attachment_id, pipeline_version`); устаревшие попытки отсекает `attempt` в сообщении.
|
||||
- **Зависший или долгий.** Долгий продлевает аренду heartbeat-ом; у зависшего она истекает. Захват — `FOR UPDATE`, фиксация — только с актуальным `lease_token` (fencing), поэтому два неконтролируемых исполнителя невозможны.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Задержка обработки под нагрузкой ограничена одним worker (1 CPU, 2 потока): ~0,9 больших картинки в секунду. Масштабирование — `WORKER_REPLICAS`; в этой работе не измерялось.
|
||||
- Брокер в одном экземпляре: отказ узла с брокером останавливает доставку (но не приём — outbox), реплики Redis/Sentinel не настроены.
|
||||
- `appendfsync always` дороже `everysec`; для учебной нагрузки это не узкое место, при большой нагрузке нужна оценка.
|
||||
- Relay и reaper работают в worker: если все worker выключены, отложенные retry ждут их запуска (новые задачи при этом публикует API).
|
||||
- Exactly-once нет и не обещается: at-least-once + идемпотентный эффект.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""
|
||||
Точка входа контейнера: `python -m app`. Настройки — из окружения (app/config.py).
|
||||
uvicorn ловит SIGTERM: перестаёт принимать запросы и дожидается текущих
|
||||
(до SHUTDOWN_TIMEOUT), затем закрывает пул БД и соединение с Redis.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .config import settings
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=settings.log_level.upper(),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
access_log=False, # свой лог в middleware: без query, токенов и тел
|
||||
log_level=settings.log_level,
|
||||
)
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
Брокер заданий (lab5): Redis Streams на ОТДЕЛЬНОМ экземпляре Redis с AOF.
|
||||
|
||||
Почему это durable-брокер, а не Pub/Sub:
|
||||
- XADD записывает сообщение в stream; Redis запущен с `appendonly yes` и
|
||||
`appendfsync always`: ответ на XADD отправляется только после fsync журнала AOF.
|
||||
Ответ XADD (ID сообщения) — наш аналог publisher confirm: брокер принял и сохранил.
|
||||
- Consumer group: XREADGROUP выдаёт сообщение одному worker и кладёт его в PEL
|
||||
(pending entries list). Пока worker не сделал XACK, сообщение остаётся в PEL
|
||||
и переживает рестарт брокера и смерть worker. XACK — consumer ack: worker
|
||||
устойчиво зафиксировал результат. Это разные границы.
|
||||
- Брошенные сообщения (worker умер до XACK) забирает другой worker через XAUTOCLAIM.
|
||||
- `maxmemory-policy noeviction`: Redis никогда не вытеснит сообщения из памяти.
|
||||
|
||||
В сообщении только ID и служебные поля (contracts/jobs.schema.json):
|
||||
ни байтов, ни URL, ни ключей bucket, ни credentials.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import redis
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
from .config import settings
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
SCHEMA_VERSION = 1
|
||||
_REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at"}
|
||||
_OPTIONAL = {"traceparent", "tracestate"}
|
||||
|
||||
|
||||
class BrokerUnavailable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BadMessage(ValueError):
|
||||
"""Сообщение не соответствует jobs.schema.json — в DLQ, без обработки."""
|
||||
|
||||
|
||||
def iso_z(ts: datetime) -> str:
|
||||
return ts.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def make_payload(job_id, attachment_id, pipeline_version: int, attempt: int,
|
||||
created_at: datetime) -> dict:
|
||||
return {"schema_version": SCHEMA_VERSION, "job_id": str(job_id), "type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id), "pipeline_version": int(pipeline_version),
|
||||
"attempt": int(attempt), "created_at": iso_z(created_at)}
|
||||
|
||||
|
||||
def parse(raw) -> dict:
|
||||
"""Строгая проверка по jobs.schema.json (additionalProperties: false)."""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BadMessage("not json") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise BadMessage("not an object")
|
||||
keys = set(data)
|
||||
if not _REQUIRED <= keys or keys - _REQUIRED - _OPTIONAL:
|
||||
raise BadMessage("unexpected fields")
|
||||
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
|
||||
raise BadMessage("unknown schema/type")
|
||||
for k in ("job_id", "attachment_id"):
|
||||
try:
|
||||
uuid.UUID(str(data[k]))
|
||||
except ValueError as exc:
|
||||
raise BadMessage(f"bad {k}") from exc
|
||||
for k in ("pipeline_version", "attempt"):
|
||||
if not isinstance(data[k], int) or isinstance(data[k], bool) or data[k] < 1:
|
||||
raise BadMessage(f"bad {k}")
|
||||
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
|
||||
raise BadMessage("bad created_at")
|
||||
return data
|
||||
|
||||
|
||||
def dumps(payload: dict) -> str:
|
||||
return json.dumps(payload, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- sync (worker, CLI)
|
||||
def sync_client(read_timeout: float | None = None) -> redis.Redis:
|
||||
"""read_timeout у worker больше BLOCK в XREADGROUP (2 с), иначе блокирующее
|
||||
чтение пустой очереди выглядело бы как обрыв связи."""
|
||||
return redis.Redis.from_url(settings.broker_url,
|
||||
socket_timeout=read_timeout or settings.broker_timeout,
|
||||
socket_connect_timeout=settings.broker_timeout,
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
def ensure_group(r: redis.Redis) -> None:
|
||||
"""Группа создаётся с ID 0: сообщения, опубликованные ДО первого запуска worker,
|
||||
тоже будут прочитаны (с '$' они бы потерялись)."""
|
||||
try:
|
||||
r.xgroup_create(settings.jobs_stream, settings.jobs_group, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
|
||||
|
||||
def publish_sync(r: redis.Redis, payload: dict) -> str:
|
||||
"""Возвращает ID сообщения — подтверждение приёма брокером (после fsync AOF)."""
|
||||
try:
|
||||
return r.xadd(settings.jobs_stream, {"job": dumps(payload)})
|
||||
except redis.RedisError as exc:
|
||||
raise BrokerUnavailable(type(exc).__name__) from exc
|
||||
|
||||
|
||||
def to_dlq(r: redis.Redis, raw: str, reason: str, source_id: str = "") -> None:
|
||||
r.xadd(settings.dlq_stream, {"job": raw if isinstance(raw, str) else "", "reason": reason[:200],
|
||||
"source_id": source_id, "at": iso_z(datetime.now(timezone.utc))})
|
||||
|
||||
|
||||
# ----------------------------------------------------------- async (API)
|
||||
_aclient: aioredis.Redis | None = None
|
||||
|
||||
|
||||
def init_async() -> None:
|
||||
global _aclient
|
||||
_aclient = aioredis.Redis.from_url(settings.broker_url, socket_timeout=settings.broker_timeout,
|
||||
socket_connect_timeout=settings.broker_timeout,
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
async def close_async() -> None:
|
||||
if _aclient is not None:
|
||||
await _aclient.aclose()
|
||||
|
||||
|
||||
async def publish(payload: dict) -> str:
|
||||
if _aclient is None:
|
||||
init_async()
|
||||
try:
|
||||
return await _aclient.xadd(settings.jobs_stream, {"job": dumps(payload)})
|
||||
except (redis.RedisError, OSError) as exc:
|
||||
raise BrokerUnavailable(type(exc).__name__) from exc
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
Конфигурация только из переменных окружения (ничего не зашито в код и образ).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
return int(os.getenv(name, str(default)))
|
||||
|
||||
|
||||
def _float(name: str, default: float) -> float:
|
||||
return float(os.getenv(name, str(default)))
|
||||
|
||||
|
||||
def _secret(name: str, default: str = "") -> str:
|
||||
"""Значение из NAME_FILE (Docker secret, файл) или из переменной NAME.
|
||||
Файл приоритетнее: так credentials не видны в `docker inspect`."""
|
||||
path = os.getenv(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read().strip()
|
||||
return os.getenv(name, default)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
# HTTP
|
||||
host: str = os.getenv("APP_HOST", "0.0.0.0")
|
||||
port: int = _int("APP_PORT", 8080)
|
||||
shutdown_timeout: int = _int("SHUTDOWN_TIMEOUT", 10)
|
||||
log_level: str = os.getenv("LOG_LEVEL", "info")
|
||||
# Имя реплики для X-Instance-Id: INSTANCE_ID или hostname контейнера
|
||||
# (в Docker это короткий ID контейнера — по нему stop-one.sh находит реплику).
|
||||
instance_id: str = os.getenv("INSTANCE_ID") or socket.gethostname()
|
||||
|
||||
# PostgreSQL — источник истины
|
||||
database_url: str = _secret(
|
||||
"DATABASE_URL", "postgresql://messenger:devpass@127.0.0.1:5432/messenger")
|
||||
db_pool_min: int = _int("DB_POOL_MIN", 1)
|
||||
db_pool_max: int = _int("DB_POOL_MAX", 10) # ограниченный пул
|
||||
db_pool_timeout: float = _float("DB_POOL_TIMEOUT", 3.0) # ждать свободное соединение
|
||||
db_connect_timeout: int = _int("DB_CONNECT_TIMEOUT", 3)
|
||||
db_statement_timeout_ms: int = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
migrate_on_start: bool = os.getenv("MIGRATE_ON_START", "true").lower() == "true"
|
||||
|
||||
# Одновременных вычислений Argon2id на реплику (каждое ≈ 64 МиБ RAM).
|
||||
password_hash_concurrency: int = _int("PASSWORD_HASH_CONCURRENCY", 2)
|
||||
|
||||
# Сессии
|
||||
session_ttl_seconds: int = _int("SESSION_TTL_SECONDS", 3600)
|
||||
|
||||
# Идемпотентность POST сообщений: сколько хранить ключ (контракт: 24 часа)
|
||||
idempotency_ttl_seconds: int = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
|
||||
# Redis/Valkey — кэш сессий (не источник истины). Пусто = кэш выключен.
|
||||
# Имя из skeleton — CACHE_URL(_FILE); REDIS_URL оставлено как синоним.
|
||||
redis_url: str = _secret("CACHE_URL") or _secret("REDIS_URL")
|
||||
redis_timeout: float = _float("REDIS_TIMEOUT", 0.5)
|
||||
session_cache_ttl_seconds: int = _int("SESSION_CACHE_TTL_SECONDS", 60)
|
||||
|
||||
# lab4: S3-совместимое хранилище (MinIO) — только байты вложений.
|
||||
# API ходит отдельным пользователем с правами только на свой bucket
|
||||
# (не root MinIO): S3_ACCESS_KEY_ID(_FILE) / S3_SECRET_ACCESS_KEY(_FILE).
|
||||
s3_endpoint: str = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
s3_region: str = os.getenv("S3_REGION", "us-east-1")
|
||||
s3_bucket: str = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
s3_access_key_id: str = _secret("S3_ACCESS_KEY_ID")
|
||||
s3_secret_access_key: str = _secret("S3_SECRET_ACCESS_KEY")
|
||||
s3_connect_timeout: float = _float("S3_CONNECT_TIMEOUT", 2.0)
|
||||
s3_read_timeout: float = _float("S3_READ_TIMEOUT", 10.0)
|
||||
s3_max_attempts: int = _int("S3_MAX_ATTEMPTS", 2)
|
||||
# Сколько секунд после ошибки соединения с S3 сразу отвечать 503 (circuit breaker).
|
||||
s3_breaker_seconds: float = _float("S3_BREAKER_SECONDS", 5.0)
|
||||
|
||||
# Лимиты загрузки (контракт: 10 MiB и 40 млн пикселей).
|
||||
max_upload_bytes: int = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
|
||||
max_image_pixels: int = _int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
# Сколько ждать тело запроса целиком (защита от «медленной» загрузки).
|
||||
upload_body_timeout_seconds: float = _float("UPLOAD_BODY_TIMEOUT_SECONDS", 60.0)
|
||||
# Сколько байт тела держим в RAM; остальное — во временном файле,
|
||||
# который удаляется сразу после ответа (не хранилище, только буфер).
|
||||
upload_spool_memory_bytes: int = _int("UPLOAD_SPOOL_MEMORY_BYTES", 256 * 1024)
|
||||
# Одновременных загрузок на реплику и одновременных проверок изображения.
|
||||
upload_max_concurrent: int = _int("UPLOAD_MAX_CONCURRENT", 16)
|
||||
image_check_concurrency: int = _int("IMAGE_CHECK_CONCURRENCY", 2)
|
||||
|
||||
# Очистка «сирот» (объект в S3 без метаданных в БД).
|
||||
# Журнал upload_intents старше grace period считается брошенным.
|
||||
upload_grace_seconds: int = _int("UPLOAD_GRACE_SECONDS", 900)
|
||||
janitor_interval_seconds: int = _int("JANITOR_INTERVAL_SECONDS", 60)
|
||||
janitor_enabled: bool = os.getenv("JANITOR_ENABLED", "true").lower() == "true"
|
||||
|
||||
# lab5: брокер — отдельный Redis со Streams и AOF (appendfsync always) на volume.
|
||||
# Не путать с CACHE_URL: кэш сессий без persistence, брокер — с persistence.
|
||||
broker_url: str = _secret("BROKER_URL", "redis://127.0.0.1:6380/0")
|
||||
broker_timeout: float = _float("BROKER_TIMEOUT", 2.0)
|
||||
jobs_stream: str = os.getenv("JOBS_STREAM", "attachments:jobs")
|
||||
jobs_group: str = os.getenv("JOBS_GROUP", "workers")
|
||||
dlq_stream: str = os.getenv("DLQ_STREAM", "attachments:jobs:dlq")
|
||||
pipeline_version: int = _int("PIPELINE_VERSION", 1)
|
||||
|
||||
# lab5: worker
|
||||
worker_concurrency: int = _int("WORKER_CONCURRENCY", 2) # одновременных задач на процесс
|
||||
job_max_attempts: int = _int("JOB_MAX_ATTEMPTS", 5) # управляемых попыток на job
|
||||
job_timeout_seconds: float = _float("JOB_TIMEOUT_SECONDS", 60) # жёсткий лимит обработки
|
||||
job_lease_seconds: float = _float("JOB_LEASE_SECONDS", 20) # аренда, продлевается heartbeat
|
||||
job_backoff_base_seconds: float = _float("JOB_BACKOFF_BASE_SECONDS", 2)
|
||||
job_backoff_max_seconds: float = _float("JOB_BACKOFF_MAX_SECONDS", 60)
|
||||
# Через сколько чужое неподтверждённое сообщение считается брошенным (XAUTOCLAIM).
|
||||
# Больше JOB_TIMEOUT: живой worker к этому времени точно закончил или убил обработку.
|
||||
claim_idle_seconds: float = _float("CLAIM_IDLE_SECONDS", 90)
|
||||
# Память дочернего процесса обработки (RLIMIT_AS), МиБ.
|
||||
image_memory_mb: int = _int("IMAGE_MEMORY_MB", 768)
|
||||
# Outbox relay: сколько ждать, прежде чем считать строку не опубликованной API.
|
||||
outbox_relay_delay_seconds: float = _float("OUTBOX_RELAY_DELAY_SECONDS", 2)
|
||||
maintenance_interval_seconds: float = _float("MAINTENANCE_INTERVAL_SECONDS", 1)
|
||||
# Исходник в staging у failed хранится столько (для replay), потом удаляется.
|
||||
staging_retention_seconds: int = _int("STAGING_RETENTION_SECONDS", 86400)
|
||||
heartbeat_file: str = os.getenv("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
|
||||
|
||||
# Внедрение отказов ТОЛЬКО для тестового профиля fault (api-fault).
|
||||
# В обычных репликах выключено: заголовок X-Fault-Inject игнорируется.
|
||||
fault_injection: bool = os.getenv("FAULT_INJECTION", "false").lower() == "true"
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Пул соединений с PostgreSQL.
|
||||
|
||||
- Пул ограничен (DB_POOL_MAX): при нагрузке запросы ждут свободное соединение
|
||||
не дольше DB_POOL_TIMEOUT, а не открывают соединения без предела.
|
||||
- connect_timeout и statement_timeout: зависший запрос или недоступная БД
|
||||
не держат обработчик бесконечно — клиент получает 503.
|
||||
- Все запросы параметризованы (%s), SQL-инъекция невозможна.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
from .config import settings
|
||||
|
||||
pool: AsyncConnectionPool | None = None
|
||||
|
||||
|
||||
def make_pool() -> AsyncConnectionPool:
|
||||
return AsyncConnectionPool(
|
||||
conninfo=settings.database_url,
|
||||
min_size=settings.db_pool_min,
|
||||
max_size=settings.db_pool_max,
|
||||
timeout=settings.db_pool_timeout,
|
||||
kwargs={
|
||||
"connect_timeout": settings.db_connect_timeout,
|
||||
"options": f"-c statement_timeout={settings.db_statement_timeout_ms}",
|
||||
"application_name": "messenger-api",
|
||||
},
|
||||
# Перед выдачей соединение проверяется (SELECT 1 на уровне протокола):
|
||||
# после рестарта PostgreSQL «мёртвые» соединения отбрасываются, а не дают 503.
|
||||
check=AsyncConnectionPool.check_connection,
|
||||
open=False,
|
||||
)
|
||||
|
||||
|
||||
async def open_pool() -> None:
|
||||
global pool
|
||||
pool = make_pool()
|
||||
# wait=False: API стартует, даже если БД ещё поднимается; /health/ready это покажет.
|
||||
await pool.open(wait=False)
|
||||
|
||||
|
||||
async def close_pool() -> None:
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
|
||||
|
||||
def get_pool() -> AsyncConnectionPool:
|
||||
assert pool is not None, "pool is not opened"
|
||||
return pool
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
Проверка загруженного изображения ПО СОДЕРЖИМОМУ (lab4).
|
||||
|
||||
Порядок важен: сначала дешёвые проверки, потом дорогие.
|
||||
1. Сигнатура (magic bytes) → реальный тип. Не PNG/JPEG или не совпадает
|
||||
с заявленным Content-Type → 415 (подмена MIME).
|
||||
2. Размеры из заголовка (PNG IHDR / JPEG SOF) → width × height ≤ MAX_IMAGE_PIXELS,
|
||||
иначе 413. Это делается ДО распаковки пикселей: «декомпрессионная бомба»
|
||||
(маленький файл, огромная картинка) отсекается, не заняв память.
|
||||
3. Целостность → повреждённый файл даёт 415:
|
||||
- PNG: потоково читаем чанки, сверяем CRC каждого, распаковываем IDAT
|
||||
кусками по 64 КиБ и только СЧИТАЕМ байты (пиксели не храним).
|
||||
Объём распакованных данных должен ровно совпасть с тем, что следует
|
||||
из IHDR; лишнее сверх ожидаемого не распаковываем вообще.
|
||||
- JPEG: Pillow в режиме draft (декодирование в масштабе 1/8) —
|
||||
весь поток проверяется декодером, а памяти нужно в ~64 раза меньше.
|
||||
Памяти на проверку: O(64 КиБ) для PNG и ~(пиксели/64)×3 байт для JPEG,
|
||||
независимо от размера файла.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import warnings
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from PIL import Image, ImageFile
|
||||
|
||||
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIG = b"\xff\xd8\xff"
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
# Усечённый файл — ошибка, а не «догрузим серым».
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
# Предупреждение Pillow о «бомбе» превращаем в исключение (→ 413), а не в строку лога.
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
class ImageTooLarge(Exception):
|
||||
"""Слишком много пикселей → 413."""
|
||||
|
||||
|
||||
class BadImage(Exception):
|
||||
"""Не PNG/JPEG, тип не совпадает с Content-Type или файл повреждён → 415."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def sniff(head: bytes) -> str | None:
|
||||
"""Реальный тип по первым байтам. Расширение и заголовок клиента не учитываются."""
|
||||
if head.startswith(PNG_SIG):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SIG):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def inspect(f: BinaryIO, declared_type: str, max_pixels: int) -> ImageInfo:
|
||||
"""Проверить файл (seekable) и вернуть реальный тип и размеры.
|
||||
Бросает ImageTooLarge или BadImage. Блокирующая: вызывать в пуле потоков."""
|
||||
f.seek(0)
|
||||
actual = sniff(f.read(16))
|
||||
if actual is None:
|
||||
raise BadImage("not a PNG or JPEG")
|
||||
if actual != declared_type:
|
||||
raise BadImage("content does not match Content-Type")
|
||||
f.seek(0)
|
||||
try:
|
||||
if actual == "image/png":
|
||||
w, h = _check_png(f, max_pixels)
|
||||
else:
|
||||
w, h = _check_jpeg(f, max_pixels)
|
||||
except (ImageTooLarge, BadImage):
|
||||
raise
|
||||
except (zlib.error, struct.error, ValueError, OSError, EOFError, IndexError) as exc:
|
||||
# Любая ошибка разбора содержимого клиента — это битый файл (415), а не 500.
|
||||
raise BadImage("corrupt image data") from exc
|
||||
return ImageInfo(actual, w, h)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- PNG
|
||||
# color type → число каналов; допустимые глубины цвета (спецификация PNG, 11.2.2)
|
||||
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
_PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
||||
# Adam7: (x0, y0, dx, dy) семи проходов
|
||||
_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
|
||||
|
||||
def _png_layout(w: int, h: int, bits_per_pixel: int, interlaced: bool) -> list[tuple[int, int]]:
|
||||
"""Список (число строк, длина строки с байтом фильтра) для всех проходов."""
|
||||
def stride(width: int) -> int:
|
||||
return 1 + (width * bits_per_pixel + 7) // 8
|
||||
|
||||
if not interlaced:
|
||||
return [(h, stride(w))]
|
||||
out = []
|
||||
for x0, y0, dx, dy in _ADAM7:
|
||||
pw = (w - x0 + dx - 1) // dx if w > x0 else 0
|
||||
ph = (h - y0 + dy - 1) // dy if h > y0 else 0
|
||||
if pw and ph:
|
||||
out.append((ph, stride(pw)))
|
||||
return out
|
||||
|
||||
|
||||
def _read_exact(f: BinaryIO, n: int) -> bytes:
|
||||
data = f.read(n)
|
||||
if len(data) != n:
|
||||
raise BadImage("truncated PNG")
|
||||
return data
|
||||
|
||||
|
||||
def _check_png(f: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
if f.read(8) != PNG_SIG:
|
||||
raise BadImage("bad PNG signature")
|
||||
width = height = 0
|
||||
layout: list[tuple[int, int]] = []
|
||||
expected = produced = 0
|
||||
seg = row_left = 0 # позиция для проверки байтов фильтра строк
|
||||
inflater = None
|
||||
seen_ihdr = seen_plte = seen_idat = seen_iend = False
|
||||
color_type = 0
|
||||
|
||||
def consume(raw: bytes) -> None:
|
||||
"""Учесть распакованные байты: длина и байт фильтра в начале каждой строки."""
|
||||
nonlocal produced, seg, row_left
|
||||
produced += len(raw)
|
||||
if produced > expected:
|
||||
raise BadImage("PNG image data is longer than declared")
|
||||
pos = 0
|
||||
while pos < len(raw):
|
||||
if row_left == 0: # начало новой строки: байт фильтра 0..4
|
||||
if raw[pos] > 4:
|
||||
raise BadImage("bad PNG filter type")
|
||||
rows, stride = layout[seg]
|
||||
layout[seg] = (rows - 1, stride)
|
||||
if layout[seg][0] == 0:
|
||||
seg += 1
|
||||
row_left = stride
|
||||
take = min(row_left, len(raw) - pos)
|
||||
row_left -= take
|
||||
pos += take
|
||||
|
||||
def feed(data: bytes) -> None:
|
||||
"""Распаковать порциями ≤64 КиБ: бомба внутри IDAT не займёт память.
|
||||
Цикл продолжается, пока zlib отдаёт полный буфер (могут остаться данные)."""
|
||||
while True:
|
||||
out = inflater.decompress(data, CHUNK)
|
||||
consume(out)
|
||||
data = inflater.unconsumed_tail
|
||||
if inflater.eof or (not data and len(out) < CHUNK):
|
||||
return
|
||||
|
||||
while not seen_iend:
|
||||
length, ctype = struct.unpack(">I4s", _read_exact(f, 8))
|
||||
if length > 0x7FFFFFFF or not ctype.isalpha():
|
||||
raise BadImage("bad PNG chunk")
|
||||
crc = zlib.crc32(ctype)
|
||||
if not seen_ihdr and ctype != b"IHDR":
|
||||
raise BadImage("PNG must start with IHDR")
|
||||
|
||||
if ctype == b"IHDR":
|
||||
if seen_ihdr or length != 13:
|
||||
raise BadImage("bad IHDR")
|
||||
data = _read_exact(f, 13)
|
||||
crc = zlib.crc32(data, crc)
|
||||
width, height, depth, color_type, comp, filt, interlace = struct.unpack(">IIBBBBB", data)
|
||||
if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
||||
raise BadImage("bad PNG dimensions")
|
||||
# Лимит пикселей — ДО распаковки данных.
|
||||
if width * height > max_pixels:
|
||||
raise ImageTooLarge(f"{width}x{height} exceeds pixel limit")
|
||||
if color_type not in _PNG_CHANNELS or depth not in _PNG_DEPTHS[color_type] \
|
||||
or comp != 0 or filt != 0 or interlace not in (0, 1):
|
||||
raise BadImage("unsupported PNG header")
|
||||
layout = _png_layout(width, height, depth * _PNG_CHANNELS[color_type], interlace == 1)
|
||||
expected = sum(rows * stride for rows, stride in layout)
|
||||
inflater = zlib.decompressobj()
|
||||
seen_ihdr = True
|
||||
else:
|
||||
left = length
|
||||
while left:
|
||||
part = _read_exact(f, min(CHUNK, left))
|
||||
left -= len(part)
|
||||
crc = zlib.crc32(part, crc)
|
||||
if ctype == b"IDAT":
|
||||
if inflater is None or inflater.eof:
|
||||
raise BadImage("unexpected PNG data")
|
||||
feed(part)
|
||||
if ctype == b"IDAT":
|
||||
seen_idat = True
|
||||
elif ctype == b"PLTE":
|
||||
seen_plte = True
|
||||
elif ctype == b"IEND":
|
||||
seen_iend = True
|
||||
stored_crc = struct.unpack(">I", _read_exact(f, 4))[0]
|
||||
if stored_crc != crc & 0xFFFFFFFF:
|
||||
raise BadImage("PNG CRC mismatch")
|
||||
|
||||
if not seen_idat or inflater is None:
|
||||
raise BadImage("PNG has no image data")
|
||||
if color_type == 3 and not seen_plte:
|
||||
raise BadImage("palette PNG without PLTE")
|
||||
if not inflater.eof or produced != expected:
|
||||
raise BadImage("PNG image data is incomplete")
|
||||
return width, height
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- JPEG
|
||||
def _check_jpeg(f: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
# Страховка Pillow от бомб; основной лимит проверяем сами ниже.
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
try:
|
||||
with Image.open(f, formats=["JPEG"]) as img: # читает только заголовок
|
||||
width, height = img.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise BadImage("bad JPEG dimensions")
|
||||
if width * height > max_pixels:
|
||||
raise ImageTooLarge(f"{width}x{height} exceeds pixel limit")
|
||||
# Декодируем весь поток в масштабе 1/8: битый/усечённый файл даст ошибку,
|
||||
# а памяти нужно (W/8)×(H/8)×3 байт вместо W×H×3.
|
||||
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
img.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
raise ImageTooLarge("pixel limit") from exc
|
||||
except (ImageTooLarge, BadImage):
|
||||
raise
|
||||
except (OSError, SyntaxError, ValueError, struct.error, EOFError, IndexError) as exc:
|
||||
raise BadImage("corrupt JPEG") from exc
|
||||
return width, height
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Построение вариантов изображения (lab5). Выполняется в ДОЧЕРНЕМ процессе worker
|
||||
(app/worker.py → run_isolated), чтобы ограничить память и время одной задачи:
|
||||
|
||||
- RLIMIT_AS (IMAGE_MEMORY_MB): нехватка памяти → MemoryError в ребёнке, а не OOM всего worker;
|
||||
- RLIMIT_CPU и wall-clock таймаут родителя (JOB_TIMEOUT_SECONDS): зависшее
|
||||
декодирование убивается, worker живёт дальше;
|
||||
- пиксели проверяются по заголовку ДО распаковки (MAX_IMAGE_PIXELS = 40 млн).
|
||||
|
||||
Варианты (контракт, contracts/README.md «Фоновая обработка»):
|
||||
thumbnail — JPEG, вписан в 256×256, пропорции сохраняются, маленькие НЕ увеличиваются,
|
||||
размеры округляются вниз, минимум 1 px (320×200 → 256×160);
|
||||
crop — JPEG, центрированный квадрат 128×128 (маленькие исходники увеличиваются);
|
||||
optimized — WebP, вписан в 1280×1280 без увеличения, quality 80, method 4.
|
||||
Во всех вариантах: EXIF orientation применяется; EXIF/XMP/ICC в результат НЕ пишутся
|
||||
(Pillow пишет метаданные только если их явно передать в save()). Прозрачность в JPEG
|
||||
компонуется на белый фон; в WebP альфа-канал сохраняется.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import traceback
|
||||
import warnings
|
||||
|
||||
from PIL import Image, ImageFile, ImageOps
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
|
||||
THUMB_BOX = 256
|
||||
CROP_SIDE = 128
|
||||
OPTIMIZED_BOX = 1280
|
||||
JPEG_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
WEBP_METHOD = 4
|
||||
|
||||
# Порядок вариантов в ответе API и набор, который обещает уровень «4».
|
||||
VARIANTS = ("thumbnail", "crop", "optimized")
|
||||
VARIANT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
||||
VARIANT_EXT = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
|
||||
|
||||
|
||||
class PermanentImageError(Exception):
|
||||
"""Изображение не обработать никогда (битое, слишком большое) → failed invalid_image."""
|
||||
|
||||
|
||||
def fit_size(w: int, h: int, box: int) -> tuple[int, int]:
|
||||
"""Вписать в box×box без увеличения, округление вниз, минимум 1 px."""
|
||||
if w <= box and h <= box:
|
||||
return w, h
|
||||
if w >= h:
|
||||
return box, max(1, h * box // w)
|
||||
return max(1, w * box // h), box
|
||||
|
||||
|
||||
def _oriented_size(img: Image.Image) -> tuple[int, int]:
|
||||
w, h = img.size
|
||||
try:
|
||||
orientation = img.getexif().get(0x0112, 1)
|
||||
except Exception: # noqa: BLE001 — битый EXIF не повод падать; ориентация по умолчанию
|
||||
orientation = 1
|
||||
return (h, w) if orientation in (5, 6, 7, 8) else (w, h)
|
||||
|
||||
|
||||
def _on_white(img: Image.Image) -> Image.Image:
|
||||
"""RGB для JPEG: прозрачность компонуется на белый."""
|
||||
if img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
bg = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
bg.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return bg
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def _has_alpha(img: Image.Image) -> bool:
|
||||
return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info)
|
||||
|
||||
|
||||
def render(src_path: str, out_dir: str, content_type: str, max_pixels: int) -> dict:
|
||||
fmt = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
try:
|
||||
with Image.open(src_path, formats=[fmt]) as img:
|
||||
w0, h0 = img.size
|
||||
if w0 <= 0 or h0 <= 0 or w0 * h0 > max_pixels:
|
||||
raise PermanentImageError("pixel limit")
|
||||
ow, oh = _oriented_size(img) # размеры после поворота по EXIF
|
||||
thumb = fit_size(ow, oh, THUMB_BOX)
|
||||
opt = fit_size(ow, oh, OPTIMIZED_BOX)
|
||||
if fmt == "JPEG":
|
||||
# Декодируем сразу в уменьшенном масштабе (1/2, 1/4, 1/8), но не меньше,
|
||||
# чем нужно самому большому варианту (optimized) и crop: экономия памяти и времени.
|
||||
rw, rh = (opt[1], opt[0]) if (ow, oh) != (w0, h0) else opt # в осях файла
|
||||
img.draft("RGB", (max(rw, min(w0, CROP_SIDE)), max(rh, min(h0, CROP_SIDE))))
|
||||
img.load()
|
||||
base = ImageOps.exif_transpose(img) # копия с применённой ориентацией
|
||||
except PermanentImageError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
raise PermanentImageError("pixel limit") from exc
|
||||
except (OSError, SyntaxError, ValueError, EOFError, IndexError, KeyError) as exc:
|
||||
raise PermanentImageError(f"cannot decode: {type(exc).__name__}") from exc
|
||||
|
||||
out = {}
|
||||
rgb = _on_white(base)
|
||||
|
||||
t = rgb.resize(thumb, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
out["thumbnail"] = _save(t, out_dir, "thumbnail", "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
|
||||
bw, bh = rgb.size
|
||||
side = min(bw, bh)
|
||||
left, top = (bw - side) // 2, (bh - side) // 2
|
||||
c = rgb.crop((left, top, left + side, top + side)).resize(
|
||||
(CROP_SIDE, CROP_SIDE), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
out["crop"] = _save(c, out_dir, "crop", "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
|
||||
src = base.convert("RGBA") if _has_alpha(base) else rgb
|
||||
o = src if src.size == opt else src.resize(opt, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
out["optimized"] = _save(o, out_dir, "optimized", "WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD)
|
||||
return out
|
||||
|
||||
|
||||
def _save(img: Image.Image, out_dir: str, name: str, fmt: str, **opts) -> dict:
|
||||
path = os.path.join(out_dir, f"{name}.{VARIANT_EXT[name]}")
|
||||
img.save(path, fmt, **opts) # без exif=/icc_profile= → метаданных нет
|
||||
return {"path": path, "content_type": VARIANT_TYPES[name],
|
||||
"width": img.size[0], "height": img.size[1]}
|
||||
|
||||
|
||||
def child_main(src_path: str, out_dir: str, content_type: str, max_pixels: int,
|
||||
memory_mb: int, cpu_seconds: int, result_path: str,
|
||||
slow_seconds: float = 0.0) -> None:
|
||||
"""Точка входа дочернего процесса. Результат — JSON-файл; код выхода 0.
|
||||
slow_seconds > 0 — только для теста «долгой, но живой» задачи (fault-профиль)."""
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory_mb * 1024 * 1024, memory_mb * 1024 * 1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1))
|
||||
if slow_seconds > 0:
|
||||
import time
|
||||
time.sleep(slow_seconds) # sleep не тратит CPU: держит только wall-clock
|
||||
try:
|
||||
result = {"ok": True, "variants": render(src_path, out_dir, content_type, max_pixels)}
|
||||
except PermanentImageError as exc:
|
||||
result = {"ok": False, "permanent": True, "code": "invalid_image", "detail": str(exc)[:200]}
|
||||
except MemoryError:
|
||||
result = {"ok": False, "permanent": True, "code": "invalid_image", "detail": "memory limit"}
|
||||
except Exception as exc: # noqa: BLE001 — неизвестное: временная ошибка, retry ограничен
|
||||
result = {"ok": False, "permanent": False, "code": "processing_failed",
|
||||
"detail": type(exc).__name__,
|
||||
"trace": traceback.format_exc(limit=3)[-500:]}
|
||||
with open(result_path, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f)
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
Очистка «сирот» после сбоев между S3 и PostgreSQL (lab4, уровень 4).
|
||||
|
||||
Протокол загрузки (app/main.py → upload_attachment):
|
||||
1. INSERT upload_intents(object_key) + COMMIT ← намерение записано
|
||||
2. PUT объекта в S3
|
||||
3. BEGIN; DELETE upload_intents …; INSERT attachments …; COMMIT ← 201 только после этого
|
||||
Если 2 или 3 не удались — API сразу компенсирует (удаляет объект и строку журнала).
|
||||
Если компенсация не удалась или процесс упал (kill -9, OOM, рестарт контейнера) —
|
||||
в журнале остаётся строка без метаданных. Её и обрабатывает janitor:
|
||||
|
||||
строка upload_intents СТАРШЕ grace period →
|
||||
BEGIN; DELETE строку (блокировка); DELETE объект в S3; COMMIT
|
||||
- ошибка S3 → ROLLBACK, строка остаётся, повтор на следующем проходе;
|
||||
- упали после DELETE в S3, до COMMIT → строка остаётся, повторное
|
||||
удаление объекта безопасно (DELETE в S3 идемпотентен).
|
||||
|
||||
Почему grace period безопасен: живая загрузка не может длиться дольше
|
||||
UPLOAD_BODY_TIMEOUT_SECONDS (60 c) + таймауты S3/БД (~30 c). По умолчанию
|
||||
UPLOAD_GRACE_SECONDS = 900 c (15 мин) — в 10 раз больше. Даже если
|
||||
загрузка «проспала» grace period, шаг 3 удаляет строку журнала в той же
|
||||
транзакции, что и INSERT метаданных: если janitor уже забрал строку, загрузка
|
||||
получит 0 строк, откатится и ответит 503 — 201 на удалённый объект невозможен.
|
||||
Готовые вложения (есть строка в attachments) janitor не трогает никогда.
|
||||
|
||||
Страховка (--scan-bucket): обход bucket по prefix originals/ — объект старше
|
||||
grace period, для которого нет ни attachments, ни upload_intents, — сирота.
|
||||
|
||||
Запуск:
|
||||
- в каждой реплике API фоновым циклом (JANITOR_INTERVAL_SECONDS), но
|
||||
одновременно работает только одна: pg_try_advisory_lock;
|
||||
- вручную: python -m app.janitor --once [--dry-run] [--scan-bucket] [--grace-seconds N]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import psycopg
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from . import db, storage
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("messenger.janitor")
|
||||
LOCK_ID = 824_113_2027
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def _sweep_intents(conn, grace: int, dry_run: bool, stats: dict) -> None:
|
||||
cur = await conn.execute(
|
||||
"SELECT object_key FROM upload_intents "
|
||||
"WHERE created_at < now() - make_interval(secs => %s) ORDER BY created_at LIMIT %s",
|
||||
(grace, BATCH))
|
||||
keys = [r[0] for r in await cur.fetchall()]
|
||||
for key in keys:
|
||||
stats["intents_expired"] += 1
|
||||
try:
|
||||
async with conn.transaction():
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM upload_intents WHERE object_key = %s "
|
||||
"AND created_at < now() - make_interval(secs => %s) RETURNING 1", (key, grace))
|
||||
if await cur.fetchone() is None:
|
||||
continue # загрузка успела завершиться — не наше
|
||||
cur = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,))
|
||||
committed = await cur.fetchone() is not None
|
||||
if dry_run:
|
||||
stats["would_delete_objects"] += 0 if committed else 1
|
||||
raise psycopg.Rollback()
|
||||
if not committed:
|
||||
await storage.delete(key) # ошибка → ROLLBACK, строка останется
|
||||
stats["objects_deleted"] += 1
|
||||
stats["intents_removed"] += 1
|
||||
except storage.StorageUnavailable:
|
||||
stats["errors"] += 1
|
||||
log.warning("janitor: s3 unavailable, will retry later")
|
||||
return
|
||||
|
||||
|
||||
async def _scan_bucket(conn, grace: int, dry_run: bool, stats: dict) -> None:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace)
|
||||
|
||||
def old_keys() -> list[str]:
|
||||
# lab5: оригиналы публикует worker в originals/, приём идёт в staging/ — обходим оба.
|
||||
return [k for prefix in (storage.ORIGINALS_PREFIX, storage.STAGING_PREFIX)
|
||||
for k, modified, _ in storage.list_sync(prefix) if modified < cutoff]
|
||||
|
||||
try:
|
||||
candidates = await run_in_threadpool(old_keys)
|
||||
except storage.StorageUnavailable:
|
||||
stats["errors"] += 1
|
||||
return
|
||||
stats["objects_scanned"] = len(candidates)
|
||||
for i in range(0, len(candidates), BATCH):
|
||||
part = candidates[i:i + BATCH]
|
||||
cur = await conn.execute(
|
||||
"SELECT object_key FROM attachments WHERE object_key = ANY(%s) "
|
||||
"UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s) "
|
||||
"UNION SELECT object_key FROM upload_intents WHERE object_key = ANY(%s)", (part, part, part))
|
||||
known = {r[0] for r in await cur.fetchall()}
|
||||
for key in part:
|
||||
if key in known:
|
||||
continue
|
||||
stats["orphans_found"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
try:
|
||||
await storage.delete(key)
|
||||
stats["objects_deleted"] += 1
|
||||
except storage.StorageUnavailable:
|
||||
stats["errors"] += 1
|
||||
return
|
||||
|
||||
|
||||
async def run_once(grace: int | None = None, dry_run: bool = False,
|
||||
scan_bucket: bool = False) -> dict:
|
||||
grace = settings.upload_grace_seconds if grace is None else grace
|
||||
stats = {"grace_seconds": grace, "dry_run": dry_run, "locked": False,
|
||||
"intents_expired": 0, "intents_removed": 0, "objects_deleted": 0,
|
||||
"would_delete_objects": 0, "orphans_found": 0, "errors": 0}
|
||||
started = time.monotonic()
|
||||
async with db.get_pool().connection() as conn:
|
||||
await conn.set_autocommit(True)
|
||||
cur = await conn.execute("SELECT pg_try_advisory_lock(%s)", (LOCK_ID,))
|
||||
if not (await cur.fetchone())[0]:
|
||||
return stats # другая реплика уже чистит
|
||||
stats["locked"] = True
|
||||
try:
|
||||
await _sweep_intents(conn, grace, dry_run, stats)
|
||||
if scan_bucket:
|
||||
await _scan_bucket(conn, grace, dry_run, stats)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
await conn.set_autocommit(False)
|
||||
stats["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
|
||||
return stats
|
||||
|
||||
|
||||
async def loop_forever() -> None:
|
||||
"""Фоновый цикл в реплике API. Ошибки логируются, цикл не падает."""
|
||||
runs = 0
|
||||
while True:
|
||||
# Случайный сдвиг: реплики не стучатся в advisory lock одновременно.
|
||||
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
|
||||
runs += 1
|
||||
try:
|
||||
stats = await run_once(scan_bucket=(runs % 60 == 0)) # обход bucket — примерно раз в час
|
||||
if stats["intents_removed"] or stats["objects_deleted"] or stats["errors"]:
|
||||
log.info("janitor: %s", stats)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — фоновая задача не должна умирать
|
||||
log.warning("janitor run failed: %s", type(exc).__name__)
|
||||
|
||||
|
||||
async def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="cleanup of orphan S3 objects")
|
||||
ap.add_argument("--once", action="store_true", help="один проход (по умолчанию)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="только показать, ничего не удалять")
|
||||
ap.add_argument("--scan-bucket", action="store_true", help="дополнительно обойти bucket")
|
||||
ap.add_argument("--grace-seconds", type=int, default=None)
|
||||
args = ap.parse_args()
|
||||
await db.open_pool()
|
||||
storage.init()
|
||||
try:
|
||||
await db.get_pool().wait(timeout=10)
|
||||
stats = await run_once(args.grace_seconds, args.dry_run, args.scan_bucket)
|
||||
finally:
|
||||
await db.close_pool()
|
||||
print(json.dumps(stats, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
asyncio.run(_cli())
|
||||
+634
@@ -0,0 +1,634 @@
|
||||
"""
|
||||
HTTP-слой мессенджера (lab4).
|
||||
|
||||
Клиент → LB (HAProxy) → эта реплика API (их ≥2, взаимозаменяемы)
|
||||
→ PostgreSQL (источник истины: данные и метаданные вложений)
|
||||
→ Redis (общий кэш сессий)
|
||||
→ MinIO, приватный bucket (только байты вложений)
|
||||
|
||||
lab2: пароль Argon2id, вход Basic → Bearer-сессия в PostgreSQL, X-User-Id игнорируется.
|
||||
lab3: X-Instance-Id в каждом ответе; POST сообщения требует Idempotency-Key,
|
||||
идемпотентность общая для всех реплик (таблица idempotency_keys);
|
||||
у реплики нет локального состояния, от которого зависит корректность.
|
||||
lab4: вложения PNG/JPEG: GET /attachments/{id} и /content (только участникам чата),
|
||||
attachment_ids в сообщениях.
|
||||
lab5: POST /chats/{id}/attachments → 202 queued. До ответа: оригинал целиком в приватном
|
||||
staging (S3), в ОДНОЙ транзакции — вложение, job и строка outbox. Затем сообщение
|
||||
публикуется в брокер (Redis Streams с AOF); если не вышло — его опубликует relay.
|
||||
Тяжёлую обработку (варианты) делает отдельный процесс worker (app/worker.py).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from psycopg_pool import PoolTimeout
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from . import broker, db, images, janitor, repo, security, sessions, storage, uploads
|
||||
from .config import settings
|
||||
from .migrate import migrate
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
# Только fullmatch: в Python "$" совпадает и перед завершающим "\n",
|
||||
# поэтому match("^…$") пропустил бы "review123\n" до CHECK в БД (500).
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
MAX_ATTACHMENTS_PER_MESSAGE = 10
|
||||
KNOWN_VARIANTS = ("thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
if settings.migrate_on_start:
|
||||
try:
|
||||
applied = await migrate()
|
||||
logger.info("migrations: %s", applied or "up to date")
|
||||
except psycopg.OperationalError as exc:
|
||||
# БД ещё не готова: API всё равно стартует, /health/ready покажет 503.
|
||||
logger.warning("migrations skipped, database unavailable: %s", type(exc).__name__)
|
||||
await db.open_pool()
|
||||
sessions.init_redis()
|
||||
storage.init()
|
||||
broker.init_async()
|
||||
cleaner = asyncio.create_task(janitor.loop_forever()) if settings.janitor_enabled else None
|
||||
yield
|
||||
if cleaner is not None:
|
||||
cleaner.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await cleaner
|
||||
await sessions.close_redis()
|
||||
await broker.close_async()
|
||||
await db.close_pool()
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — lab5", lifespan=lifespan)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 1. ЕДИНЫЙ ФОРМАТ ОШИБОК {"error":{"code","message","request_id"}}
|
||||
# ============================================================================
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def error_response(request: Request, status: int, code: str, message: str,
|
||||
headers: dict | None = None) -> JSONResponse:
|
||||
rid = getattr(request.state, "request_id", str(uuid.uuid4()))
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||||
headers={"X-Request-Id": rid, "X-Instance-Id": settings.instance_id, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError):
|
||||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def handle_validation(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "invalid request body or parameters")
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def handle_http(request: Request, exc: StarletteHTTPException):
|
||||
# Только коды из enum контракта (contracts/openapi.json → Error.code).
|
||||
code = {401: "unauthorized", 404: "not_found", 409: "conflict",
|
||||
413: "payload_too_large", 415: "unsupported_media_type",
|
||||
429: "rate_limited", 503: "unavailable"}.get(
|
||||
exc.status_code, "internal_error" if exc.status_code >= 500 else "invalid_request")
|
||||
return error_response(request, exc.status_code, code, str(exc.detail))
|
||||
|
||||
|
||||
@app.exception_handler(psycopg.OperationalError)
|
||||
@app.exception_handler(PoolTimeout)
|
||||
async def handle_db_down(request: Request, exc: Exception):
|
||||
# БД недоступна или не ответила за таймаут: ограниченная ошибка, а не зависание.
|
||||
logger.warning("database unavailable: %s", type(exc).__name__)
|
||||
return error_response(request, 503, "unavailable", "service temporarily unavailable")
|
||||
|
||||
|
||||
@app.exception_handler(storage.StorageUnavailable)
|
||||
async def handle_storage_down(request: Request, exc: Exception):
|
||||
# S3 недоступен/не ответил за таймаут: 503, текстовая часть мессенджера продолжает работать.
|
||||
return error_response(request, 503, "unavailable", "file storage temporarily unavailable")
|
||||
|
||||
|
||||
@app.exception_handler(uploads.UploadError)
|
||||
async def handle_upload_error(request: Request, exc: uploads.UploadError):
|
||||
return error_response(request, exc.status, exc.code, exc.message)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def handle_unexpected(request: Request, exc: Exception):
|
||||
logger.exception("unexpected error")
|
||||
return error_response(request, 500, "internal_error", "internal error")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 2. X-Request-Id, X-Instance-Id + лог без тела запроса, токенов и паролей
|
||||
# ============================================================================
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
rid = str(uuid.uuid4())
|
||||
request.state.request_id = rid
|
||||
start = time.monotonic()
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = settings.instance_id # какая реплика ответила
|
||||
logger.info("rid=%s %s %s -> %s %sms", rid, request.method, request.url.path,
|
||||
response.status_code, round((time.monotonic() - start) * 1000, 1))
|
||||
return response
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 3. АУТЕНТИФИКАЦИЯ
|
||||
# Bearer — для всех бизнес-запросов. Basic — только на POST /auth/sessions.
|
||||
# X-User-Id больше ничего не значит и ни на что не влияет.
|
||||
# ============================================================================
|
||||
def unauthorized(message: str = "invalid or missing bearer token") -> ApiError:
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": BEARER_CHALLENGE})
|
||||
|
||||
|
||||
def _bearer_token(request: Request) -> str:
|
||||
auth = request.headers.get("authorization", "")
|
||||
scheme, _, token = auth.partition(" ")
|
||||
token = token.strip()
|
||||
if scheme.lower() != "bearer" or not token or len(token) > 512:
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
async def current_user(request: Request) -> dict:
|
||||
"""Зависимость FastAPI: выполняется ДО разбора тела → сначала 401, потом 400."""
|
||||
token = _bearer_token(request)
|
||||
user = await sessions.resolve(db.get_pool(), token, repo.get_user)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
request.state.token = token
|
||||
return user
|
||||
|
||||
|
||||
def _parse_basic(request: Request) -> tuple[str, str]:
|
||||
bad = ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "basic" or not value:
|
||||
raise bad
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
raise bad
|
||||
username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо
|
||||
if not sep or not username.isascii() or not (12 <= len(password) <= 128):
|
||||
raise bad
|
||||
return username, password
|
||||
|
||||
|
||||
def require_json(request: Request) -> None:
|
||||
"""Тип тела проверяем ДО валидации тела: не JSON → 415, а не 400.
|
||||
Подключается после current_user, поэтому без токена по-прежнему сначала 401.
|
||||
Параметры вроде '; charset=utf-8' допустимы."""
|
||||
media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media_type != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
|
||||
|
||||
_kdf_slots: asyncio.Semaphore | None = None
|
||||
|
||||
|
||||
async def kdf(fn, *args):
|
||||
"""Argon2id по умолчанию берёт 64 МиБ RAM на один хэш. Без ограничения 20
|
||||
одновременных регистраций/входов на реплике = >1 ГиБ и OOM-kill контейнера
|
||||
с лимитом 256 МиБ. Поэтому одновременно считаем не больше
|
||||
PASSWORD_HASH_CONCURRENCY хэшей, остальные ждут в очереди."""
|
||||
global _kdf_slots
|
||||
if _kdf_slots is None:
|
||||
_kdf_slots = asyncio.Semaphore(settings.password_hash_concurrency)
|
||||
async with _kdf_slots:
|
||||
return await run_in_threadpool(fn, *args)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 4. МОДЕЛИ ВВОДА
|
||||
# ============================================================================
|
||||
class CreateUser(BaseModel):
|
||||
model_config = {"extra": "forbid"}
|
||||
username: str
|
||||
display_name: str
|
||||
password: str
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def check_username(cls, v: str) -> str:
|
||||
if not USERNAME_RE.fullmatch(v):
|
||||
raise ValueError("username must match [a-z0-9_]{3,32}")
|
||||
return v
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def check_display_name(cls, v: str) -> str:
|
||||
if not (1 <= len(v) <= 100) or not v.strip():
|
||||
raise ValueError("display_name must be 1..100 non-blank chars")
|
||||
return v
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def check_password(cls, v: str) -> str:
|
||||
if not (12 <= len(v) <= 128) or not v.strip():
|
||||
raise ValueError("password must be 12..128 chars")
|
||||
return v
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = {"extra": "forbid"}
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def check_title(cls, v: str) -> str:
|
||||
if not (1 <= len(v) <= 100) or not v.strip():
|
||||
raise ValueError("title must be 1..100 non-blank chars")
|
||||
return v
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = {"extra": "forbid"}
|
||||
text: str
|
||||
# lab4: 0–10 уникальных id вложений этого чата; отсутствие поля == [].
|
||||
attachment_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
@field_validator("text")
|
||||
@classmethod
|
||||
def check_text(cls, v: str) -> str:
|
||||
if len(v) > 4000:
|
||||
raise ValueError("text must be at most 4000 chars")
|
||||
return v
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def check_attachment_ids(cls, v: list[str]) -> list[str]:
|
||||
if len(v) > MAX_ATTACHMENTS_PER_MESSAGE:
|
||||
raise ValueError("at most 10 attachment_ids")
|
||||
if len(set(v)) != len(v):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def text_or_attachments(self):
|
||||
# Пустой/пробельный текст допустим только вместе с вложениями.
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError("text must be non-blank when there are no attachments")
|
||||
return self
|
||||
|
||||
|
||||
def parse_uuid(value: str, what: str) -> uuid.UUID:
|
||||
try:
|
||||
return uuid.UUID(value)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", f"{what} is not a valid uuid")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 5. ENDPOINTS
|
||||
# ============================================================================
|
||||
@app.get("/health/live")
|
||||
async def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def health_ready(request: Request):
|
||||
# Критичная зависимость — только PostgreSQL. Redis — кэш: его отказ не делает API неготовым.
|
||||
try:
|
||||
async with db.get_pool().connection() as conn:
|
||||
await conn.execute("SELECT 1 FROM schema_migrations LIMIT 1")
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
return error_response(request, 503, "unavailable", "database is not ready")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# -------- users и сессии --------
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(body: CreateUser, _json: None = Depends(require_json)):
|
||||
# Argon2 намеренно медленный — считаем в пуле потоков, чтобы не блокировать event loop.
|
||||
password_hash = await kdf(security.hash_password, body.password)
|
||||
async with db.get_pool().connection() as conn:
|
||||
try:
|
||||
return await repo.create_user(conn, body.username, body.display_name, password_hash)
|
||||
except repo.UsernameTaken:
|
||||
raise ApiError(409, "conflict", "username already taken")
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def create_session(request: Request):
|
||||
username, password = _parse_basic(request)
|
||||
async with db.get_pool().connection() as conn:
|
||||
row = await repo.get_credentials(conn, username)
|
||||
stored_hash = row[4] if row else None
|
||||
# Проверяем хэш даже для несуществующего пользователя — одинаковое время ответа.
|
||||
ok = await kdf(security.verify_password, stored_hash, password)
|
||||
if not ok:
|
||||
raise ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
user = repo.user_out(row)
|
||||
async with db.get_pool().connection() as conn:
|
||||
if security.needs_rehash(stored_hash):
|
||||
await repo.update_password_hash(conn, row[0], await kdf(security.hash_password, password))
|
||||
token, expires_at = await sessions.create_session(conn, row[0])
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": repo.iso(expires_at), "user": user}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def delete_session(request: Request, user: dict = Depends(current_user)):
|
||||
try:
|
||||
revoked = await sessions.revoke(db.get_pool(), request.state.token)
|
||||
except sessions.CacheUnavailable:
|
||||
# 204 только при гарантированном отзыве; иначе ограниченная ошибка, повторить позже.
|
||||
raise ApiError(503, "unavailable", "session cache unavailable, logout not confirmed")
|
||||
if not revoked:
|
||||
raise unauthorized()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def get_me(user: dict = Depends(current_user)):
|
||||
return user
|
||||
|
||||
|
||||
# -------- chats --------
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(body: CreateChat, user: dict = Depends(current_user),
|
||||
_json: None = Depends(require_json)):
|
||||
members = body.member_ids
|
||||
if not (1 <= len(members) <= 100):
|
||||
raise ApiError(400, "invalid_request", "member_ids must have 1..100 items")
|
||||
if len(set(members)) != len(members):
|
||||
raise ApiError(400, "invalid_request", "duplicate member_ids")
|
||||
ids = [parse_uuid(m, "member id") for m in members]
|
||||
if uuid.UUID(user["id"]) not in ids:
|
||||
raise ApiError(400, "invalid_request", "caller must be a member")
|
||||
async with db.get_pool().connection() as conn:
|
||||
# Одна транзакция: чат и все участники создаются вместе или не создаются вовсе.
|
||||
async with conn.transaction():
|
||||
if await repo.count_existing_users(conn, ids) != len(ids):
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
return await repo.insert_chat(conn, body.title, ids, user["id"])
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(user: dict = Depends(current_user),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, min_length=1)):
|
||||
async with db.get_pool().connection() as conn:
|
||||
try:
|
||||
items, nxt = await repo.list_chats(conn, user["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "bad cursor")
|
||||
return {"items": items, "next_cursor": nxt}
|
||||
|
||||
|
||||
async def accessible_chat(conn, chat_id: str, user: dict) -> dict:
|
||||
"""Кривой UUID → 400. Чата нет ИЛИ ты не участник → одинаковый 404."""
|
||||
cid = parse_uuid(chat_id, "chat_id")
|
||||
chat = await repo.get_chat_for_member(conn, cid, user["id"])
|
||||
if chat is None:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return chat
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
||||
async with db.get_pool().connection() as conn:
|
||||
return await accessible_chat(conn, chat_id, user)
|
||||
|
||||
|
||||
# -------- messages --------
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def send_message(chat_id: str, body: CreateMessage, user: dict = Depends(current_user),
|
||||
_json: None = Depends(require_json),
|
||||
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key")):
|
||||
# Ключ обязателен: без него клиент не может безопасно повторить запрос после таймаута.
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}")
|
||||
attachment_ids = [parse_uuid(a, "attachment_id") for a in body.attachment_ids]
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise ApiError(400, "invalid_request", "attachment_ids must be unique")
|
||||
async with db.get_pool().connection() as conn:
|
||||
# Повтор снова проверяет сессию (current_user) и доступ к чату.
|
||||
chat = await accessible_chat(conn, chat_id, user)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
# Отправитель — из сессии, а не из тела запроса.
|
||||
return await repo.send_message_idempotent(
|
||||
conn, chat["id"], user["id"], idempotency_key, body.text,
|
||||
settings.idempotency_ttl_seconds, attachment_ids)
|
||||
except repo.IdempotencyConflict:
|
||||
raise ApiError(409, "conflict", "Idempotency-Key was already used with a different request")
|
||||
except repo.AttachmentNotFound:
|
||||
# Нет такого вложения ИЛИ оно из другого чата — одинаковый 404.
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
except repo.AttachmentNotReady:
|
||||
raise ApiError(409, "conflict", "attachment is not usable")
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def list_messages(chat_id: str, user: dict = Depends(current_user),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, min_length=1)):
|
||||
async with db.get_pool().connection() as conn:
|
||||
chat = await accessible_chat(conn, chat_id, user)
|
||||
try:
|
||||
items, nxt = await repo.list_messages(conn, chat["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "bad cursor")
|
||||
return {"items": items, "next_cursor": nxt}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 6. ВЛОЖЕНИЯ (lab4): байты — в приватном S3, метаданные — в PostgreSQL
|
||||
# ============================================================================
|
||||
_upload_slots: asyncio.Semaphore | None = None
|
||||
_image_checks: asyncio.Semaphore | None = None
|
||||
|
||||
|
||||
def _slots() -> tuple[asyncio.Semaphore, asyncio.Semaphore]:
|
||||
global _upload_slots, _image_checks
|
||||
if _upload_slots is None:
|
||||
_upload_slots = asyncio.Semaphore(settings.upload_max_concurrent)
|
||||
_image_checks = asyncio.Semaphore(settings.image_check_concurrency)
|
||||
return _upload_slots, _image_checks
|
||||
|
||||
|
||||
async def _discard_upload(key: str) -> None:
|
||||
"""Компенсация: удалить объект и строку журнала. Если не вышло —
|
||||
строка журнала останется, и объект удалит janitor после grace period."""
|
||||
try:
|
||||
async with db.get_pool().connection() as conn:
|
||||
async with conn.transaction():
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM upload_intents WHERE object_key = %s RETURNING 1", (key,))
|
||||
if await cur.fetchone() is not None:
|
||||
await storage.delete(key)
|
||||
except Exception as exc: # noqa: BLE001 — компенсация best effort
|
||||
logger.warning("upload compensation failed, janitor will clean up: %s", type(exc).__name__)
|
||||
|
||||
|
||||
def _fault(request: Request, point: str) -> None:
|
||||
"""Внедрение отказа для тестов уровня 4 — только при FAULT_INJECTION=true
|
||||
(отдельная тестовая реплика api-fault). В обычных репликах — no-op."""
|
||||
if not settings.fault_injection or request.headers.get("x-fault-inject") != point:
|
||||
return
|
||||
logger.warning("FAULT INJECTION: %s", point)
|
||||
if point in ("crash-after-s3-put", "crash-after-commit-before-publish"):
|
||||
os._exit(1) # как kill -9: процесс умирает мгновенно, без finally
|
||||
raise RuntimeError("injected fault: " + point)
|
||||
|
||||
|
||||
async def _publish_now(outbox_id: int, payload: dict) -> None:
|
||||
"""Быстрый путь после COMMIT: сразу отдать сообщение брокеру и отметить строку outbox.
|
||||
Любая ошибка здесь НЕ влияет на ответ 202: строка outbox останется неопубликованной,
|
||||
и relay (app/worker.py, поток maintenance) опубликует её позже."""
|
||||
try:
|
||||
async with asyncio.timeout(settings.broker_timeout):
|
||||
stream_id = await broker.publish(payload) # ответ XADD = брокер принял (AOF fsync)
|
||||
async with db.get_pool().connection() as conn:
|
||||
await repo.mark_published(conn, outbox_id, stream_id)
|
||||
except Exception as exc: # noqa: BLE001 — намерение уже в outbox
|
||||
logger.warning("direct publish failed, outbox relay will retry: %s", type(exc).__name__)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request, user: dict = Depends(current_user),
|
||||
filename: str | None = Query(default=None)):
|
||||
# 1) доступ к чату → 2) имя → 3) заявленный тип → 4) заявленный размер; всё — до чтения тела.
|
||||
async with db.get_pool().connection() as conn:
|
||||
chat = await accessible_chat(conn, chat_id, user)
|
||||
name = uploads.validate_filename(filename)
|
||||
ctype = uploads.declared_type(request)
|
||||
uploads.check_declared_length(request, settings.max_upload_bytes)
|
||||
|
||||
slots, checks = _slots()
|
||||
async with slots: # не больше N загрузок на реплику
|
||||
body = await uploads.receive(request, settings.max_upload_bytes,
|
||||
settings.upload_body_timeout_seconds,
|
||||
settings.upload_spool_memory_bytes)
|
||||
try:
|
||||
async with checks: # CPU-тяжёлая проверка — ограниченно
|
||||
try:
|
||||
info = await run_in_threadpool(images.inspect, body.file, ctype,
|
||||
settings.max_image_pixels)
|
||||
except images.ImageTooLarge:
|
||||
raise ApiError(413, "payload_too_large", "image has too many pixels")
|
||||
except images.BadImage:
|
||||
raise ApiError(415, "unsupported_media_type",
|
||||
"file is not a valid image of the declared type")
|
||||
|
||||
storage.ensure_available() # S3 заведомо недоступен → 503 без записи в журнал
|
||||
attachment_id = uuid.uuid4()
|
||||
key = storage.staging_key(attachment_id) # ключ назначает сервер; сначала — staging
|
||||
async with db.get_pool().connection() as conn:
|
||||
await repo.add_upload_intent(conn, key, attachment_id) # журнал ДО PUT
|
||||
try:
|
||||
await storage.put(key, body.file, body.size, info.content_type,
|
||||
body.md5_b64, body.sha256_hex)
|
||||
except storage.StorageUnavailable:
|
||||
await _discard_upload(key)
|
||||
raise
|
||||
try:
|
||||
_fault(request, "crash-after-s3-put")
|
||||
_fault(request, "db-error-after-s3-put")
|
||||
async with db.get_pool().connection() as conn:
|
||||
async with conn.transaction():
|
||||
att, outbox_id, payload = await repo.commit_attachment(
|
||||
conn, attachment_id=attachment_id, chat_id=chat["id"],
|
||||
owner_id=user["id"], filename=name, content_type=info.content_type,
|
||||
size_bytes=body.size, sha256=body.sha256_hex,
|
||||
width=info.width, height=info.height,
|
||||
object_key=storage.object_key(attachment_id), staging_key=key,
|
||||
pipeline_version=settings.pipeline_version,
|
||||
max_attempts=settings.job_max_attempts)
|
||||
except Exception as exc:
|
||||
# PUT прошёл, COMMIT — нет: компенсируем сразу, 202 не отдаём.
|
||||
logger.warning("attachment commit failed: %s", type(exc).__name__)
|
||||
await _discard_upload(key)
|
||||
raise ApiError(503, "unavailable", "attachment was not saved, retry later")
|
||||
# COMMIT прошёл: оригинал в staging, job и outbox в БД — работа уже не потеряется.
|
||||
_fault(request, "crash-after-commit-before-publish")
|
||||
await _publish_now(outbox_id, payload)
|
||||
return att
|
||||
finally:
|
||||
body.file.close() # буфер запроса удаляется всегда
|
||||
|
||||
|
||||
async def accessible_attachment(conn, attachment_id: str, user: dict):
|
||||
aid = parse_uuid(attachment_id, "attachment_id")
|
||||
row = await repo.get_attachment_for_member(conn, aid, user["id"])
|
||||
if row is None:
|
||||
# Нет вложения ИЛИ пользователь не участник его чата — одинаковый 404.
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
return row
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def get_attachment(attachment_id: str, user: dict = Depends(current_user)):
|
||||
async with db.get_pool().connection() as conn:
|
||||
row = await accessible_attachment(conn, attachment_id, user)
|
||||
variants = await repo.get_variants(conn, row[0], settings.pipeline_version) \
|
||||
if row[7] == "ready" else []
|
||||
return repo.attachment_out(row, variants)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(attachment_id: str, user: dict = Depends(current_user),
|
||||
variant: str = Query(default="original")):
|
||||
parse_uuid(attachment_id, "attachment_id")
|
||||
if variant != "original" and variant not in KNOWN_VARIANTS:
|
||||
raise ApiError(400, "invalid_request", "unknown variant")
|
||||
async with db.get_pool().connection() as conn:
|
||||
row = await accessible_attachment(conn, attachment_id, user)
|
||||
att = repo.attachment_out(row)
|
||||
if att["status"] != "ready":
|
||||
# queued / processing / failed: байтов результата ещё (или уже) нет.
|
||||
raise ApiError(409, "conflict", "attachment is not ready")
|
||||
key, ctype, etag = row[11], att["content_type"], att["sha256"]
|
||||
if variant != "original":
|
||||
v = await repo.get_variant(conn, row[0], settings.pipeline_version, variant)
|
||||
if v is None:
|
||||
raise ApiError(404, "not_found", "variant not found")
|
||||
key, ctype, etag = v[0], v[1], v[3]
|
||||
try:
|
||||
stream, length = await storage.open_object(key)
|
||||
except storage.ObjectMissing:
|
||||
logger.error("object for attachment %s is missing in storage", att["id"])
|
||||
raise ApiError(500, "internal_error", "internal error")
|
||||
headers = {
|
||||
"Content-Length": str(length),
|
||||
"ETag": f'"{etag}"',
|
||||
"Cache-Control": "private, no-cache",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
# filename только для отображения; RFC 5987 — любые символы безопасно закодированы.
|
||||
"Content-Disposition": "inline; filename*=UTF-8''" + quote(att["filename"], safe=""),
|
||||
}
|
||||
# Байты идут из S3 клиенту кусками по 64 КиБ — файл целиком в RAM не загружается.
|
||||
return StreamingResponse(storage.iter_body(stream), media_type=ctype, headers=headers)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
Версионируемые миграции: migrations/NNNN_name.sql.
|
||||
|
||||
- Таблица schema_migrations хранит применённые версии → повторный запуск
|
||||
ничего не дублирует (идемпотентно).
|
||||
- Каждая миграция применяется в своей транзакции: либо целиком, либо никак.
|
||||
- pg_advisory_lock: если стартуют несколько API одновременно, миграции
|
||||
выполнит только один, остальные подождут.
|
||||
|
||||
Запуск вручную: python -m app.migrate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("messenger.migrate")
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
|
||||
LOCK_ID = 824_113_2026 # произвольная константа для advisory lock
|
||||
|
||||
|
||||
async def migrate(conninfo: str | None = None) -> list[str]:
|
||||
applied_now: list[str] = []
|
||||
async with await psycopg.AsyncConnection.connect(
|
||||
conninfo or settings.database_url,
|
||||
connect_timeout=settings.db_connect_timeout, autocommit=True) as conn:
|
||||
await conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
try:
|
||||
await conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
cur = await conn.execute("SELECT version FROM schema_migrations")
|
||||
done = {row[0] for row in await cur.fetchall()}
|
||||
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
version = path.stem
|
||||
if version in done:
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(path.read_text(encoding="utf-8"))
|
||||
await conn.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES (%s)", (version,))
|
||||
applied_now.append(version)
|
||||
log.info("migration applied: %s", version)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
return applied_now
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
result = asyncio.run(migrate())
|
||||
print("applied:", result or "nothing (up to date)")
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Реестр failed-задач и безопасный явный replay (lab5, уровень 4). Запуск в контейнере worker:
|
||||
|
||||
python -m app.replay --list # реестр failed (из БД)
|
||||
python -m app.replay --dlq # последние записи DLQ-stream брокера
|
||||
python -m app.replay --attachment <id> [--dry-run] [--actor имя]
|
||||
python -m app.replay --all-failed [--error-code retry_exhausted] [--dry-run]
|
||||
|
||||
Почему «повторить» безопасно:
|
||||
- replay меняет только job в статусе failed (FOR UPDATE): ready/queued/processing не трогает —
|
||||
уже успешную задачу второй раз исполнить нельзя;
|
||||
- job_id тот же, attempt растёт (attempt+1), бюджет попыток — ещё JOB_MAX_ATTEMPTS;
|
||||
- эффект защищён ключом (attachment_id, pipeline_version, variant) и детерминированными
|
||||
ключами S3: даже дубль сообщения не создаст второй набор вариантов;
|
||||
- если исходник в staging уже удалён по retention — replay отказывает (повторять нечего);
|
||||
- каждое действие пишется в job_events (audit: кто, когда, какая попытка).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from . import broker
|
||||
from .config import settings
|
||||
|
||||
|
||||
def replay_one(pool: ConnectionPool, attachment_id: str, actor: str, dry_run: bool) -> str:
|
||||
with pool.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT j.job_id, j.status, j.attempt, j.error_code, a.staging_key, j.pipeline_version "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||||
"WHERE j.attachment_id = %s AND j.pipeline_version = %s FOR UPDATE OF j",
|
||||
(attachment_id, settings.pipeline_version)).fetchone()
|
||||
if row is None:
|
||||
return "not found"
|
||||
job_id, status, attempt, error_code, staging_key, pv = row
|
||||
if status != "failed":
|
||||
return f"refused: status is {status} (replay only for failed)"
|
||||
if not staging_key:
|
||||
return "refused: source already removed by staging retention"
|
||||
if dry_run:
|
||||
return f"would replay job {job_id} ({error_code}), next attempt {attempt + 1}"
|
||||
nxt = attempt + 1
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, max_attempts = %s, "
|
||||
"error_code = NULL, replays = replays + 1, finished_at = NULL, next_attempt_at = now(), "
|
||||
"updated_at = now() WHERE job_id = %s",
|
||||
(nxt, attempt + settings.job_max_attempts, job_id))
|
||||
conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() "
|
||||
"WHERE id = %s", (attachment_id,))
|
||||
created = conn.execute("SELECT now()").fetchone()[0]
|
||||
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
|
||||
conn.execute("INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
|
||||
"VALUES (%s, %s, %s, now(), now())", (job_id, nxt, json.dumps(payload)))
|
||||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
||||
"VALUES (%s, 'replay', %s, %s, %s)",
|
||||
(job_id, nxt, actor, f"was failed: {error_code}"))
|
||||
return f"replayed job {job_id}: attempt {nxt} queued (outbox → broker)"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="failed jobs registry and explicit replay")
|
||||
g = ap.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument("--list", action="store_true")
|
||||
g.add_argument("--dlq", action="store_true")
|
||||
g.add_argument("--attachment")
|
||||
g.add_argument("--all-failed", action="store_true")
|
||||
ap.add_argument("--error-code")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--actor", default=None)
|
||||
args = ap.parse_args()
|
||||
actor = args.actor or f"operator:{getpass.getuser()}"
|
||||
|
||||
if args.dlq:
|
||||
r = broker.sync_client()
|
||||
for msg_id, fields in r.xrevrange(settings.dlq_stream, count=20):
|
||||
print(msg_id, json.dumps(fields, ensure_ascii=False))
|
||||
return
|
||||
|
||||
with ConnectionPool(settings.database_url, min_size=1, max_size=2, open=True) as pool:
|
||||
if args.list:
|
||||
with pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT j.attachment_id, j.job_id, j.attempt, j.error_code, j.finished_at, "
|
||||
"a.staging_key IS NOT NULL FROM attachment_jobs j JOIN attachments a "
|
||||
"ON a.id = j.attachment_id WHERE j.status = 'failed' ORDER BY j.finished_at").fetchall()
|
||||
for aid, jid, att, code, fin, has_src in rows:
|
||||
print(f"{aid} job={jid} attempt={att} error={code} at={fin:%Y-%m-%d %H:%M:%S} "
|
||||
f"replayable={'yes' if has_src else 'no'}")
|
||||
print(f"failed jobs: {len(rows)}")
|
||||
return
|
||||
targets = [args.attachment] if args.attachment else []
|
||||
if args.all_failed:
|
||||
with pool.connection() as conn:
|
||||
q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'"
|
||||
params: tuple = ()
|
||||
if args.error_code:
|
||||
q += " AND error_code = %s"
|
||||
params = (args.error_code,)
|
||||
targets = [str(r[0]) for r in conn.execute(q, params).fetchall()]
|
||||
for aid in targets:
|
||||
print(aid, "→", replay_one(pool, aid, actor, args.dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+380
@@ -0,0 +1,380 @@
|
||||
"""
|
||||
Доступ к данным в PostgreSQL. Только параметризованные запросы (%s):
|
||||
значения передаются отдельно от текста SQL, инъекция невозможна.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from psycopg.errors import UniqueViolation
|
||||
|
||||
|
||||
def iso(dt: datetime) -> str:
|
||||
"""RFC 3339 UTC с Z — формат контракта."""
|
||||
return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- курсор
|
||||
def encode_cursor(scope: str, created_at: datetime, id_) -> str:
|
||||
raw = json.dumps({"s": scope, "c": iso(created_at), "i": str(id_)}).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, scope: str) -> tuple[datetime, uuid.UUID]:
|
||||
"""(created_at, id) последнего показанного элемента или ValueError → 400."""
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
if data.get("s") != scope:
|
||||
raise ValueError("cursor scope mismatch")
|
||||
return datetime.fromisoformat(data["c"].replace("Z", "+00:00")), uuid.UUID(data["i"])
|
||||
except (ValueError, KeyError, TypeError, AttributeError) as exc:
|
||||
raise ValueError("bad cursor") from exc
|
||||
|
||||
|
||||
def _page(rows: list, limit: int, scope: str, key):
|
||||
"""rows получены с LIMIT limit+1: лишняя строка = есть следующая страница."""
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
next_cursor = encode_cursor(scope, *key(page[-1])) if has_more and page else None
|
||||
return page, next_cursor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- users
|
||||
USER_COLS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def user_out(row) -> dict:
|
||||
return {"id": str(row[0]), "username": row[1], "display_name": row[2], "created_at": iso(row[3])}
|
||||
|
||||
|
||||
class UsernameTaken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def create_user(conn, username: str, display_name: str, password_hash: str) -> dict:
|
||||
try:
|
||||
cur = await conn.execute(
|
||||
f"INSERT INTO users (id, username, display_name, password_hash, created_at) "
|
||||
f"VALUES (%s, %s, %s, %s, %s) RETURNING {USER_COLS}",
|
||||
(uuid.uuid4(), username, display_name, password_hash, now()))
|
||||
except UniqueViolation as exc:
|
||||
# Уникальность username гарантирует БД, а не проверка «есть ли такой»
|
||||
# перед вставкой: две одновременные регистрации не пройдут обе.
|
||||
raise UsernameTaken() from exc
|
||||
return user_out(await cur.fetchone())
|
||||
|
||||
|
||||
async def get_user(conn, user_id) -> dict | None:
|
||||
cur = await conn.execute(f"SELECT {USER_COLS} FROM users WHERE id = %s", (user_id,))
|
||||
row = await cur.fetchone()
|
||||
return user_out(row) if row else None
|
||||
|
||||
|
||||
async def get_credentials(conn, username: str):
|
||||
cur = await conn.execute(
|
||||
f"SELECT {USER_COLS}, password_hash FROM users WHERE username = %s", (username,))
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def update_password_hash(conn, user_id, password_hash: str) -> None:
|
||||
await conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (password_hash, user_id))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- chats
|
||||
CHAT_SELECT = (
|
||||
"SELECT c.id, c.title, c.created_at, "
|
||||
" (SELECT array_agg(m.user_id ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) "
|
||||
"FROM chats c ")
|
||||
|
||||
|
||||
def chat_out(row) -> dict:
|
||||
return {"id": str(row[0]), "title": row[1],
|
||||
"member_ids": [str(x) for x in row[3]], "created_at": iso(row[2])}
|
||||
|
||||
|
||||
async def count_existing_users(conn, ids: list[uuid.UUID]) -> int:
|
||||
cur = await conn.execute("SELECT count(*) FROM users WHERE id = ANY(%s)", (ids,))
|
||||
return (await cur.fetchone())[0]
|
||||
|
||||
|
||||
async def insert_chat(conn, title: str, member_ids: list[uuid.UUID], creator) -> dict:
|
||||
"""Чат + все участники. Вызывать внутри транзакции: либо всё, либо ничего."""
|
||||
chat_id, created = uuid.uuid4(), now()
|
||||
await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_by, created_at) VALUES (%s, %s, %s, %s)",
|
||||
(chat_id, title, creator, created))
|
||||
async with conn.cursor() as cur:
|
||||
await cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
[(chat_id, uid, pos) for pos, uid in enumerate(member_ids)])
|
||||
return {"id": str(chat_id), "title": title,
|
||||
"member_ids": [str(x) for x in member_ids], "created_at": iso(created)}
|
||||
|
||||
|
||||
async def get_chat_for_member(conn, chat_id, user_id) -> dict | None:
|
||||
"""Чат, если он существует И пользователь в нём состоит; иначе None (→ 404)."""
|
||||
cur = await conn.execute(
|
||||
CHAT_SELECT + "WHERE c.id = %s AND EXISTS "
|
||||
"(SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = %s)",
|
||||
(chat_id, user_id))
|
||||
row = await cur.fetchone()
|
||||
return chat_out(row) if row else None
|
||||
|
||||
|
||||
async def list_chats(conn, user_id, limit: int, cursor: str | None):
|
||||
scope = f"chats:{user_id}"
|
||||
params: list = [user_id]
|
||||
where = ""
|
||||
if cursor:
|
||||
c_at, c_id = decode_cursor(cursor, scope)
|
||||
where = "WHERE (c.created_at, c.id) > (%s, %s) "
|
||||
params += [c_at, c_id]
|
||||
cur = await conn.execute(
|
||||
CHAT_SELECT + "JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s "
|
||||
+ where + "ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1))
|
||||
rows = await cur.fetchall()
|
||||
page, nxt = _page(rows, limit, scope, key=lambda r: (r[2], r[0]))
|
||||
return [chat_out(r) for r in page], nxt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- messages
|
||||
MSG_COLS = "id, chat_id, sender_id, text, created_at"
|
||||
# lab4: id вложений сообщения в порядке из запроса (пустой массив, если их нет).
|
||||
MSG_ATTACHMENTS = (
|
||||
"COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) "
|
||||
"FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])")
|
||||
MSG_SELECT = f"SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, {MSG_ATTACHMENTS} FROM messages m "
|
||||
|
||||
|
||||
def message_out(row) -> dict:
|
||||
return {"id": str(row[0]), "chat_id": str(row[1]), "sender_id": str(row[2]),
|
||||
"text": row[3], "created_at": iso(row[4]),
|
||||
"attachment_ids": [str(x) for x in row[5]]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- идемпотентность
|
||||
class IdempotencyConflict(Exception):
|
||||
"""Тот же Idempotency-Key, но другое тело запроса → 409."""
|
||||
|
||||
|
||||
class AttachmentNotFound(Exception):
|
||||
"""Вложения нет или оно из другого чата → 404."""
|
||||
|
||||
|
||||
class AttachmentNotReady(Exception):
|
||||
"""Вложение в состоянии failed → 409 (в lab4 статус всегда ready)."""
|
||||
|
||||
|
||||
def request_hash(text: str, attachment_ids: list | None = None) -> bytes:
|
||||
"""Хэш канонического тела: порядок ключей JSON не влияет на результат.
|
||||
Отсутствующий attachment_ids и [] — одно и то же (контракт), поэтому пустой
|
||||
список в хэш не входит: ключи, сохранённые lab3 для текстовых сообщений,
|
||||
после обновления дают тот же хэш."""
|
||||
body: dict = {"text": text}
|
||||
if attachment_ids:
|
||||
body["attachment_ids"] = [str(x) for x in attachment_ids]
|
||||
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
async def check_attachments(conn, chat_id, attachment_ids: list[uuid.UUID]) -> None:
|
||||
"""Все вложения существуют и принадлежат ЭТОМУ чату; иначе 404 (не раскрываем,
|
||||
что чужое вложение существует). FOR SHARE: строку нельзя удалить до COMMIT."""
|
||||
if not attachment_ids:
|
||||
return
|
||||
cur = await conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s) FOR SHARE",
|
||||
(attachment_ids,))
|
||||
found = {row[0]: row for row in await cur.fetchall()}
|
||||
for aid in attachment_ids:
|
||||
row = found.get(aid)
|
||||
if row is None or str(row[1]) != str(chat_id):
|
||||
raise AttachmentNotFound()
|
||||
if any(row[2] == "failed" for row in found.values()):
|
||||
raise AttachmentNotReady()
|
||||
|
||||
|
||||
async def get_message(conn, message_id) -> dict:
|
||||
cur = await conn.execute(MSG_SELECT + "WHERE m.id = %s", (message_id,))
|
||||
return message_out(await cur.fetchone())
|
||||
|
||||
|
||||
async def send_message_idempotent(conn, chat_id, sender_id, key: str, text: str,
|
||||
ttl_seconds: int,
|
||||
attachment_ids: list[uuid.UUID] | None = None) -> dict:
|
||||
"""Записать сообщение ровно один раз для (пользователь, чат, ключ).
|
||||
Вызывать внутри транзакции: ключ, сообщение и его связи с вложениями
|
||||
фиксируются вместе или никак, поэтому ошибка (5xx) не оставляет «записанного результата».
|
||||
|
||||
Конкурентные повторы через разные реплики: второй INSERT того же PK ждёт
|
||||
COMMIT первого (блокировка уникального индекса в PostgreSQL), затем получает
|
||||
конфликт и читает уже сохранённый результат."""
|
||||
attachment_ids = attachment_ids or []
|
||||
rhash = request_hash(text, attachment_ids)
|
||||
msg_id, created = uuid.uuid4(), now()
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id, created_at) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s) "
|
||||
"ON CONFLICT (user_id, chat_id, idem_key) DO NOTHING RETURNING 1",
|
||||
(sender_id, chat_id, key, rhash, msg_id, created))
|
||||
if await cur.fetchone() is None:
|
||||
cur = await conn.execute(
|
||||
"SELECT request_hash, message_id, created_at < now() - make_interval(secs => %s) "
|
||||
"FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND idem_key = %s FOR UPDATE",
|
||||
(ttl_seconds, sender_id, chat_id, key))
|
||||
stored_hash, stored_msg, expired = await cur.fetchone()
|
||||
if not expired:
|
||||
if bytes(stored_hash) != rhash:
|
||||
raise IdempotencyConflict()
|
||||
return await get_message(conn, stored_msg) # тот же Message, что и в первый раз
|
||||
# Ключ старше TTL: используем его заново как новый.
|
||||
await conn.execute(
|
||||
"UPDATE idempotency_keys SET request_hash = %s, message_id = %s, created_at = %s "
|
||||
"WHERE user_id = %s AND chat_id = %s AND idem_key = %s",
|
||||
(rhash, msg_id, created, sender_id, chat_id, key))
|
||||
# Вложения проверяются в той же транзакции, что и запись сообщения.
|
||||
await check_attachments(conn, chat_id, attachment_ids)
|
||||
await conn.execute(
|
||||
f"INSERT INTO messages ({MSG_COLS}) VALUES (%s, %s, %s, %s, %s)",
|
||||
(msg_id, chat_id, sender_id, text, created))
|
||||
if attachment_ids:
|
||||
async with conn.cursor() as c:
|
||||
await c.executemany(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
[(msg_id, aid, pos) for pos, aid in enumerate(attachment_ids)])
|
||||
return {"id": str(msg_id), "chat_id": str(chat_id), "sender_id": str(sender_id),
|
||||
"text": text, "created_at": iso(created),
|
||||
"attachment_ids": [str(x) for x in attachment_ids]}
|
||||
|
||||
|
||||
async def list_messages(conn, chat_id, limit: int, cursor: str | None):
|
||||
"""Keyset-пагинация по индексу (chat_id, created_at, id): без OFFSET и без скана истории."""
|
||||
scope = f"messages:{chat_id}"
|
||||
params: list = [chat_id]
|
||||
where = ""
|
||||
if cursor:
|
||||
c_at, c_id = decode_cursor(cursor, scope)
|
||||
where = "AND (m.created_at, m.id) > (%s, %s) "
|
||||
params += [c_at, c_id]
|
||||
cur = await conn.execute(
|
||||
MSG_SELECT + "WHERE m.chat_id = %s " + where
|
||||
+ "ORDER BY m.created_at, m.id LIMIT %s", (*params, limit + 1))
|
||||
rows = await cur.fetchall()
|
||||
page, nxt = _page(rows, limit, scope, key=lambda r: (r[4], r[0]))
|
||||
return [message_out(r) for r in page], nxt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- вложения (lab4–5)
|
||||
ATT_COLS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, "
|
||||
"error_code, created_at, updated_at, object_key")
|
||||
VARIANT_ORDER = ("thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
def attachment_out(row, variants: list | None = None) -> dict:
|
||||
"""Публичное представление: без object_key, staging, bucket и внутренних адресов.
|
||||
size_bytes/sha256/content_type всегда описывают ОРИГИНАЛ; у вариантов — свои поля.
|
||||
Варианты показываются только у ready (статус ставится после записи всех вариантов)."""
|
||||
return {"id": str(row[0]), "chat_id": str(row[1]), "owner_id": str(row[2]),
|
||||
"filename": row[3], "content_type": row[4], "size_bytes": row[5],
|
||||
"sha256": row[6], "status": row[7],
|
||||
"variants": (variants or []) if row[7] == "ready" else [],
|
||||
"error_code": row[8] if row[7] == "failed" else None,
|
||||
"created_at": iso(row[9]), "updated_at": iso(row[10])}
|
||||
|
||||
|
||||
async def add_upload_intent(conn, object_key: str, attachment_id) -> None:
|
||||
"""Журнал намерения: запись ДО PUT в S3 (staging). Если процесс упадёт между PUT и
|
||||
COMMIT метаданных, по этой строке janitor найдёт и удалит объект-сироту."""
|
||||
await conn.execute(
|
||||
"INSERT INTO upload_intents (object_key, attachment_id, created_at) VALUES (%s, %s, now())",
|
||||
(object_key, attachment_id))
|
||||
|
||||
|
||||
class IntentLost(Exception):
|
||||
"""Журнал загрузки уже забрал janitor (загрузка шла дольше grace period)."""
|
||||
|
||||
|
||||
async def commit_attachment(conn, *, attachment_id, chat_id, owner_id, filename: str,
|
||||
content_type: str, size_bytes: int, sha256: str,
|
||||
width: int, height: int, object_key: str, staging_key: str,
|
||||
pipeline_version: int, max_attempts: int) -> tuple[dict, int, dict]:
|
||||
"""lab5. Вызывать внутри ОДНОЙ транзакции. Атомарно фиксирует:
|
||||
- снятие журнала загрузки (staging-объект теперь «принадлежит» вложению),
|
||||
- вложение status=queued,
|
||||
- job (queued, attempt=1),
|
||||
- строку outbox с сообщением для брокера (transactional outbox).
|
||||
После COMMIT работа не теряется: оригинал в staging, намерение — в БД.
|
||||
Возвращает (Attachment, id строки outbox, payload сообщения)."""
|
||||
from .broker import make_payload # локальный импорт: repo не зависит от redis
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM upload_intents WHERE object_key = %s RETURNING 1", (staging_key,))
|
||||
if await cur.fetchone() is None:
|
||||
raise IntentLost()
|
||||
ts = now()
|
||||
cur = await conn.execute(
|
||||
f"INSERT INTO attachments ({ATT_COLS}, width, height, staging_key) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, %s, %s, %s, %s, %s) "
|
||||
f"RETURNING {ATT_COLS}",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
ts, ts, object_key, width, height, staging_key))
|
||||
att = attachment_out(await cur.fetchone())
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute(
|
||||
"INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, "
|
||||
"max_attempts, created_at, updated_at) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s)",
|
||||
(job_id, attachment_id, pipeline_version, max_attempts, ts, ts))
|
||||
payload = make_payload(job_id, attachment_id, pipeline_version, 1, ts)
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
|
||||
"VALUES (%s, 1, %s, %s, %s) RETURNING id",
|
||||
(job_id, json.dumps(payload), ts, ts))
|
||||
outbox_id = (await cur.fetchone())[0]
|
||||
await conn.execute(
|
||||
"INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, 'accepted', 1, 'api')",
|
||||
(job_id,))
|
||||
return att, outbox_id, payload
|
||||
|
||||
|
||||
async def mark_published(conn, outbox_id: int, stream_id: str) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE job_outbox SET published_at = now(), stream_id = %s, "
|
||||
"publish_attempts = publish_attempts + 1 WHERE id = %s AND published_at IS NULL",
|
||||
(stream_id, outbox_id))
|
||||
|
||||
|
||||
async def get_variants(conn, attachment_id, pipeline_version: int) -> list[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT variant, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id = %s AND pipeline_version = %s", (attachment_id, pipeline_version))
|
||||
rows = {r[0]: r for r in await cur.fetchall()}
|
||||
return [{"name": r[0], "content_type": r[1], "size_bytes": r[2], "width": r[3],
|
||||
"height": r[4], "sha256": r[5]} for name in VARIANT_ORDER if (r := rows.get(name))]
|
||||
|
||||
|
||||
async def get_variant(conn, attachment_id, pipeline_version: int, name: str):
|
||||
"""(object_key, content_type, size_bytes, sha256) или None."""
|
||||
cur = await conn.execute(
|
||||
"SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id = %s AND pipeline_version = %s AND variant = %s",
|
||||
(attachment_id, pipeline_version, name))
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def drop_upload_intent(conn, object_key: str) -> None:
|
||||
await conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (object_key,))
|
||||
|
||||
|
||||
async def get_attachment_for_member(conn, attachment_id, user_id):
|
||||
"""Строка вложения, если пользователь состоит в его чате; иначе None (→ 404)."""
|
||||
cur = await conn.execute(
|
||||
f"SELECT {ATT_COLS} FROM attachments a WHERE a.id = %s AND EXISTS "
|
||||
"(SELECT 1 FROM chat_members x WHERE x.chat_id = a.chat_id AND x.user_id = %s)",
|
||||
(attachment_id, user_id))
|
||||
return await cur.fetchone()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Пароли и токены.
|
||||
|
||||
Пароль: Argon2id (password KDF). Он специально медленный и требует память,
|
||||
поэтому перебор украденных хэшей дорог. Соль своя у каждого хэша.
|
||||
|
||||
Токен сессии: 32 случайных байта (256 бит) из secrets — непрозрачная строка,
|
||||
не связанная с username/UUID. В БД кладём SHA-256 от токена: для случайного
|
||||
256-битного значения быстрый хэш достаточен (перебирать нечего), а утечка
|
||||
таблицы sessions не даёт рабочих токенов.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
_hasher = PasswordHasher() # Argon2id, параметры по умолчанию библиотеки (RFC 9106 low-memory)
|
||||
|
||||
# Хэш-заглушка: проверяем пароль даже для несуществующего пользователя,
|
||||
# чтобы по времени ответа нельзя было узнать, существует ли username.
|
||||
_DUMMY_HASH = _hasher.hash("dummy-password-for-timing")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str | None, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash or _DUMMY_HASH, password) and password_hash is not None
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def needs_rehash(password_hash: str) -> bool:
|
||||
return _hasher.check_needs_rehash(password_hash)
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32) # 256 бит энтропии
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Сессии: PostgreSQL — источник истины, Redis/Valkey — кэш поиска сессии.
|
||||
|
||||
Чтение (проверка Bearer):
|
||||
1. Ищем в Redis ключ sess:<sha256(token)>. Нашли и не истёк → пользователь.
|
||||
2. Нет в кэше / Redis недоступен → читаем sessions в PostgreSQL
|
||||
(не отозвана и expires_at > now()) и кладём результат в кэш.
|
||||
|
||||
Политика expiry: срок жизни сессии задаёт PostgreSQL (expires_at).
|
||||
TTL ключа в Redis = min(SESSION_CACHE_TTL_SECONDS, время до expires_at),
|
||||
поэтому кэш никогда не продлевает сессию.
|
||||
|
||||
Инвалидация (logout) — атомарно для БД и кэша:
|
||||
в одной транзакции UPDATE sessions SET revoked_at (строка блокируется),
|
||||
затем DEL ключа в Redis. Если Redis не ответил — транзакция откатывается и
|
||||
клиент получает 503: logout НЕ подтверждён, 204 отдаётся только когда
|
||||
копии отозванной сессии в кэше гарантированно нет.
|
||||
Гонка «чтение кладёт старую сессию в кэш после DEL» закрыта блокировкой:
|
||||
чтение из БД идёт с FOR SHARE и кладёт в кэш, не отпуская строку, поэтому
|
||||
logout ждёт окончания такого чтения и удаляет ключ уже после него.
|
||||
|
||||
Отказ Redis: кэш обходится, всё читается из PostgreSQL (медленнее, но верно).
|
||||
Flush/рестарт Redis: кэш пустой → сессии восстанавливаются из PostgreSQL.
|
||||
Отказ PostgreSQL: новые проверки идут в БД → 503, анонимный запрос не принимается.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .config import settings
|
||||
from .security import new_token, token_hash
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
_redis: aioredis.Redis | None = None
|
||||
|
||||
|
||||
def init_redis() -> None:
|
||||
global _redis
|
||||
if settings.redis_url:
|
||||
_redis = aioredis.Redis.from_url(
|
||||
settings.redis_url,
|
||||
socket_timeout=settings.redis_timeout,
|
||||
socket_connect_timeout=settings.redis_timeout,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
async def close_redis() -> None:
|
||||
if _redis is not None:
|
||||
await _redis.aclose()
|
||||
|
||||
|
||||
def redis_enabled() -> bool:
|
||||
return _redis is not None
|
||||
|
||||
|
||||
async def redis_ping() -> bool:
|
||||
if _redis is None:
|
||||
return False
|
||||
try:
|
||||
return bool(await _redis.ping())
|
||||
except (RedisError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _key(th: bytes) -> str:
|
||||
return "sess:" + th.hex()
|
||||
|
||||
|
||||
async def _cache_get(th: bytes) -> dict | None:
|
||||
if _redis is None:
|
||||
return None
|
||||
try:
|
||||
raw = await _redis.get(_key(th))
|
||||
except (RedisError, OSError) as exc:
|
||||
log.warning("redis unavailable on read, bypassing cache: %s", type(exc).__name__)
|
||||
return None
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
|
||||
async def _cache_put(th: bytes, user: dict, expires_at: datetime) -> None:
|
||||
if _redis is None:
|
||||
return
|
||||
left = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
ttl = min(settings.session_cache_ttl_seconds, left)
|
||||
if ttl <= 0:
|
||||
return
|
||||
try:
|
||||
await _redis.set(_key(th), json.dumps({"user": user, "exp": expires_at.isoformat()}), ex=ttl)
|
||||
except (RedisError, OSError) as exc:
|
||||
log.warning("redis unavailable on write, skipping cache: %s", type(exc).__name__)
|
||||
|
||||
|
||||
class CacheUnavailable(Exception):
|
||||
"""Redis не подтвердил удаление ключа — отзыв сессии нельзя гарантировать."""
|
||||
|
||||
|
||||
async def _cache_delete_strict(th: bytes) -> None:
|
||||
"""Удалить копию сессии из кэша или поднять CacheUnavailable.
|
||||
Молча проглатывать ошибку нельзя: если Redis «приостановлен», ключ
|
||||
переживёт сбой и после восстановления отозванный токен снова примут."""
|
||||
if _redis is None:
|
||||
return
|
||||
try:
|
||||
await _redis.delete(_key(th))
|
||||
except (RedisError, OSError) as exc:
|
||||
log.warning("redis unavailable on invalidate: %s", type(exc).__name__)
|
||||
raise CacheUnavailable() from exc
|
||||
|
||||
|
||||
async def create_session(conn, user_id) -> tuple[str, datetime]:
|
||||
token = new_token()
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + timedelta(seconds=settings.session_ttl_seconds)
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at) "
|
||||
"VALUES (%s, %s, %s, %s)",
|
||||
(token_hash(token), user_id, now, expires_at))
|
||||
return token, expires_at
|
||||
|
||||
|
||||
async def resolve(pool, token: str, load_user) -> dict | None:
|
||||
"""Вернёт пользователя (dict) для действующего токена или None."""
|
||||
th = token_hash(token)
|
||||
cached = await _cache_get(th)
|
||||
if cached is not None:
|
||||
if datetime.fromisoformat(cached["exp"]) > datetime.now(timezone.utc):
|
||||
return cached["user"]
|
||||
return None
|
||||
async with pool.connection() as conn:
|
||||
# FOR SHARE: пока эта транзакция не закончилась (вместе с записью в кэш),
|
||||
# logout не сможет отозвать строку — он дождётся и удалит ключ после нас.
|
||||
cur = await conn.execute(
|
||||
"SELECT user_id, expires_at FROM sessions "
|
||||
"WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > now() "
|
||||
"FOR SHARE",
|
||||
(th,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = await load_user(conn, row[0])
|
||||
if user is None:
|
||||
return None
|
||||
await _cache_put(th, user, row[1])
|
||||
return user
|
||||
|
||||
|
||||
async def revoke(pool, token: str) -> bool:
|
||||
"""Отозвать сессию. True — была действующая и теперь отозвана (и в БД, и в кэше).
|
||||
False — сессии нет/уже отозвана. CacheUnavailable — Redis не ответил:
|
||||
транзакция откатана, сессия осталась действующей, logout не подтверждён."""
|
||||
th = token_hash(token)
|
||||
async with pool.connection() as conn:
|
||||
async with conn.transaction():
|
||||
cur = await conn.execute(
|
||||
"UPDATE sessions SET revoked_at = now() "
|
||||
"WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > now() "
|
||||
"RETURNING 1", (th,))
|
||||
if await cur.fetchone() is None:
|
||||
return False
|
||||
# Внутри транзакции: ошибка Redis → исключение → ROLLBACK отзыва.
|
||||
await _cache_delete_strict(th)
|
||||
return True
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
"""
|
||||
Объектное хранилище (S3-совместимое, MinIO) — только БАЙТЫ вложений.
|
||||
|
||||
- Метаданные (кто, какой чат, sha256, размер, тип) — в PostgreSQL, источник истины.
|
||||
- Ключ объекта назначает сервер: originals/<uuid>. Имя файла от клиента
|
||||
в ключ не попадает (нет path traversal и подбора чужих ключей по имени).
|
||||
- Bucket приватный, API ходит отдельными credentials с правами только на
|
||||
этот bucket (см. infra/minio/). Root MinIO у API нет.
|
||||
- boto3 синхронный → все вызовы идут в пуле потоков (не блокируют event loop).
|
||||
- Таймауты и ограниченные повторы: недоступный S3 даёт 503, а не зависание.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import (BotoCoreError, ClientError, ConnectionClosedError,
|
||||
ConnectTimeoutError, EndpointConnectionError)
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("messenger.storage")
|
||||
|
||||
ORIGINALS_PREFIX = "originals/"
|
||||
STAGING_PREFIX = "staging/"
|
||||
VARIANTS_PREFIX = "variants/"
|
||||
_client = None
|
||||
|
||||
# Предохранитель (circuit breaker) реплики: если S3 не отвечает на уровне соединения
|
||||
# (контейнер остановлен, имя не резолвится), следующие S3_BREAKER_SECONDS секунд
|
||||
# запросы к S3 сразу получают 503, не тратя время на DNS/connect и повторы.
|
||||
_CONNECTION_ERRORS = (EndpointConnectionError, ConnectTimeoutError, ConnectionClosedError)
|
||||
_down_until = 0.0
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 недоступен/не ответил/отказал → 503."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Объекта нет (NoSuchKey)."""
|
||||
|
||||
|
||||
def object_key(attachment_id) -> str:
|
||||
"""Конечный ключ оригинала. С lab5 объект там появляется, когда worker его опубликует."""
|
||||
return f"{ORIGINALS_PREFIX}{attachment_id}"
|
||||
|
||||
|
||||
def staging_key(attachment_id) -> str:
|
||||
"""lab5: приватный prefix приёма. До 202 оригинал целиком лежит здесь."""
|
||||
return f"{STAGING_PREFIX}{attachment_id}"
|
||||
|
||||
|
||||
def variant_key(attachment_id, pipeline_version: int, variant: str, ext: str) -> str:
|
||||
"""Детерминированный ключ: повтор job перезаписывает тот же объект теми же байтами,
|
||||
второго набора вариантов не появляется."""
|
||||
return f"{VARIANTS_PREFIX}{attachment_id}/v{pipeline_version}/{variant}.{ext}"
|
||||
|
||||
|
||||
def make_client():
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id or None,
|
||||
aws_secret_access_key=settings.s3_secret_access_key or None,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"}, # MinIO: http://s3:9000/<bucket>/<key>
|
||||
connect_timeout=settings.s3_connect_timeout,
|
||||
read_timeout=settings.s3_read_timeout,
|
||||
retries={"max_attempts": settings.s3_max_attempts, "mode": "standard"},
|
||||
max_pool_connections=max(10, settings.upload_max_concurrent),
|
||||
# Контрольную сумму передаём сами (Content-MD5); не включаем
|
||||
# aws-chunked/CRC по умолчанию новых botocore — совместимость с MinIO.
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def init() -> None:
|
||||
global _client
|
||||
_client = make_client()
|
||||
|
||||
|
||||
def client():
|
||||
if _client is None:
|
||||
init()
|
||||
return _client
|
||||
|
||||
|
||||
def _why(exc: Exception) -> str:
|
||||
"""Тип и код ошибки S3 для лога (без ключей, подписей и адресов)."""
|
||||
if isinstance(exc, ClientError):
|
||||
return f"{type(exc).__name__}:{exc.response.get('Error', {}).get('Code', '?')}"
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def _guard() -> None:
|
||||
if time.monotonic() < _down_until:
|
||||
raise StorageUnavailable("circuit open")
|
||||
|
||||
|
||||
def ensure_available() -> None:
|
||||
"""Быстрый отказ ДО записи журнала загрузки, пока предохранитель открыт."""
|
||||
_guard()
|
||||
|
||||
|
||||
def _trip(exc: Exception) -> None:
|
||||
global _down_until
|
||||
if isinstance(exc, _CONNECTION_ERRORS):
|
||||
_down_until = time.monotonic() + settings.s3_breaker_seconds
|
||||
|
||||
|
||||
def _is_missing(exc: ClientError) -> bool:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
return code in ("NoSuchKey", "404", "NotFound")
|
||||
|
||||
|
||||
# ----------------------------------------------------------- синхронные вызовы
|
||||
def put_sync(key: str, body: BinaryIO, size: int, content_type: str,
|
||||
md5_b64: str, sha256_hex: str) -> None:
|
||||
_guard()
|
||||
body.seek(0)
|
||||
try:
|
||||
client().put_object(
|
||||
Bucket=settings.s3_bucket, Key=key, Body=body, ContentLength=size,
|
||||
ContentType=content_type,
|
||||
ContentMD5=md5_b64, # S3 сверит и отклонит повреждённую передачу
|
||||
Metadata={"sha256": sha256_hex},
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
_trip(exc)
|
||||
log.warning("s3 put failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
|
||||
|
||||
def open_sync(key: str):
|
||||
"""(поток байтов, длина). Поток читается кусками — файл не грузится в RAM целиком."""
|
||||
_guard()
|
||||
try:
|
||||
obj = client().get_object(Bucket=settings.s3_bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
raise ObjectMissing() from exc
|
||||
log.warning("s3 get failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
except BotoCoreError as exc:
|
||||
_trip(exc)
|
||||
log.warning("s3 get failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
return obj["Body"], obj["ContentLength"]
|
||||
|
||||
|
||||
def iter_body(body, chunk_size: int = 64 * 1024) -> Iterator[bytes]:
|
||||
try:
|
||||
yield from body.iter_chunks(chunk_size)
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
|
||||
def delete_sync(key: str) -> None:
|
||||
"""Идемпотентно: удаление отсутствующего объекта в S3 — тоже успех."""
|
||||
_guard()
|
||||
try:
|
||||
client().delete_object(Bucket=settings.s3_bucket, Key=key)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
_trip(exc)
|
||||
log.warning("s3 delete failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
|
||||
|
||||
def exists_sync(key: str) -> bool:
|
||||
try:
|
||||
client().head_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
return False
|
||||
raise StorageUnavailable() from exc
|
||||
except BotoCoreError as exc:
|
||||
raise StorageUnavailable() from exc
|
||||
|
||||
|
||||
def list_sync(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]:
|
||||
"""(key, LastModified, Size) всех объектов под prefix — постранично, без загрузки всего списка."""
|
||||
try:
|
||||
pages = client().get_paginator("list_objects_v2").paginate(
|
||||
Bucket=settings.s3_bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000})
|
||||
for page in pages:
|
||||
for obj in page.get("Contents", []):
|
||||
yield obj["Key"], obj["LastModified"], obj["Size"]
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise StorageUnavailable() from exc
|
||||
|
||||
|
||||
def head_sync(key: str) -> dict | None:
|
||||
"""{size, sha256} объекта или None, если его нет."""
|
||||
_guard()
|
||||
try:
|
||||
h = client().head_object(Bucket=settings.s3_bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
return None
|
||||
raise StorageUnavailable() from exc
|
||||
except BotoCoreError as exc:
|
||||
_trip(exc)
|
||||
raise StorageUnavailable() from exc
|
||||
return {"size": h["ContentLength"], "sha256": h.get("Metadata", {}).get("sha256")}
|
||||
|
||||
|
||||
def download_sync(key: str, fileobj, limit: int) -> int:
|
||||
"""Скачать объект в файл кусками; больше limit байт — ошибка (защита worker)."""
|
||||
body, length = open_sync(key)
|
||||
if length > limit:
|
||||
body.close()
|
||||
raise ValueError("object larger than limit")
|
||||
total = 0
|
||||
for chunk in iter_body(body, 256 * 1024):
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise ValueError("object larger than limit")
|
||||
fileobj.write(chunk)
|
||||
return total
|
||||
|
||||
|
||||
def copy_sync(src_key: str, dst_key: str) -> None:
|
||||
"""Серверное копирование внутри bucket (staging → originals). Идемпотентно."""
|
||||
_guard()
|
||||
try:
|
||||
client().copy_object(Bucket=settings.s3_bucket, Key=dst_key,
|
||||
CopySource={"Bucket": settings.s3_bucket, "Key": src_key},
|
||||
MetadataDirective="COPY")
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
raise ObjectMissing() from exc
|
||||
log.warning("s3 copy failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
except BotoCoreError as exc:
|
||||
_trip(exc)
|
||||
log.warning("s3 copy failed: %s", _why(exc))
|
||||
raise StorageUnavailable() from exc
|
||||
|
||||
|
||||
def put_file_sync(key: str, path: str, content_type: str, sha256_hex: str) -> None:
|
||||
"""Загрузить небольшой файл варианта (≤ нескольких МиБ) с проверкой Content-MD5."""
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
md5_b64 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode()
|
||||
put_sync(key, io.BytesIO(data), len(data), content_type, md5_b64, sha256_hex)
|
||||
|
||||
|
||||
def ping_sync() -> bool:
|
||||
try:
|
||||
client().head_bucket(Bucket=settings.s3_bucket)
|
||||
return True
|
||||
except (BotoCoreError, ClientError):
|
||||
return False
|
||||
|
||||
|
||||
# ----------------------------------------------------------- async-обёртки
|
||||
async def put(key, body, size, content_type, md5_b64, sha256_hex) -> None:
|
||||
await run_in_threadpool(put_sync, key, body, size, content_type, md5_b64, sha256_hex)
|
||||
|
||||
|
||||
async def open_object(key: str):
|
||||
return await run_in_threadpool(open_sync, key)
|
||||
|
||||
|
||||
async def delete(key: str) -> None:
|
||||
await run_in_threadpool(delete_sync, key)
|
||||
|
||||
|
||||
async def ping() -> bool:
|
||||
return await run_in_threadpool(ping_sync)
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
Служебные проверки хранилища для тестов и защиты (не HTTP API).
|
||||
Запуск внутри контейнера api (он в сети storage):
|
||||
|
||||
python -m app.tools private-check # bucket приватный, права API ограничены
|
||||
python -m app.tools stats # число объектов / attachments / upload_intents
|
||||
python -m app.tools state <attachment_id> # объекты staging/originals, метаданные, журнал
|
||||
python -m app.tools job <attachment_id> # lab5: job, попытки, события, варианты, объекты S3
|
||||
python -m app.tools republish <attachment_id> # lab5: отправить ДУБЛЬ сообщения job в брокер
|
||||
python -m app.tools queue # lab5: длина stream, pending, DLQ, outbox
|
||||
python -m app.tools chat-attachments <chat_id> # lab5: вложения чата и их статусы (для тестов)
|
||||
python -m app.tools limits-selftest # lab5: лимиты памяти/времени/пикселей worker
|
||||
python -m app.tools find-local <sha256> # нет ли на диске реплики копии файла
|
||||
|
||||
Секреты не печатаются. Ключи объектов печатаются только для служебного prefix.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from . import storage
|
||||
from .config import settings
|
||||
|
||||
|
||||
def _anon(method: str, path: str, data: bytes | None = None) -> int:
|
||||
"""HTTP-запрос к MinIO БЕЗ подписи (как посторонний клиент)."""
|
||||
req = urllib.request.Request(settings.s3_endpoint.rstrip("/") + path, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as r:
|
||||
return r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code
|
||||
|
||||
|
||||
def _denied(fn) -> str:
|
||||
try:
|
||||
fn()
|
||||
return "ALLOWED"
|
||||
except ClientError as exc:
|
||||
return exc.response.get("Error", {}).get("Code", "error")
|
||||
|
||||
|
||||
def private_check() -> int:
|
||||
s3 = storage.client()
|
||||
bucket = settings.s3_bucket
|
||||
probe = f"{storage.ORIGINALS_PREFIX}privacy-probe-{uuid.uuid4()}"
|
||||
results: list[tuple[str, str, bool]] = []
|
||||
|
||||
def check(name: str, observed, ok: bool) -> None:
|
||||
results.append((name, str(observed), ok))
|
||||
|
||||
s3.put_object(Bucket=bucket, Key=probe, Body=b"probe", ContentType="text/plain")
|
||||
try:
|
||||
# Позитивный контроль: объект существует и API-пользователь его читает.
|
||||
body = s3.get_object(Bucket=bucket, Key=probe)["Body"].read()
|
||||
check("API credentials: GET own object", "200", body == b"probe")
|
||||
# Аноним: ни чтения, ни листинга, ни записи.
|
||||
st = _anon("GET", f"/{bucket}/{probe}")
|
||||
check("anonymous GET object", st, st == 403)
|
||||
st = _anon("GET", f"/{bucket}?list-type=2")
|
||||
check("anonymous LIST bucket", st, st == 403)
|
||||
st = _anon("PUT", f"/{bucket}/{storage.ORIGINALS_PREFIX}anon-{uuid.uuid4()}", b"x")
|
||||
check("anonymous PUT object", st, st == 403)
|
||||
# Ограниченные права API-пользователя: только объекты своего bucket.
|
||||
# MinIO без s3:ListAllMyBuckets либо отказывает, либо показывает только
|
||||
# бакеты, на которые есть права, — чужие бакеты API-пользователь не видит.
|
||||
try:
|
||||
names = sorted(b["Name"] for b in s3.list_buckets().get("Buckets", []))
|
||||
r = "only " + ",".join(names) if names else "empty"
|
||||
ok = set(names) <= {bucket}
|
||||
except ClientError as exc:
|
||||
r = exc.response.get("Error", {}).get("Code", "error")
|
||||
ok = r == "AccessDenied"
|
||||
check("API credentials: ListBuckets (no foreign buckets)", r, ok)
|
||||
r = _denied(lambda: s3.create_bucket(Bucket="lab4-forbidden-" + uuid.uuid4().hex[:8]))
|
||||
check("API credentials: CreateBucket", r, r == "AccessDenied")
|
||||
public = json.dumps({"Version": "2012-10-17", "Statement": [{
|
||||
"Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
|
||||
r = _denied(lambda: s3.put_bucket_policy(Bucket=bucket, Policy=public))
|
||||
check("API credentials: PutBucketPolicy (make public)", r, r == "AccessDenied")
|
||||
r = _denied(lambda: s3.delete_bucket(Bucket=bucket))
|
||||
check("API credentials: DeleteBucket", r, r == "AccessDenied")
|
||||
finally:
|
||||
s3.delete_object(Bucket=bucket, Key=probe)
|
||||
|
||||
width = max(len(n) for n, _, _ in results)
|
||||
for name, observed, ok in results:
|
||||
print(f" {'ok ' if ok else 'FAIL'} {name.ljust(width)} -> {observed}")
|
||||
failed = sum(1 for _, _, ok in results if not ok)
|
||||
print(f"\n{len(results) - failed} passed, {failed} failed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def _db():
|
||||
return psycopg.connect(settings.database_url, connect_timeout=3)
|
||||
|
||||
|
||||
def stats() -> int:
|
||||
# Оригиналы и staging (lab5): сирота может остаться в любом из двух prefix.
|
||||
objects = sum(1 for _ in storage.list_sync(storage.ORIGINALS_PREFIX)) \
|
||||
+ sum(1 for _ in storage.list_sync(storage.STAGING_PREFIX))
|
||||
with _db() as conn:
|
||||
attachments = conn.execute("SELECT count(*) FROM attachments").fetchone()[0]
|
||||
intents = conn.execute("SELECT count(*) FROM upload_intents").fetchone()[0]
|
||||
print(json.dumps({"objects": objects, "attachments": attachments, "upload_intents": intents}))
|
||||
return 0
|
||||
|
||||
|
||||
def state(attachment_id: str) -> int:
|
||||
aid = uuid.UUID(attachment_id)
|
||||
original, staging = storage.object_key(aid), storage.staging_key(aid)
|
||||
with _db() as conn:
|
||||
att = conn.execute("SELECT status FROM attachments WHERE id = %s", (aid,)).fetchone()
|
||||
intent = conn.execute("SELECT 1 FROM upload_intents WHERE attachment_id = %s", (aid,)).fetchone()
|
||||
print(json.dumps({"staging_object_exists": storage.exists_sync(staging),
|
||||
"original_object_exists": storage.exists_sync(original),
|
||||
"attachment_row": att is not None,
|
||||
"attachment_status": att[0] if att else None,
|
||||
"upload_intent": intent is not None}))
|
||||
return 0
|
||||
|
||||
|
||||
def job(attachment_id: str) -> int:
|
||||
"""Полное состояние обработки одного вложения (без секретов)."""
|
||||
aid = uuid.UUID(attachment_id)
|
||||
with _db() as conn:
|
||||
j = conn.execute(
|
||||
"SELECT job_id, status, attempt, max_attempts, error_code, last_error, replays, "
|
||||
"lease_owner, lease_until FROM attachment_jobs WHERE attachment_id = %s", (aid,)).fetchone()
|
||||
a = conn.execute("SELECT status, error_code, staging_key IS NOT NULL FROM attachments "
|
||||
"WHERE id = %s", (aid,)).fetchone()
|
||||
variants = conn.execute(
|
||||
"SELECT variant, width, height, size_bytes, object_key FROM attachment_variants "
|
||||
"WHERE attachment_id = %s ORDER BY variant", (aid,)).fetchall()
|
||||
events = conn.execute(
|
||||
"SELECT to_char(at, 'HH24:MI:SS.MS'), event, attempt, actor, coalesce(detail, '') "
|
||||
"FROM job_events e WHERE job_id = %s ORDER BY id", (j[0],)).fetchall() if j else []
|
||||
outbox = conn.execute("SELECT attempt, published_at IS NOT NULL FROM job_outbox WHERE job_id = %s "
|
||||
"ORDER BY id", (j[0],)).fetchall() if j else []
|
||||
objs = sorted(k for k, _, _ in storage.list_sync(f"{storage.VARIANTS_PREFIX}{aid}/"))
|
||||
print(json.dumps({
|
||||
"attachment": {"status": a[0], "error_code": a[1], "staging_present": a[2]} if a else None,
|
||||
"job": {"job_id": str(j[0]), "status": j[1], "attempt": j[2], "max_attempts": j[3],
|
||||
"error_code": j[4], "last_error": j[5], "replays": j[6], "lease_owner": j[7],
|
||||
"lease_until": j[8].isoformat() if j[8] else None} if j else None,
|
||||
"variant_rows": [{"name": v[0], "size": f"{v[1]}x{v[2]}", "bytes": v[3]} for v in variants],
|
||||
"variant_objects_in_s3": len(objs),
|
||||
"original_object_exists": storage.exists_sync(storage.object_key(aid)),
|
||||
"staging_object_exists": storage.exists_sync(storage.staging_key(aid)),
|
||||
"outbox": [{"attempt": o[0], "published": o[1]} for o in outbox],
|
||||
"events": [" ".join(str(x) for x in e if x not in (None, "")) for e in events],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def republish(attachment_id: str) -> int:
|
||||
"""Отправить в брокер копию сообщения ТЕКУЩЕЙ попытки job (проверка повторной доставки)."""
|
||||
from . import broker
|
||||
aid = uuid.UUID(attachment_id)
|
||||
with _db() as conn:
|
||||
j = conn.execute("SELECT job_id, pipeline_version, attempt, created_at FROM attachment_jobs "
|
||||
"WHERE attachment_id = %s", (aid,)).fetchone()
|
||||
if j is None:
|
||||
print("job not found")
|
||||
return 1
|
||||
payload = broker.make_payload(j[0], aid, j[1], j[2], j[3])
|
||||
print("published duplicate:", broker.publish_sync(broker.sync_client(), payload), json.dumps(payload))
|
||||
return 0
|
||||
|
||||
|
||||
def chat_attachments(chat_id: str) -> int:
|
||||
with _db() as conn:
|
||||
rows = conn.execute("SELECT id, status FROM attachments WHERE chat_id = %s ORDER BY created_at",
|
||||
(uuid.UUID(chat_id),)).fetchall()
|
||||
print(json.dumps([{"id": str(r[0]), "status": r[1]} for r in rows]))
|
||||
return 0
|
||||
|
||||
|
||||
def limits_selftest() -> int:
|
||||
"""Защита worker от «тяжёлых» изображений проверяется на настоящем дочернем процессе
|
||||
(тот же код и те же лимиты, что в worker), без API — API такие файлы и так отсекает."""
|
||||
import multiprocessing
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from . import imaging
|
||||
ctx = multiprocessing.get_context("forkserver")
|
||||
results = []
|
||||
|
||||
def run(name, src, ctype, max_pixels, memory_mb, timeout, slow=0.0, expect=""):
|
||||
d = Path(tempfile.mkdtemp())
|
||||
res = d / "r.json"
|
||||
(d / "out").mkdir()
|
||||
t0 = time.monotonic()
|
||||
p = ctx.Process(target=imaging.child_main, args=(str(src), str(d / "out"), ctype, max_pixels,
|
||||
memory_mb, int(timeout) + 5, str(res), slow))
|
||||
p.start()
|
||||
p.join(timeout)
|
||||
killed = p.is_alive()
|
||||
if killed:
|
||||
p.kill()
|
||||
p.join()
|
||||
got = "timeout(killed)" if killed else (json.loads(res.read_text()) if res.exists() else f"exit {p.exitcode}")
|
||||
outcome = got if isinstance(got, str) else ("ok" if got["ok"] else f'{got["code"]}: {got["detail"]}')
|
||||
ok = expect in outcome
|
||||
results.append(ok)
|
||||
print(f" {'ok ' if ok else 'FAIL'} {name}: {outcome} ({time.monotonic() - t0:.1f}s)")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
big = Path(tmp) / "big.png" # 6000×6000 = 36 Мп: в пределах 40 Мп, но ~100+ МБ после распаковки
|
||||
Image.new("RGB", (6000, 6000), (200, 30, 30)).save(big, optimize=False, compress_level=1)
|
||||
small = Path(tmp) / "small.png"
|
||||
Image.new("RGBA", (320, 200), (0, 0, 255, 128)).save(small)
|
||||
corrupt = Path(tmp) / "corrupt.png"
|
||||
corrupt.write_bytes(small.read_bytes()[:200])
|
||||
run("normal 320x200 with alpha", small, "image/png", 40_000_000, 768, 30, expect="ok")
|
||||
run("pixel limit (36 Mp > limit 10 Mp)", big, "image/png", 10_000_000, 768, 30, expect="invalid_image: pixel limit")
|
||||
run("memory limit (36 Mp, RLIMIT_AS 120 MiB)", big, "image/png", 40_000_000, 120, 30, expect="invalid_image")
|
||||
run("corrupt/truncated PNG", corrupt, "image/png", 40_000_000, 768, 30, expect="invalid_image")
|
||||
run("time limit (hangs 10 s, timeout 2 s)", small, "image/png", 40_000_000, 768, 2, slow=10, expect="timeout")
|
||||
failed = results.count(False)
|
||||
print(f"\n{len(results) - failed} passed, {failed} failed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def queue() -> int:
|
||||
from . import broker
|
||||
r = broker.sync_client()
|
||||
try:
|
||||
groups = r.xinfo_groups(settings.jobs_stream)
|
||||
except Exception: # noqa: BLE001
|
||||
groups = []
|
||||
with _db() as conn:
|
||||
pending_outbox = conn.execute("SELECT count(*) FROM job_outbox WHERE published_at IS NULL").fetchone()[0]
|
||||
by_status = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall())
|
||||
print(json.dumps({
|
||||
"stream_length": r.xlen(settings.jobs_stream) if r.exists(settings.jobs_stream) else 0,
|
||||
"groups": [{"name": g["name"], "consumers": g["consumers"], "pending": g["pending"],
|
||||
"lag": g.get("lag")} for g in groups],
|
||||
"dlq_length": r.xlen(settings.dlq_stream) if r.exists(settings.dlq_stream) else 0,
|
||||
"outbox_unpublished": pending_outbox,
|
||||
"jobs_by_status": by_status,
|
||||
}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def find_local(digest: str) -> int:
|
||||
"""Обойти файловую систему контейнера API и найти файлы с таким SHA-256.
|
||||
Доказательство, что оригинал не хранится на реплике (только в S3)."""
|
||||
import hashlib
|
||||
import os
|
||||
skip = ("/proc", "/sys", "/dev", "/usr", "/lib", "/bin", "/sbin", "/etc", "/var/lib/dpkg")
|
||||
scanned, found = 0, []
|
||||
for root, dirs, files in os.walk("/"):
|
||||
if root.startswith(skip):
|
||||
dirs[:] = []
|
||||
continue
|
||||
for name in files:
|
||||
path = os.path.join(root, name)
|
||||
try:
|
||||
if not os.path.isfile(path) or os.path.getsize(path) > 32 * 1024 * 1024:
|
||||
continue
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
scanned += 1
|
||||
if h.hexdigest() == digest:
|
||||
found.append(path)
|
||||
except OSError:
|
||||
continue
|
||||
print(json.dumps({"files_scanned": scanned, "copies_found": found}))
|
||||
return 1 if found else 0
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if not argv:
|
||||
print(__doc__)
|
||||
return 2
|
||||
cmd, args = argv[0], argv[1:]
|
||||
if cmd == "private-check":
|
||||
return private_check()
|
||||
if cmd == "stats":
|
||||
return stats()
|
||||
if cmd == "state" and len(args) == 1:
|
||||
return state(args[0])
|
||||
if cmd == "job" and len(args) == 1:
|
||||
return job(args[0])
|
||||
if cmd == "republish" and len(args) == 1:
|
||||
return republish(args[0])
|
||||
if cmd == "queue":
|
||||
return queue()
|
||||
if cmd == "chat-attachments" and len(args) == 1:
|
||||
return chat_attachments(args[0])
|
||||
if cmd == "limits-selftest":
|
||||
return limits_selftest()
|
||||
if cmd == "find-local" and len(args) == 1:
|
||||
return find_local(args[0])
|
||||
print(__doc__)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
Приём тела загрузки (lab4): потоково, с лимитами, без накопления файла в RAM.
|
||||
|
||||
- Content-Length больше лимита → 413 сразу, тело не читается.
|
||||
- Тело читается кусками (как пришли из сокета) и одновременно:
|
||||
считается SHA-256 и MD5, растёт счётчик байт. Как только байт больше лимита —
|
||||
413, чтение прекращается (работает и для chunked без Content-Length).
|
||||
- Байты копятся в SpooledTemporaryFile: первые UPLOAD_SPOOL_MEMORY_BYTES в памяти,
|
||||
дальше — во временном файле. Это буфер одного запроса, а не хранилище:
|
||||
файл удаляется при закрытии (в finally), оригинал живёт только в S3.
|
||||
- Общий таймаут на всё тело: «медленный» клиент не держит слот вечно.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from starlette.requests import ClientDisconnect, Request
|
||||
|
||||
ALLOWED_TYPES = ("image/png", "image/jpeg")
|
||||
# Управляющие символы направления текста: «photognp.exe» выглядит как «photoexe.png».
|
||||
_BIDI_CONTROLS = set("")
|
||||
|
||||
|
||||
class UploadError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def validate_filename(name: str | None) -> str:
|
||||
"""filename — только отображаемое имя. Отклоняем всё, что похоже на путь
|
||||
или прячет настоящий вид имени. Никаких «исправлений»: либо как есть, либо 400."""
|
||||
if name is None or not (1 <= len(name) <= 255) or not name.strip():
|
||||
raise UploadError(400, "invalid_request", "filename must be 1..255 characters")
|
||||
if name in (".", ".."):
|
||||
raise UploadError(400, "invalid_request", "filename is not allowed")
|
||||
for ch in name:
|
||||
if ch in "/\\" or ch == "\x00" or ch in _BIDI_CONTROLS \
|
||||
or unicodedata.category(ch) in ("Cc", "Cs"):
|
||||
raise UploadError(400, "invalid_request", "filename contains forbidden characters")
|
||||
return name
|
||||
|
||||
|
||||
def declared_type(request: Request) -> str:
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media not in ALLOWED_TYPES:
|
||||
raise UploadError(415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
|
||||
return media
|
||||
|
||||
|
||||
def check_declared_length(request: Request, limit: int) -> int | None:
|
||||
raw = request.headers.get("content-length")
|
||||
if raw is None:
|
||||
return None
|
||||
if not raw.isdigit():
|
||||
raise UploadError(400, "invalid_request", "bad Content-Length")
|
||||
length = int(raw)
|
||||
if length > limit:
|
||||
# Отказ до чтения тела: ни байта сверх заголовков не принимаем.
|
||||
raise UploadError(413, "payload_too_large", f"file is larger than {limit} bytes")
|
||||
return length
|
||||
|
||||
|
||||
@dataclass
|
||||
class Received:
|
||||
file: BinaryIO
|
||||
size: int
|
||||
sha256_hex: str
|
||||
md5_b64: str
|
||||
|
||||
|
||||
async def receive(request: Request, limit: int, timeout: float, spool_memory: int) -> Received:
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=spool_memory, prefix="upload-")
|
||||
sha, md5 = hashlib.sha256(), hashlib.md5(usedforsecurity=False)
|
||||
size = 0
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise UploadError(413, "payload_too_large", f"file is larger than {limit} bytes")
|
||||
sha.update(chunk)
|
||||
md5.update(chunk)
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
spool.close()
|
||||
raise UploadError(400, "invalid_request", "request body was not received in time") from None
|
||||
except ClientDisconnect:
|
||||
# Прерванная загрузка: в S3 ничего не записано, буфер удаляется здесь.
|
||||
spool.close()
|
||||
raise UploadError(400, "invalid_request", "upload was interrupted") from None
|
||||
except BaseException:
|
||||
spool.close()
|
||||
raise
|
||||
if size == 0:
|
||||
spool.close()
|
||||
raise UploadError(400, "invalid_request", "request body is empty")
|
||||
spool.seek(0)
|
||||
return Received(spool, size, sha.hexdigest(), base64.b64encode(md5.digest()).decode())
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
"""
|
||||
Worker фоновой обработки вложений (lab5). Отдельный процесс/контейнер: `python -m app.worker`.
|
||||
|
||||
Поток одной задачи (сообщение брокера = только ID, см. app/broker.py):
|
||||
|
||||
XREADGROUP ─► claim в БД (FOR UPDATE): queued → processing, lease_token, lease_until
|
||||
─► скачать оригинал из staging, сверить sha256 с БД (доверенные метаданные)
|
||||
─► дочерний процесс: thumbnail / crop / optimized (лимиты памяти и времени)
|
||||
─► S3: copy staging → originals/<id>, PUT variants/<id>/v1/* (детерминированные ключи)
|
||||
─► COMMIT результата (только владелец аренды): variants ON CONFLICT DO NOTHING,
|
||||
attachment и job → ready
|
||||
─► XACK (+XDEL) ─► удалить staging (после безопасной фиксации)
|
||||
|
||||
Ошибки:
|
||||
постоянные (битое изображение, sha256 не совпал, исходника нет) → failed + error_code;
|
||||
временные (S3/БД недоступны, таймаут обработки) → queued, attempt+1, backoff с jitter
|
||||
через outbox (available_at в будущем); попытки кончились → failed retry_exhausted.
|
||||
|
||||
Отказы worker:
|
||||
умер до XACK → сообщение остаётся в PEL брокера; после рестарта worker читает свой PEL,
|
||||
чужой брошенный PEL забирает XAUTOCLAIM (CLAIM_IDLE_SECONDS);
|
||||
завис/умер с арендой → аренду никто не продлевает, после истечения задачу
|
||||
забирает другой (lease takeover) или reaper переводит её в queued с новой попыткой.
|
||||
Устаревший worker после пробуждения ничего не перезапишет: COMMIT проверяет lease_token.
|
||||
|
||||
Поток maintenance (в каждом worker, безопасно параллельно — FOR UPDATE SKIP LOCKED):
|
||||
outbox relay, reaper просроченных аренд, очистка staging, heartbeat-файл для healthcheck.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import signal
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
import redis
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from . import broker, imaging, storage
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail
|
||||
|
||||
|
||||
class LeaseLost(Exception):
|
||||
"""Аренду забрали (истекла и задачу взял другой) — результат фиксировать нельзя."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Job:
|
||||
job_id: str
|
||||
attachment_id: str
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
token: str
|
||||
staging_key: str | None
|
||||
object_key: str
|
||||
sha256: str
|
||||
size_bytes: int
|
||||
content_type: str
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int) -> float:
|
||||
"""Экспоненциальный backoff с jitter («equal jitter»): половина задержки
|
||||
фиксирована, половина случайна — повторы разных задач не идут синхронной волной."""
|
||||
d = min(settings.job_backoff_max_seconds, settings.job_backoff_base_seconds * 2 ** (attempt - 1))
|
||||
return d / 2 + random.uniform(0, d / 2)
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self) -> None:
|
||||
self.stop = threading.Event()
|
||||
self.name = settings.instance_id
|
||||
self.pool = ConnectionPool(
|
||||
conninfo=settings.database_url, min_size=1,
|
||||
max_size=settings.worker_concurrency + 3, timeout=settings.db_pool_timeout,
|
||||
kwargs={"connect_timeout": settings.db_connect_timeout,
|
||||
"options": f"-c statement_timeout={settings.db_statement_timeout_ms}",
|
||||
"application_name": "messenger-worker"},
|
||||
check=ConnectionPool.check_connection, open=False)
|
||||
self.r = broker.sync_client(read_timeout=settings.broker_timeout + 5)
|
||||
self.mp = multiprocessing.get_context("forkserver")
|
||||
self.mp.set_forkserver_preload(["app.imaging"])
|
||||
self.work_root = Path(tempfile.gettempdir()) / "worker-jobs"
|
||||
|
||||
# ================================================================ запуск
|
||||
def run(self) -> None:
|
||||
signal.signal(signal.SIGTERM, lambda *_: self.stop.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: self.stop.set())
|
||||
self.pool.open(wait=False)
|
||||
storage.init()
|
||||
shutil.rmtree(self.work_root, ignore_errors=True) # мусор прошлой жизни контейнера
|
||||
self.work_root.mkdir(parents=True, exist_ok=True)
|
||||
threads = [threading.Thread(target=self.maintenance_loop, name="maintenance", daemon=True)]
|
||||
for i in range(settings.worker_concurrency):
|
||||
threads.append(threading.Thread(target=self.consume_loop, args=(f"{self.name}-{i}",),
|
||||
name=f"consumer-{i}", daemon=True))
|
||||
for t in threads:
|
||||
t.start()
|
||||
log.info("worker %s started: concurrency=%d max_attempts=%d timeout=%ss lease=%ss",
|
||||
self.name, settings.worker_concurrency, settings.job_max_attempts,
|
||||
settings.job_timeout_seconds, settings.job_lease_seconds)
|
||||
while not self.stop.is_set():
|
||||
self.stop.wait(1)
|
||||
log.info("worker %s stopping: finishing current jobs", self.name)
|
||||
for t in threads[1:]:
|
||||
t.join(timeout=settings.job_timeout_seconds + 10)
|
||||
self.pool.close()
|
||||
|
||||
# ================================================================ потребитель
|
||||
def consume_loop(self, consumer: str) -> None:
|
||||
"""Ограниченная конкуренция: каждый поток берёт по ОДНОМУ сообщению (COUNT 1),
|
||||
потоков WORKER_CONCURRENCY → не больше N задач на процесс (аналог prefetch)."""
|
||||
last_scan = 0.0
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
self._ensure_group()
|
||||
batch = []
|
||||
if time.monotonic() - last_scan > 10:
|
||||
last_scan = time.monotonic()
|
||||
# 1) свои неподтверждённые (worker перезапустился в том же контейнере);
|
||||
resp = self.r.xreadgroup(settings.jobs_group, consumer,
|
||||
{settings.jobs_stream: "0"}, count=10)
|
||||
batch += [m for _, msgs in resp for m in msgs]
|
||||
# 2) брошенные другими (их процесс умер до XACK).
|
||||
claimed = self.r.xautoclaim(settings.jobs_stream, settings.jobs_group, consumer,
|
||||
min_idle_time=int(settings.claim_idle_seconds * 1000),
|
||||
start_id="0-0", count=5)
|
||||
batch += [m for m in claimed[1] if m]
|
||||
if not batch:
|
||||
resp = self.r.xreadgroup(settings.jobs_group, consumer,
|
||||
{settings.jobs_stream: ">"}, count=1, block=2000)
|
||||
batch = [m for _, msgs in resp for m in msgs]
|
||||
for msg_id, fields in batch:
|
||||
if self.stop.is_set():
|
||||
break
|
||||
if self.handle(consumer, msg_id, fields) == "ack":
|
||||
self._ack(msg_id)
|
||||
except redis.ResponseError as exc:
|
||||
if "NOGROUP" in str(exc): # брокер пересоздан без данных — создаём группу снова
|
||||
self._group_ok = False
|
||||
log.warning("broker error (%s), retry in 1s", str(exc)[:80])
|
||||
self.stop.wait(1)
|
||||
except redis.RedisError as exc:
|
||||
log.warning("broker unavailable (%s), retry in 1s", type(exc).__name__)
|
||||
self.stop.wait(1)
|
||||
except Exception: # noqa: BLE001 — поток не должен умирать
|
||||
log.exception("consumer loop error")
|
||||
self.stop.wait(1)
|
||||
|
||||
def _ensure_group(self) -> None:
|
||||
if getattr(self, "_group_ok", False):
|
||||
return
|
||||
broker.ensure_group(self.r)
|
||||
self._group_ok = True
|
||||
|
||||
def _ack(self, msg_id: str) -> None:
|
||||
# XACK = consumer ack: результат уже устойчиво зафиксирован в БД и S3.
|
||||
self.r.xack(settings.jobs_stream, settings.jobs_group, msg_id)
|
||||
self.r.xdel(settings.jobs_stream, msg_id)
|
||||
|
||||
# ================================================================ одна задача
|
||||
def handle(self, consumer: str, msg_id: str, fields: dict | None) -> str:
|
||||
raw = (fields or {}).get("job")
|
||||
if raw is None: # запись удалена из stream (XDEL) — подтверждаем
|
||||
return "ack"
|
||||
try:
|
||||
payload = broker.parse(raw)
|
||||
except broker.BadMessage as exc:
|
||||
log.error("bad message %s → DLQ: %s", msg_id, exc)
|
||||
broker.to_dlq(self.r, raw, f"bad_message: {exc}", msg_id)
|
||||
return "ack"
|
||||
try:
|
||||
decision, job = self.claim(payload, consumer)
|
||||
except Exception as exc: # noqa: BLE001 — БД недоступна: не ack, повтор позже
|
||||
log.warning("claim failed for job %s: %s", payload["job_id"], type(exc).__name__)
|
||||
self.stop.wait(1)
|
||||
return "leave"
|
||||
if decision != "process":
|
||||
log.info("job=%s attempt=%s msg=%s → %s", payload["job_id"], payload["attempt"],
|
||||
msg_id, decision)
|
||||
if decision == "poison":
|
||||
broker.to_dlq(self.r, raw, "unknown job", msg_id)
|
||||
return "leave" if decision == "busy" else "ack"
|
||||
return self.process(job, msg_id)
|
||||
|
||||
def claim(self, p: dict, consumer: str) -> tuple[str, Job | None]:
|
||||
"""Таблица решений (сообщение несёт attempt):
|
||||
job нет / не то вложение → poison (DLQ, ack)
|
||||
ready → already_ready (no-op, ack): повторная доставка
|
||||
failed → failed (ack); вернуть может только replay
|
||||
attempt сообщения ≠ attempt job → stale (ack): устаревшая попытка
|
||||
queued → берём в работу
|
||||
processing, аренда истекла → takeover той же попытки
|
||||
processing, аренда жива → busy: не ack, проверим позже (другой работает)"""
|
||||
with self.pool.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT j.status, j.attempt, j.max_attempts, j.lease_until < now(), "
|
||||
"j.pipeline_version, j.attachment_id, a.staging_key, a.object_key, a.sha256, "
|
||||
"a.size_bytes, a.content_type "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||||
"WHERE j.job_id = %s FOR UPDATE OF j", (p["job_id"],)).fetchone()
|
||||
if row is None or str(row[5]) != p["attachment_id"] or row[4] != p["pipeline_version"]:
|
||||
return "poison", None
|
||||
status, attempt, max_attempts, expired = row[0], row[1], row[2], row[3]
|
||||
if status == "ready":
|
||||
return "already_ready", None
|
||||
if status == "failed":
|
||||
return "failed", None
|
||||
if p["attempt"] != attempt:
|
||||
return "stale", None
|
||||
if status == "processing" and not expired:
|
||||
return "busy", None
|
||||
event = "lease_takeover" if status == "processing" else "claimed"
|
||||
token = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'processing', lease_token = %s, lease_owner = %s, "
|
||||
"lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s",
|
||||
(token, consumer, settings.job_lease_seconds, p["job_id"]))
|
||||
conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() "
|
||||
"WHERE id = %s AND status <> 'ready'", (p["attachment_id"],))
|
||||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, %s, %s, %s)",
|
||||
(p["job_id"], event, attempt, consumer))
|
||||
return "process", Job(p["job_id"], p["attachment_id"], row[4], attempt, max_attempts, token,
|
||||
row[6], row[7], row[8], row[9], row[10])
|
||||
|
||||
def process(self, job: Job, msg_id: str) -> str:
|
||||
log.info("job=%s attachment=%s attempt=%d/%d msg=%s: processing", job.job_id,
|
||||
job.attachment_id, job.attempt, job.max_attempts, msg_id)
|
||||
lost = threading.Event()
|
||||
frozen = threading.Event()
|
||||
done = threading.Event()
|
||||
hb = threading.Thread(target=self._heartbeat, args=(job, lost, frozen, done), daemon=True)
|
||||
hb.start()
|
||||
workdir = Path(tempfile.mkdtemp(prefix=f"{job.attachment_id}-", dir=self.work_root))
|
||||
try:
|
||||
if self._fault("freeze-after-claim"):
|
||||
# Имитация зависшего worker: аренду не продлеваем, «спим» дольше аренды.
|
||||
frozen.set()
|
||||
log.warning("FAULT INJECTION: freeze-after-claim, sleeping %ss", 3 * settings.job_lease_seconds)
|
||||
time.sleep(3 * settings.job_lease_seconds)
|
||||
self._check_lease(job)
|
||||
variants = self._build(job, workdir, lost)
|
||||
self._check_lease(job) # не пишем в S3, если задачу уже забрали
|
||||
self._publish_objects(job, variants)
|
||||
if self._fault("crash-after-s3-write"):
|
||||
log.warning("FAULT INJECTION: crash-after-s3-write (before COMMIT and ack)")
|
||||
os._exit(1)
|
||||
self._finalize(job, variants)
|
||||
if self._fault("crash-after-commit-before-ack"):
|
||||
log.warning("FAULT INJECTION: crash-after-commit-before-ack")
|
||||
os._exit(1)
|
||||
log.info("job=%s attachment=%s attempt=%d: ready", job.job_id, job.attachment_id, job.attempt)
|
||||
done.set()
|
||||
self._ack(msg_id)
|
||||
self._drop_staging(job)
|
||||
return "acked"
|
||||
except LeaseLost:
|
||||
log.warning("job=%s attempt=%d: lease lost, result discarded (another worker owns it)",
|
||||
job.job_id, job.attempt)
|
||||
return "ack"
|
||||
except (Transient, Permanent) as exc:
|
||||
return self._failure(job, exc, msg_id)
|
||||
except storage.StorageUnavailable as exc:
|
||||
return self._failure(job, Transient("storage_unavailable", type(exc).__name__), msg_id)
|
||||
except storage.ObjectMissing:
|
||||
return self._failure(job, Permanent("processing_failed", "staging object is missing"), msg_id)
|
||||
except psycopg.Error as exc:
|
||||
return self._failure(job, Transient("database_unavailable", type(exc).__name__), msg_id)
|
||||
finally:
|
||||
done.set()
|
||||
hb.join(timeout=2)
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
def _heartbeat(self, job: Job, lost: threading.Event, frozen: threading.Event,
|
||||
done: threading.Event) -> None:
|
||||
"""Живой worker продлевает аренду каждые lease/3 с — так «долгая» задача
|
||||
отличается от «зависшей»: у долгой аренда свежая, у мёртвой/замёрзшей — истекает."""
|
||||
interval = settings.job_lease_seconds / 3
|
||||
while not done.wait(interval):
|
||||
if frozen.is_set():
|
||||
continue
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
ok = conn.execute(
|
||||
"UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), "
|
||||
"updated_at = now() WHERE job_id = %s AND lease_token = %s "
|
||||
"AND status = 'processing' RETURNING 1",
|
||||
(settings.job_lease_seconds, job.job_id, job.token)).fetchone()
|
||||
if ok is None:
|
||||
lost.set()
|
||||
return
|
||||
except psycopg.Error as exc:
|
||||
log.warning("lease renew failed: %s", type(exc).__name__)
|
||||
|
||||
# ---------------------------------------------------------------- шаги
|
||||
def _check_lease(self, job: Job) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s",
|
||||
(job.job_id,)).fetchone()
|
||||
if row is None or row[0] != "processing" or str(row[1]) != job.token:
|
||||
raise LeaseLost()
|
||||
|
||||
def _build(self, job: Job, workdir: Path, lost: threading.Event) -> dict:
|
||||
if not job.staging_key:
|
||||
raise Permanent("processing_failed", "staging source is gone")
|
||||
if self._fault("transient-error", counter=True):
|
||||
raise Transient("injected_transient", "fault injection")
|
||||
src = workdir / "source"
|
||||
with open(src, "wb") as f:
|
||||
try:
|
||||
storage.download_sync(job.staging_key, f, settings.max_upload_bytes)
|
||||
except storage.ObjectMissing:
|
||||
raise Permanent("processing_failed", "staging object is missing")
|
||||
except ValueError:
|
||||
raise Permanent("invalid_image", "source larger than limit")
|
||||
h = hashlib.sha256()
|
||||
with open(src, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() != job.sha256:
|
||||
raise Permanent("processing_failed", "source checksum mismatch")
|
||||
|
||||
out = workdir / "out"
|
||||
out.mkdir()
|
||||
result_path = workdir / "result.json"
|
||||
slow = float(self.r.get("fault:slow-seconds") or 0) if settings.fault_injection else 0.0
|
||||
proc = self.mp.Process(target=imaging.child_main, args=(
|
||||
str(src), str(out), job.content_type, settings.max_image_pixels,
|
||||
settings.image_memory_mb, int(settings.job_timeout_seconds) + 5, str(result_path), slow))
|
||||
proc.start()
|
||||
deadline = time.monotonic() + settings.job_timeout_seconds
|
||||
while proc.is_alive() and time.monotonic() < deadline and not lost.is_set():
|
||||
proc.join(timeout=0.5)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
proc.join()
|
||||
if lost.is_set():
|
||||
raise LeaseLost()
|
||||
raise Transient("processing_timeout", f"> {settings.job_timeout_seconds}s")
|
||||
if lost.is_set():
|
||||
raise LeaseLost()
|
||||
if not result_path.exists():
|
||||
raise Transient("processing_crashed", f"exit code {proc.exitcode}")
|
||||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
if not result["ok"]:
|
||||
if result.get("trace"):
|
||||
log.warning("job=%s child error: %s", job.job_id, result["trace"])
|
||||
exc = Permanent if result["permanent"] else Transient
|
||||
raise exc(result["code"], result.get("detail", ""))
|
||||
variants = {}
|
||||
for name in imaging.VARIANTS:
|
||||
v = result["variants"][name]
|
||||
data = Path(v["path"]).read_bytes()
|
||||
v["size_bytes"] = len(data)
|
||||
v["sha256"] = hashlib.sha256(data).hexdigest()
|
||||
v["object_key"] = storage.variant_key(job.attachment_id, job.pipeline_version, name,
|
||||
imaging.VARIANT_EXT[name])
|
||||
variants[name] = v
|
||||
return variants
|
||||
|
||||
def _publish_objects(self, job: Job, variants: dict) -> None:
|
||||
"""S3-эффекты идемпотентны: те же ключи и те же байты при повторе.
|
||||
Оригинал копируется на стороне S3 (staging → originals), без скачивания."""
|
||||
storage.copy_sync(job.staging_key, job.object_key)
|
||||
for name, v in variants.items():
|
||||
storage.put_file_sync(v["object_key"], v["path"], v["content_type"], v["sha256"])
|
||||
|
||||
def _finalize(self, job: Job, variants: dict) -> None:
|
||||
"""Атомарная публикация результата. Только владелец аренды (lease_token);
|
||||
ready не откатывается; ключ эффекта (attachment, pipeline_version, variant)."""
|
||||
with self.pool.connection() as conn, conn.transaction():
|
||||
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s "
|
||||
"FOR UPDATE", (job.job_id,)).fetchone()
|
||||
if row is None or row[0] != "processing" or str(row[1]) != job.token:
|
||||
raise LeaseLost()
|
||||
for name, v in variants.items():
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, variant, "
|
||||
"object_key, content_type, width, height, size_bytes, sha256, created_at) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) "
|
||||
"ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING",
|
||||
(job.attachment_id, job.pipeline_version, name, v["object_key"],
|
||||
v["content_type"], v["width"], v["height"], v["size_bytes"], v["sha256"]))
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, "
|
||||
"lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() "
|
||||
"WHERE job_id = %s", (job.job_id,))
|
||||
conn.execute("UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() "
|
||||
"WHERE id = %s", (job.attachment_id,))
|
||||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) "
|
||||
"VALUES (%s, 'ready', %s, %s)", (job.job_id, job.attempt, self.name))
|
||||
|
||||
def _drop_staging(self, job: Job) -> None:
|
||||
"""Staging удаляется только ПОСЛЕ COMMIT ready и ack. Не вышло — подчистит maintenance."""
|
||||
try:
|
||||
storage.delete_sync(job.staging_key)
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'",
|
||||
(job.attachment_id,))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.info("staging cleanup deferred: %s", type(exc).__name__)
|
||||
|
||||
def _failure(self, job: Job, exc: Transient | Permanent, msg_id: str) -> str:
|
||||
permanent = isinstance(exc, Permanent)
|
||||
log.warning("job=%s attempt=%d/%d %s error: %s %s", job.job_id, job.attempt, job.max_attempts,
|
||||
"permanent" if permanent else "transient", exc.code, exc.detail)
|
||||
try:
|
||||
outcome = schedule_retry_or_fail(self.pool, job.job_id, job.token, exc.code, exc.detail,
|
||||
permanent, self.name)
|
||||
except psycopg.Error as db_exc:
|
||||
log.warning("cannot record failure (%s): leave message pending", type(db_exc).__name__)
|
||||
return "leave"
|
||||
if outcome == "failed":
|
||||
try:
|
||||
broker.to_dlq(self.r, broker.dumps({"job_id": job.job_id, "attachment_id": job.attachment_id}),
|
||||
exc.code if permanent else f"retry_exhausted ({exc.code})", msg_id)
|
||||
except redis.RedisError:
|
||||
pass # реестр failed — в БД; DLQ-stream лишь копия для оператора
|
||||
return "ack"
|
||||
|
||||
# ---------------------------------------------------------------- fault injection
|
||||
def _fault(self, point: str, counter: bool = False) -> bool:
|
||||
"""Только при FAULT_INJECTION=true (сервис worker-fault). Флаг ставит тест:
|
||||
redis-cli SET fault:<point> 1 (одноразовый), для counter — число срабатываний."""
|
||||
if not settings.fault_injection:
|
||||
return False
|
||||
key = f"fault:{point}"
|
||||
try:
|
||||
if counter:
|
||||
left = self.r.decr(key)
|
||||
if left < 0:
|
||||
self.r.delete(key)
|
||||
return False
|
||||
return True
|
||||
return self.r.getdel(key) is not None
|
||||
except redis.RedisError:
|
||||
return False
|
||||
|
||||
# ================================================================ maintenance
|
||||
def maintenance_loop(self) -> None:
|
||||
while not self.stop.is_set():
|
||||
Path(settings.heartbeat_file).touch()
|
||||
for step in (self.relay_outbox, self.reap_leases, self.sweep_staging):
|
||||
try:
|
||||
step()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("maintenance %s failed: %s", step.__name__, type(exc).__name__)
|
||||
self.stop.wait(settings.maintenance_interval_seconds)
|
||||
|
||||
def relay_outbox(self) -> None:
|
||||
"""Transactional outbox → брокер. Публикует строки, которые API не отметил
|
||||
(упал между COMMIT и publish, брокер был недоступен) и отложенные retry.
|
||||
SKIP LOCKED: несколько worker не публикуют одну строку одновременно.
|
||||
Упали после XADD, до COMMIT → строка опубликуется ещё раз: дубль безопасен."""
|
||||
with self.pool.connection() as conn, conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT id, payload FROM job_outbox WHERE published_at IS NULL "
|
||||
"AND available_at <= now() - make_interval(secs => %s) "
|
||||
"ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED",
|
||||
(settings.outbox_relay_delay_seconds,)).fetchall()
|
||||
for oid, payload in rows:
|
||||
try:
|
||||
stream_id = broker.publish_sync(self.r, payload)
|
||||
except broker.BrokerUnavailable as exc:
|
||||
conn.execute("UPDATE job_outbox SET publish_attempts = publish_attempts + 1, "
|
||||
"last_error = %s WHERE id = %s", (str(exc)[:100], oid))
|
||||
break
|
||||
conn.execute("UPDATE job_outbox SET published_at = now(), stream_id = %s, "
|
||||
"publish_attempts = publish_attempts + 1 WHERE id = %s", (stream_id, oid))
|
||||
log.info("relay: published outbox=%s job=%s attempt=%s", oid, payload["job_id"],
|
||||
payload["attempt"])
|
||||
|
||||
def reap_leases(self) -> None:
|
||||
"""Аренда истекла и за ещё одну аренду никто не перехватил (redelivery) →
|
||||
worker мёртв: новая управляемая попытка через outbox (или failed, если попыток нет)."""
|
||||
with self.pool.connection() as conn:
|
||||
ids = [r[0] for r in conn.execute(
|
||||
"SELECT job_id FROM attachment_jobs WHERE status = 'processing' "
|
||||
"AND lease_until < now() - make_interval(secs => %s) LIMIT 20",
|
||||
(settings.job_lease_seconds,)).fetchall()]
|
||||
for job_id in ids:
|
||||
outcome = schedule_retry_or_fail(self.pool, job_id, None, "lease_expired",
|
||||
"worker stopped renewing lease", False, "reaper")
|
||||
log.warning("reaper: job=%s lease expired → %s", job_id, outcome)
|
||||
|
||||
def sweep_staging(self) -> None:
|
||||
"""Очистка staging: ready — сразу после фиксации (страховка, если worker не успел);
|
||||
failed — после STAGING_RETENTION_SECONDS (до этого исходник нужен для replay)."""
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND ("
|
||||
" (status = 'ready' AND updated_at < now() - interval '30 seconds') OR"
|
||||
" (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) LIMIT 50",
|
||||
(settings.staging_retention_seconds,)).fetchall()
|
||||
for aid, key in rows:
|
||||
storage.delete_sync(key)
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s "
|
||||
"AND status IN ('ready', 'failed')", (aid,))
|
||||
log.info("staging cleaned for attachment %s", aid)
|
||||
|
||||
|
||||
def schedule_retry_or_fail(pool: ConnectionPool, job_id, token: str | None, code: str, detail: str,
|
||||
permanent: bool, actor: str) -> str:
|
||||
"""processing → queued (новая попытка через outbox с backoff) или → failed.
|
||||
token задан — проверяем владельца аренды; None — вызывает reaper (аренда уже истекла)."""
|
||||
with pool.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT status, attempt, max_attempts, lease_token, attachment_id, pipeline_version, "
|
||||
"lease_until < now() - make_interval(secs => %s) FROM attachment_jobs "
|
||||
"WHERE job_id = %s FOR UPDATE SKIP LOCKED", (settings.job_lease_seconds, job_id)).fetchone()
|
||||
if row is None or row[0] != "processing":
|
||||
return "skipped"
|
||||
status, attempt, max_attempts, lease_token, attachment_id, pv, long_expired = row
|
||||
if token is not None and str(lease_token) != token:
|
||||
return "skipped" # аренду уже забрали
|
||||
if token is None and not long_expired:
|
||||
return "skipped"
|
||||
if permanent or attempt >= max_attempts:
|
||||
final = code if permanent else "retry_exhausted"
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, "
|
||||
"lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), "
|
||||
"updated_at = now() WHERE job_id = %s", (final, f"{code}: {detail}"[:300], job_id))
|
||||
conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() "
|
||||
"WHERE id = %s", (final, attachment_id))
|
||||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
||||
"VALUES (%s, 'failed', %s, %s, %s)", (job_id, attempt, actor, f"{code}: {detail}"[:300]))
|
||||
return "failed"
|
||||
delay = backoff_seconds(attempt)
|
||||
nxt = attempt + 1
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, "
|
||||
"next_attempt_at = now() + make_interval(secs => %s), lease_token = NULL, "
|
||||
"lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE job_id = %s",
|
||||
(nxt, f"{code}: {detail}"[:300], delay, job_id))
|
||||
conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s",
|
||||
(attachment_id,))
|
||||
created = conn.execute("SELECT now()").fetchone()[0]
|
||||
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
|
||||
"VALUES (%s, %s, %s, now() + make_interval(secs => %s), now())",
|
||||
(job_id, nxt, json.dumps(payload), delay))
|
||||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
||||
"VALUES (%s, 'retry_scheduled', %s, %s, %s)",
|
||||
(job_id, attempt, actor, f"{code}; next attempt {nxt} in {delay:.1f}s"))
|
||||
return f"retry #{nxt} in {delay:.1f}s"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=settings.log_level.upper(),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
logging.getLogger("botocore").setLevel(logging.WARNING)
|
||||
Worker().run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
# Стенд lab5: клиент → lb (HAProxy) → api × 2 → PostgreSQL (метаданные, jobs, outbox) + Redis (кэш сессий)
|
||||
# ├→ MinIO, приватный bucket: staging/ → originals/, variants/
|
||||
# └→ broker (Redis Streams + AOF) → worker → MinIO + PostgreSQL
|
||||
# Запуск: docker compose --env-file .env up -d --build
|
||||
# Публичный только lb (127.0.0.1:8080). Реплики API, worker, broker и MinIO наружу не публикуются.
|
||||
#
|
||||
# Сети:
|
||||
# app — lb, api, worker, postgres, redis, broker;
|
||||
# storage — internal: только s3, s3-init, api и worker. Ни lb, ни хост туда не попадают:
|
||||
# API и админ-API MinIO (порт 9000) доступны лишь внутри этой сети,
|
||||
# веб-консоль MinIO выключена (MINIO_BROWSER=off).
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-3}
|
||||
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
|
||||
REDIS_TIMEOUT: ${REDIS_TIMEOUT:-0.5}
|
||||
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
# Миграции выполняет отдельный сервис migrate, а не каждая реплика.
|
||||
MIGRATE_ON_START: "false"
|
||||
# lab4: S3. У API — ОТДЕЛЬНЫЙ пользователь MinIO с правами только на свой bucket.
|
||||
# Root-credentials MinIO в API не передаются.
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
S3_CONNECT_TIMEOUT: ${S3_CONNECT_TIMEOUT:-2}
|
||||
S3_READ_TIMEOUT: ${S3_READ_TIMEOUT:-10}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_BODY_TIMEOUT_SECONDS: ${UPLOAD_BODY_TIMEOUT_SECONDS:-60}
|
||||
UPLOAD_SPOOL_MEMORY_BYTES: ${UPLOAD_SPOOL_MEMORY_BYTES:-262144}
|
||||
UPLOAD_MAX_CONCURRENT: ${UPLOAD_MAX_CONCURRENT:-16}
|
||||
IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2}
|
||||
UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900}
|
||||
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
|
||||
S3_BREAKER_SECONDS: ${S3_BREAKER_SECONDS:-5}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
|
||||
# lab5: брокер заданий (ОТДЕЛЬНЫЙ Redis с AOF, не кэш сессий) и параметры обработки.
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
BROKER_TIMEOUT: ${BROKER_TIMEOUT:-2}
|
||||
PIPELINE_VERSION: "1"
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-20}
|
||||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||||
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||||
CLAIM_IDLE_SECONDS: ${CLAIM_IDLE_SECONDS:-90}
|
||||
IMAGE_MEMORY_MB: ${IMAGE_MEMORY_MB:-768}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
|
||||
# Сеть storage — internal: если s3 остановлен, его имя не резолвится, и встроенный DNS
|
||||
# Docker пытается спросить внешний DNS, которого из internal-сети нет → по 5+ c на запрос.
|
||||
# Короткий таймаут резолвера: недоступный S3 даёт быстрый 503, а не «зависание».
|
||||
x-dns-opt: &dns-opt ["timeout:1", "attempts:1"]
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [app]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
# Только кэш сессий: источник истины — PostgreSQL, persistence не нужен.
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no",
|
||||
"--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
networks: [app]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 96M}
|
||||
restart: unless-stopped
|
||||
|
||||
# lab5: брокер заданий — Redis Streams с persistence (тот же образ, другой экземпляр).
|
||||
# --appendonly yes --appendfsync always: ответ на XADD уходит после fsync AOF
|
||||
# (это наш «publisher confirm»: принятое брокером сообщение переживёт рестарт);
|
||||
# --save "": RDB-снимки не нужны, источник — AOF;
|
||||
# --maxmemory-policy noeviction: сообщения никогда не вытесняются из памяти.
|
||||
broker:
|
||||
image: redis:7.4.1-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "",
|
||||
"--maxmemory", "200mb", "--maxmemory-policy", "noeviction"]
|
||||
volumes:
|
||||
- brokerdata:/data
|
||||
networks: [app]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256M}
|
||||
restart: unless-stopped
|
||||
|
||||
# lab4: S3-совместимое хранилище. Версия закреплена; данные — в volume s3data.
|
||||
# Официальные образы MinIO больше не скачиваются анонимно (Docker Hub — удалены,
|
||||
# quay.io — 401). Используем сборку MinIO от сообщества Pigsty (тот же код MinIO,
|
||||
# тот же CLI/env), версия закреплена.
|
||||
s3:
|
||||
image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z
|
||||
command: ["server", "/data"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_BROWSER: "off" # веб-консоль выключена совсем
|
||||
volumes:
|
||||
- s3data:/data
|
||||
networks: [storage] # портов наружу нет
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
restart: unless-stopped
|
||||
|
||||
# Однократный идемпотентный bootstrap: bucket (private), политика, пользователь API.
|
||||
# Повторный запуск ничего не ломает (mb --ignore-existing, policy/user перезаписываются).
|
||||
s3-init:
|
||||
image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z # клиент MinIO (mc) той же сборки
|
||||
entrypoint: ["/bin/sh", "/infra/bootstrap.sh"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
volumes:
|
||||
- ./infra/minio:/infra:ro
|
||||
networks: [storage]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
restart: "no"
|
||||
|
||||
# Однократные миграции до старта реплик (advisory lock защищает и от повтора).
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment: *api-env
|
||||
networks: [app]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256M}
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi-messenger:lab5
|
||||
environment: *api-env
|
||||
dns_opt: *dns-opt
|
||||
# Портов наружу нет: доступ только через lb. container_name не задан — сервис масштабируется.
|
||||
expose: ["8080"]
|
||||
networks: [app, storage]
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256M}
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
# docker stop → SIGTERM: uvicorn перестаёт принимать соединения и дожидается текущих запросов.
|
||||
stop_grace_period: 15s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_started
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# lab5: worker — отдельный процесс, тот же образ. Портов нет, в lb не входит.
|
||||
# Ограниченная конкуренция: WORKER_CONCURRENCY задач на процесс, каждая — в дочернем
|
||||
# процессе с лимитом памяти (IMAGE_MEMORY_MB) и времени (JOB_TIMEOUT_SECONDS).
|
||||
# docker stop → SIGTERM: новых задач не берёт, текущую дорабатывает (до таймаута).
|
||||
worker:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never # образ собирает сервис api
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment: *api-env
|
||||
dns_opt: *dns-opt
|
||||
networks: [app, storage]
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 1G}
|
||||
healthcheck:
|
||||
# heartbeat-файл обновляет поток maintenance раз в секунду
|
||||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat')<30 else 1)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
stop_grace_period: 75s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
api:
|
||||
condition: service_started # образ mtusi-messenger:lab5 собирается сервисом api
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
- "127.0.0.1:8081:8081"
|
||||
- "127.0.0.1:8082:8082"
|
||||
networks: [app]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
# API с коротким сроком сессии — только для теста истечения (профиль ttl).
|
||||
# Доступен тоже только через lb: http://localhost:8081
|
||||
api-ttl:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never # образ собирается локально сервисом api
|
||||
environment:
|
||||
<<: *api-env
|
||||
SESSION_TTL_SECONDS: "2"
|
||||
JANITOR_ENABLED: "false"
|
||||
dns_opt: *dns-opt
|
||||
expose: ["8080"]
|
||||
networks: [app, storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
profiles: ["ttl"]
|
||||
|
||||
# Реплика с внедрением отказов (профиль fault) — только для student-tests/orphans.py.
|
||||
# Заголовок X-Fault-Inject работает ТОЛЬКО здесь (FAULT_INJECTION=true).
|
||||
# Доступна только через lb: http://localhost:8082
|
||||
api-fault:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
environment:
|
||||
<<: *api-env
|
||||
FAULT_INJECTION: "true"
|
||||
JANITOR_ENABLED: "false"
|
||||
dns_opt: *dns-opt
|
||||
expose: ["8080"]
|
||||
networks: [app, storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256M}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped # после crash-after-s3-put контейнер поднимется сам
|
||||
profiles: ["fault"]
|
||||
|
||||
# lab5: worker с внедрением отказов (профиль fault) — только для student-tests/jobs.py.
|
||||
# Отказ включает одноразовый флаг в брокере: redis-cli SET fault:<точка> 1.
|
||||
# restart: unless-stopped — после os._exit(1) контейнер поднимается с тем же hostname,
|
||||
# поэтому worker находит своё неподтверждённое сообщение в PEL.
|
||||
worker-fault:
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
FAULT_INJECTION: "true"
|
||||
JOB_BACKOFF_BASE_SECONDS: "1" # короче паузы в тесте retry
|
||||
dns_opt: *dns-opt
|
||||
networks: [app, storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 1G}
|
||||
stop_grace_period: 75s
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
profiles: ["fault"]
|
||||
|
||||
networks:
|
||||
app: {}
|
||||
storage:
|
||||
internal: true # без выхода наружу и без публикации портов
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
brokerdata:
|
||||
@@ -0,0 +1,108 @@
|
||||
# attachments
|
||||
# date: 2026-11-11T23:52:28 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/attachments.py --base-url http://localhost:8080
|
||||
== позитивный сценарий ==
|
||||
ok - upload PNG -> 202 (got 202)
|
||||
ok - 202 queued, variants=[], error_code=null
|
||||
ok - sha256/size describe the original
|
||||
ok - no S3 key/bucket in public JSON
|
||||
ok - upload JPEG with Unicode filename -> 202 (got 202)
|
||||
ok - filename returned as is (display only)
|
||||
ok - both become ready (worker)
|
||||
ok - member downloads PNG -> 200 (got 200)
|
||||
ok - downloaded bytes are identical (sha256)
|
||||
ok - true Content-Type image/png
|
||||
ok - nosniff header
|
||||
ok - member downloads JPEG (variant=original) -> 200 (got 200)
|
||||
ok - JPEG bytes and type
|
||||
ok - message with empty text + 2 attachments -> 201 (got 201)
|
||||
ok - attachment_ids kept in request order
|
||||
ok - history shows attachment_ids
|
||||
ok - text-only POST (lab3) still valid, attachment_ids=[]
|
||||
ok - member reads metadata -> 200 (got 200)
|
||||
ok - thumbnail variant exists in lab5 -> 200 (got 200)
|
||||
ok - unknown variant name -> 400 (got 400)
|
||||
== доступ ==
|
||||
ok - outsider GET attachment metadata -> 404 (got 404)
|
||||
ok - unknown attachment metadata -> 404 (got 404)
|
||||
ok - outsider GET attachment/content -> 404 (got 404)
|
||||
ok - unknown attachment/content -> 404 (got 404)
|
||||
ok - anonymous download -> 401 (got 401)
|
||||
ok - attachment id is not a UUID -> 400 (got 400)
|
||||
ok - outsider uploads into foreign chat -> 404 (got 404)
|
||||
ok - anonymous upload -> 401 (got 401)
|
||||
== привязка к сообщению ==
|
||||
ok - eve uploads into her own chat -> 202 (got 202)
|
||||
ok - alice uploads into her second chat -> 202 (got 202)
|
||||
ok - foreign attachment_id (eve's chat) in message -> 404 (got 404)
|
||||
ok - attachment of ANOTHER chat of the same user -> 404 (got 404)
|
||||
ok - unknown attachment_id -> 404 (got 404)
|
||||
ok - one valid + one foreign -> whole message rejected -> 404 (got 404)
|
||||
ok - duplicate attachment_ids -> 400 (got 400)
|
||||
ok - same UUID in different case is a duplicate -> 400 (got 400)
|
||||
ok - more than 10 attachment_ids -> 400 (got 400)
|
||||
ok - attachment_id is not a UUID -> 400 (got 400)
|
||||
ok - attachment_ids: null -> 400 (got 400)
|
||||
ok - blank text without attachments -> 400 (got 400)
|
||||
ok - text field is required -> 400 (got 400)
|
||||
ok - any member may reference a chat attachment again -> 201 (got 201)
|
||||
ok - eve attaches alice's file to her own chat -> 404 (got 404)
|
||||
== идемпотентность с вложениями ==
|
||||
ok - missing attachment_ids == [] (same Message)
|
||||
ok - same key, different attachment_ids -> 409 (got 409)
|
||||
ok - replay with attachments returns the same Message
|
||||
== размер ==
|
||||
ok - Content-Length = 10 MiB + 1, body never sent (limit before reading) -> 413 (got 413)
|
||||
ok - Content-Length = 500 MiB -> 413 (got 413)
|
||||
ok - chunked body > 10 MiB without Content-Length (stream cut at limit) -> 413 (got 413)
|
||||
ok - real PNG of 11424004 bytes -> 413 (got 413)
|
||||
ok - empty body -> 400 (got 400)
|
||||
== пиксели ==
|
||||
ok - decompression bomb 8000x8000 in 62290 bytes -> 413 (got 413)
|
||||
ok - 40 040 001 pixels (just over the limit) -> 413 (got 413)
|
||||
== повреждённые файлы ==
|
||||
ok - truncated PNG -> 415 (got 415)
|
||||
ok - truncated PNG: error code
|
||||
ok - PNG with flipped byte (CRC) -> 415 (got 415)
|
||||
ok - PNG with flipped byte (CRC): error code
|
||||
ok - PNG signature only -> 415 (got 415)
|
||||
ok - PNG signature only: error code
|
||||
ok - PNG header + garbage -> 415 (got 415)
|
||||
ok - PNG header + garbage: error code
|
||||
ok - truncated JPEG -> 415 (got 415)
|
||||
ok - truncated JPEG: error code
|
||||
ok - JPEG SOI + garbage -> 415 (got 415)
|
||||
ok - JPEG SOI + garbage: error code
|
||||
== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) ==
|
||||
ok - 30 corrupted files -> only 202/415 (got [202, 415])
|
||||
== подмена MIME ==
|
||||
ok - PNG bytes sent as image/jpeg -> 415 (got 415)
|
||||
ok - JPEG bytes sent as image/png -> 415 (got 415)
|
||||
ok - text renamed to .png -> 415 (got 415)
|
||||
ok - GIF sent as image/png -> 415 (got 415)
|
||||
ok - HTML sent as image/jpeg -> 415 (got 415)
|
||||
ok - PNG with Content-Type image/gif -> 415 (got 415)
|
||||
ok - PNG with Content-Type application/octet-stream -> 415 (got 415)
|
||||
ok - PNG with Content-Type text/html -> 415 (got 415)
|
||||
== опасные filename ==
|
||||
ok - filename '../../etc/passwd' -> 400 (got 400)
|
||||
ok - filename 'dir/a.png' -> 400 (got 400)
|
||||
ok - filename 'C:\\Windows\\a.png' -> 400 (got 400)
|
||||
ok - filename 'a\x00.png' -> 400 (got 400)
|
||||
ok - filename 'a\nb.png' -> 400 (got 400)
|
||||
ok - filename 'tab\t.png' -> 400 (got 400)
|
||||
ok - filename '\x1b[31mred.png' -> 400 (got 400)
|
||||
ok - filename 'photo\u202egnp.exe' -> 400 (got 400)
|
||||
ok - filename '..' -> 400 (got 400)
|
||||
ok - filename '.' -> 400 (got 400)
|
||||
ok - filename ' ' -> 400 (got 400)
|
||||
ok - filename 'xxxxxxxxxxxxxxxxxxxx' -> 400 (got 400)
|
||||
ok - filename missing -> 400 (got 400)
|
||||
ok - filename of exactly 255 chars -> 202 (got 202)
|
||||
ok - filename with spaces, Unicode and punctuation -> 202 (got 202)
|
||||
ok - media type parameters are ignored -> 202 (got 202)
|
||||
ok - no 500 and no dropped connection among 99 checked responses
|
||||
|
||||
91 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,22 @@
|
||||
# broker-config
|
||||
# date: 2026-11-11T23:54:10 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env exec -T broker redis-cli CONFIG GET append*
|
||||
appendfilename
|
||||
appendonly.aof
|
||||
appendonly
|
||||
yes
|
||||
appendfsync
|
||||
always
|
||||
appenddirname
|
||||
appendonlydir
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T broker redis-cli CONFIG GET maxmemory-policy
|
||||
maxmemory-policy
|
||||
noeviction
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.tools queue
|
||||
{"stream_length": 11, "groups": [{"name": "workers", "consumers": 2, "pending": 2, "lag": 9}], "dlq_length": 0, "outbox_unpublished": 0, "jobs_by_status": {"ready": 8, "processing": 2, "queued": 9}}
|
||||
[exit 0]
|
||||
@@ -0,0 +1,6 @@
|
||||
# check
|
||||
# date: 2026-11-11T23:51:41 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
[exit 0]
|
||||
@@ -0,0 +1,56 @@
|
||||
# failover
|
||||
# date: 2026-11-12T15:08:35 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode failover --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\stop-one.cmd --base-url http://localhost:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 1.536}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\Desktop\mtusi2026lab5\scripts\restore-api.cmd
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-s3-init-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-s3-init-1 Started
|
||||
Container mtusi-lab5-broker-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode failover --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\stop-one.cmd --base-url http://localhost:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.007}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\Desktop\mtusi2026lab5\scripts\restore-api.cmd
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-s3-init-1 Starting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-s3-init-1 Started
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
[exit 0]
|
||||
@@ -0,0 +1,17 @@
|
||||
# jobs-broker-down
|
||||
# date: 2026-11-11T23:55:16 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario broker-down
|
||||
== брокер недоступен: приём не зависит от брокера (transactional outbox) ==
|
||||
ok - /health/ready = 200: broker is not critical for API
|
||||
ok - upload → 202 queued while broker is down (2.1s, publish timeout ≤ 2s)
|
||||
ok - intent kept in outbox, not published yet
|
||||
ok - after broker is back: status ready, error_code null
|
||||
ok - after broker is back: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after broker is back: original byte-for-byte
|
||||
ok - after broker is back: thumbnail 256x160, sha256 matches
|
||||
ok - relay published the outbox row when broker came back
|
||||
[broker-down: 14s]
|
||||
|
||||
8 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-crash-ack
|
||||
# date: 2026-11-11T23:56:55 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario crash-ack
|
||||
== crash после COMMIT ready, до ack ==
|
||||
ok - commit before crash: status ready, error_code null
|
||||
ok - commit before crash: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - commit before crash: original byte-for-byte
|
||||
ok - commit before crash: thumbnail 256x160, sha256 matches
|
||||
ok - unacked message redelivered after restart and acked as no-op
|
||||
ok - ready recorded once, 3 variants
|
||||
[crash-ack: 26s]
|
||||
|
||||
6 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,17 @@
|
||||
# jobs-crash-s3
|
||||
# date: 2026-11-11T23:56:03 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario crash-s3
|
||||
== crash после записи в S3, до COMMIT и ack ==
|
||||
recovered in 26.9s (worker restart + lease expiry)
|
||||
ok - after crash before ack: status ready, error_code null
|
||||
ok - after crash before ack: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after crash before ack: original byte-for-byte
|
||||
ok - after crash before ack: thumbnail 256x160, sha256 matches
|
||||
ok - same attempt taken over after lease expiry (redelivery, not a new job)
|
||||
ok - S3 objects from the crashed attempt reused: still exactly 3 variants
|
||||
events: ['22:56:14.755 accepted 1 api', '22:56:14.773 claimed 1 e4d49d50907a-1', '22:56:39.238 lease_takeover 1 e4d49d50907a-1', '22:56:41.607 ready 1 e4d49d50907a']
|
||||
[crash-s3: 48s]
|
||||
|
||||
6 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,19 @@
|
||||
# jobs-lease
|
||||
# date: 2026-11-11T23:58:48 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario lease
|
||||
== долгая, но живая задача: heartbeat продлевает аренду ==
|
||||
ok - job ran 32s > lease 20s, no takeover, attempt 1
|
||||
== замёрзший worker: аренда истекает, задачу забирают ==
|
||||
ok - claimed by worker-fault, then it froze (processing)
|
||||
recovered in 37.3s
|
||||
ok - after frozen worker: status ready, error_code null
|
||||
ok - after frozen worker: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after frozen worker: original byte-for-byte
|
||||
ok - after frozen worker: thumbnail 256x160, sha256 matches
|
||||
ok - reaper: lease expired → attempt 2 → ready
|
||||
ok - woken stale worker could not commit (lease_token fencing), result unchanged
|
||||
[lease: 127s]
|
||||
|
||||
8 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,16 @@
|
||||
# jobs-outbox
|
||||
# date: 2026-11-11T23:57:25 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario outbox
|
||||
== crash API между COMMIT и publish (transactional outbox) ==
|
||||
ok - client did NOT get 202: API died after COMMIT (got 502)
|
||||
ok - attachment and job were committed before the crash (1 new)
|
||||
ok - relay-published job: status ready, error_code null
|
||||
ok - relay-published job: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - relay-published job: original byte-for-byte
|
||||
ok - relay-published job: thumbnail 256x160, sha256 matches
|
||||
ok - outbox row published by relay (no new job created)
|
||||
[outbox: 20s]
|
||||
|
||||
7 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-redelivery
|
||||
# date: 2026-11-11T23:55:34 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario redelivery
|
||||
== повторная доставка уже ready-задачи ==
|
||||
published duplicate: 1794437748730-0 {"schema_version": 1, "job_id": "29c2c47f-13f7-4064-ab1e-56d9860223d8", "type": "at
|
||||
published duplicate: 1794437752117-0 {"schema_version": 1, "job_id": "29c2c47f-13f7-4064-ab1e-56d9860223d8", "type": "at
|
||||
ok - duplicates: status stays ready, variants unchanged (same sha256)
|
||||
ok - no second set of variants: rows=3, S3 objects=3
|
||||
ok - job finished exactly once, attempt unchanged
|
||||
ok - duplicates acked (no pending messages)
|
||||
[redelivery: 25s]
|
||||
|
||||
4 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,21 @@
|
||||
# jobs-registry
|
||||
# date: 2026-11-12T00:00:58 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.replay --list
|
||||
failed jobs: 0
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.replay --dlq
|
||||
1794437902645-0 {"job": "{\"attachment_id\":\"3916ea43-cfd7-4506-838a-5b206345e6b4\",\"job_id\":\"0bb5d8ca-8fad-4a3c-974e-1781acfa27e4\"}", "reason": "retry_exhausted (injected_transient)", "source_id": "1794437902611-0", "at": "2026-11-11T22:58:22.645225Z"}
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.tools queue
|
||||
{"stream_length": 0, "groups": [{"name": "workers", "consumers": 4, "pending": 0, "lag": 0}], "dlq_length": 1, "outbox_unpublished": 0, "jobs_by_status": {"ready": 28}}
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env --profile fault stop api-fault worker-fault
|
||||
Container mtusi-lab5-worker-fault-1 Stopping
|
||||
Container mtusi-lab5-api-fault-1 Stopping
|
||||
Container mtusi-lab5-worker-fault-1 Stopped
|
||||
Container mtusi-lab5-api-fault-1 Stopped
|
||||
[exit 0]
|
||||
@@ -0,0 +1,22 @@
|
||||
# jobs-retry-replay
|
||||
# date: 2026-11-11T23:57:49 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario retry-replay
|
||||
== временные ошибки → backoff+jitter → failed → replay ==
|
||||
ok - after 22s: failed, error_code=retry_exhausted
|
||||
ok - bounded: 5 attempts, 4 scheduled retries, delays [0.7, 1.1, 3.6, 6.3]
|
||||
ok - backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1)
|
||||
ok - content of failed → 409 (got 409)
|
||||
ok - failed attachment cannot be attached → 409 (got 409)
|
||||
ok - failed job is in the registry (python -m app.replay --list)
|
||||
ok - explicit replay accepted
|
||||
ok - after replay: status ready, error_code null
|
||||
ok - after replay: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after replay: original byte-for-byte
|
||||
ok - after replay: thumbnail 256x160, sha256 matches
|
||||
ok - same job_id, attempt 6, replays=1 (audit)
|
||||
ok - replay of a ready job is refused (no duplicate execution)
|
||||
[retry-replay: 55s]
|
||||
|
||||
13 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,26 @@
|
||||
# jobs-worker-down
|
||||
# date: 2026-11-11T23:54:34 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario worker-down
|
||||
== worker остановлен: приём не зависит от обработки ==
|
||||
ok - upload → 202 queued without worker (0.06s)
|
||||
ok - still queued after 3 s: API did not resize synchronously
|
||||
ok - content of queued attachment → 409 (got 409)
|
||||
ok - thumbnail of queued attachment → 409 (got 409)
|
||||
ok - message with queued attachment → 201 (got 201)
|
||||
ok - other messages keep working
|
||||
ok - job is in the broker stream (len=1), outbox published
|
||||
== рестарт брокера и API с сохранением volumes ==
|
||||
ok - after broker restart the message is still there (len=1)
|
||||
ok - same session, same attachment, still queued
|
||||
== worker запущен ==
|
||||
ok - after restart: status ready, error_code null
|
||||
ok - after restart: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after restart: original byte-for-byte
|
||||
ok - after restart: thumbnail 256x160, sha256 matches
|
||||
ok - staging removed after ready (original lives in originals/)
|
||||
attachment 6297945b-c47d-4659-846b-8a2bba4cedb2, job 41928f68-bbf4-415a-b615-022121ec24f2, events: ['22:54:40.477 accepted 1 api', '22:55:06.052 claimed 1 7f4ae0224d1d-0', '22:55:08.563 ready 1 7f4ae0224d1d']
|
||||
[worker-down: 38s]
|
||||
|
||||
14 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,105 @@
|
||||
== позитивный сценарий ==
|
||||
ok - upload PNG -> 202 (got 202)
|
||||
ok - 202 queued, variants=[], error_code=null
|
||||
ok - sha256/size describe the original
|
||||
ok - no S3 key/bucket in public JSON
|
||||
ok - upload JPEG with Unicode filename -> 202 (got 202)
|
||||
ok - filename returned as is (display only)
|
||||
ok - both become ready (worker)
|
||||
ok - member downloads PNG -> 200 (got 200)
|
||||
ok - downloaded bytes are identical (sha256)
|
||||
ok - true Content-Type image/png
|
||||
ok - nosniff header
|
||||
ok - member downloads JPEG (variant=original) -> 200 (got 200)
|
||||
ok - JPEG bytes and type
|
||||
ok - message with empty text + 2 attachments -> 201 (got 201)
|
||||
ok - attachment_ids kept in request order
|
||||
ok - history shows attachment_ids
|
||||
ok - text-only POST (lab3) still valid, attachment_ids=[]
|
||||
ok - member reads metadata -> 200 (got 200)
|
||||
ok - thumbnail variant exists in lab5 -> 200 (got 200)
|
||||
ok - unknown variant name -> 400 (got 400)
|
||||
== доступ ==
|
||||
ok - outsider GET attachment metadata -> 404 (got 404)
|
||||
ok - unknown attachment metadata -> 404 (got 404)
|
||||
ok - outsider GET attachment/content -> 404 (got 404)
|
||||
ok - unknown attachment/content -> 404 (got 404)
|
||||
ok - anonymous download -> 401 (got 401)
|
||||
ok - attachment id is not a UUID -> 400 (got 400)
|
||||
ok - outsider uploads into foreign chat -> 404 (got 404)
|
||||
ok - anonymous upload -> 401 (got 401)
|
||||
== привязка к сообщению ==
|
||||
ok - eve uploads into her own chat -> 202 (got 202)
|
||||
ok - alice uploads into her second chat -> 202 (got 202)
|
||||
ok - foreign attachment_id (eve's chat) in message -> 404 (got 404)
|
||||
ok - attachment of ANOTHER chat of the same user -> 404 (got 404)
|
||||
ok - unknown attachment_id -> 404 (got 404)
|
||||
ok - one valid + one foreign -> whole message rejected -> 404 (got 404)
|
||||
ok - duplicate attachment_ids -> 400 (got 400)
|
||||
ok - same UUID in different case is a duplicate -> 400 (got 400)
|
||||
ok - more than 10 attachment_ids -> 400 (got 400)
|
||||
ok - attachment_id is not a UUID -> 400 (got 400)
|
||||
ok - attachment_ids: null -> 400 (got 400)
|
||||
ok - blank text without attachments -> 400 (got 400)
|
||||
ok - text field is required -> 400 (got 400)
|
||||
ok - any member may reference a chat attachment again -> 201 (got 201)
|
||||
ok - eve attaches alice's file to her own chat -> 404 (got 404)
|
||||
== идемпотентность с вложениями ==
|
||||
ok - missing attachment_ids == [] (same Message)
|
||||
ok - same key, different attachment_ids -> 409 (got 409)
|
||||
ok - replay with attachments returns the same Message
|
||||
== размер ==
|
||||
ok - Content-Length = 10 MiB + 1, body never sent (limit before reading) -> 413 (got 413)
|
||||
ok - Content-Length = 500 MiB -> 413 (got 413)
|
||||
ok - chunked body > 10 MiB without Content-Length (stream cut at limit) -> 413 (got 413)
|
||||
ok - real PNG of 10835273 bytes -> 413 (got 413)
|
||||
ok - real PNG of 10439723 bytes (≤ 10 MiB) accepted -> 202 (got 202)
|
||||
ok - big file sha256/size
|
||||
ok - empty body -> 400 (got 400)
|
||||
== пиксели ==
|
||||
ok - decompression bomb 8000x8000 in 62290 bytes -> 413 (got 413)
|
||||
ok - 40 040 001 pixels (just over the limit) -> 413 (got 413)
|
||||
== повреждённые файлы ==
|
||||
ok - truncated PNG -> 415 (got 415)
|
||||
ok - truncated PNG: error code
|
||||
ok - PNG with flipped byte (CRC) -> 415 (got 415)
|
||||
ok - PNG with flipped byte (CRC): error code
|
||||
ok - PNG signature only -> 415 (got 415)
|
||||
ok - PNG signature only: error code
|
||||
ok - PNG header + garbage -> 415 (got 415)
|
||||
ok - PNG header + garbage: error code
|
||||
ok - truncated JPEG -> 415 (got 415)
|
||||
ok - truncated JPEG: error code
|
||||
ok - JPEG SOI + garbage -> 415 (got 415)
|
||||
ok - JPEG SOI + garbage: error code
|
||||
== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) ==
|
||||
ok - 30 corrupted files -> only 202/415 (got [202, 415])
|
||||
== подмена MIME ==
|
||||
ok - PNG bytes sent as image/jpeg -> 415 (got 415)
|
||||
ok - JPEG bytes sent as image/png -> 415 (got 415)
|
||||
ok - text renamed to .png -> 415 (got 415)
|
||||
ok - GIF sent as image/png -> 415 (got 415)
|
||||
ok - HTML sent as image/jpeg -> 415 (got 415)
|
||||
ok - PNG with Content-Type image/gif -> 415 (got 415)
|
||||
ok - PNG with Content-Type application/octet-stream -> 415 (got 415)
|
||||
ok - PNG with Content-Type text/html -> 415 (got 415)
|
||||
== опасные filename ==
|
||||
ok - filename '../../etc/passwd' -> 400 (got 400)
|
||||
ok - filename 'dir/a.png' -> 400 (got 400)
|
||||
ok - filename 'C:\\Windows\\a.png' -> 400 (got 400)
|
||||
ok - filename 'a\x00.png' -> 400 (got 400)
|
||||
ok - filename 'a\nb.png' -> 400 (got 400)
|
||||
ok - filename 'tab\t.png' -> 400 (got 400)
|
||||
ok - filename '\x1b[31mred.png' -> 400 (got 400)
|
||||
ok - filename 'photo\u202egnp.exe' -> 400 (got 400)
|
||||
ok - filename '..' -> 400 (got 400)
|
||||
ok - filename '.' -> 400 (got 400)
|
||||
ok - filename ' ' -> 400 (got 400)
|
||||
ok - filename 'xxxxxxxxxxxxxxxxxxxx' -> 400 (got 400)
|
||||
ok - filename missing -> 400 (got 400)
|
||||
ok - filename of exactly 255 chars -> 202 (got 202)
|
||||
ok - filename with spaces, Unicode and punctuation -> 202 (got 202)
|
||||
ok - media type parameters are ignored -> 202 (got 202)
|
||||
ok - no 500 and no dropped connection among 100 checked responses
|
||||
|
||||
93 passed, 0 failed
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 143, "failed_attempts": 0, "first_recovery_seconds": 0.114}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,93 @@
|
||||
== worker остановлен: приём не зависит от обработки ==
|
||||
ok - upload → 202 queued without worker (0.02s)
|
||||
ok - still queued after 3 s: API did not resize synchronously
|
||||
ok - content of queued attachment → 409 (got 409)
|
||||
ok - thumbnail of queued attachment → 409 (got 409)
|
||||
ok - message with queued attachment → 201 (got 201)
|
||||
ok - other messages keep working
|
||||
ok - job is in the broker stream (len=1), outbox published
|
||||
== рестарт брокера и API с сохранением volumes ==
|
||||
ok - after broker restart the message is still there (len=1)
|
||||
ok - same session, same attachment, still queued
|
||||
== worker запущен ==
|
||||
ok - after restart: status ready, error_code null
|
||||
ok - after restart: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after restart: original byte-for-byte
|
||||
ok - after restart: thumbnail 256x160, sha256 matches
|
||||
ok - staging removed after ready (original lives in originals/)
|
||||
attachment b442df77-ff58-4ab7-89e9-979b507b38f8, job e2871dfb-6dd2-46c7-8dd1-ce29c7f22558, events: ['01:30:28.325 accepted 1 api', '01:30:37.814 claimed 1 worker-1-0', '01:30:38.282 ready 1 worker-1']
|
||||
[worker-down: 14s]
|
||||
== брокер недоступен: приём не зависит от брокера (transactional outbox) ==
|
||||
ok - /health/ready = 200: broker is not critical for API
|
||||
ok - upload → 202 queued while broker is down (0.0s, publish timeout ≤ 2s)
|
||||
ok - intent kept in outbox, not published yet
|
||||
ok - after broker is back: status ready, error_code null
|
||||
ok - after broker is back: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after broker is back: original byte-for-byte
|
||||
ok - after broker is back: thumbnail 256x160, sha256 matches
|
||||
ok - relay published the outbox row when broker came back
|
||||
[broker-down: 4s]
|
||||
== повторная доставка уже ready-задачи ==
|
||||
published duplicate: 1794436247461-0 {"schema_version": 1, "job_id": "5c7dd513-a5f2-4663-b2ad-c498ea0163e5", "type": "at
|
||||
published duplicate: 1794436247942-0 {"schema_version": 1, "job_id": "5c7dd513-a5f2-4663-b2ad-c498ea0163e5", "type": "at
|
||||
ok - duplicates: status stays ready, variants unchanged (same sha256)
|
||||
ok - no second set of variants: rows=3, S3 objects=3
|
||||
ok - job finished exactly once, attempt unchanged
|
||||
ok - duplicates acked (no pending messages)
|
||||
[redelivery: 7s]
|
||||
== crash после записи в S3, до COMMIT и ack ==
|
||||
recovered in 22.7s (worker restart + lease expiry)
|
||||
ok - after crash before ack: status ready, error_code null
|
||||
ok - after crash before ack: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after crash before ack: original byte-for-byte
|
||||
ok - after crash before ack: thumbnail 256x160, sha256 matches
|
||||
ok - same attempt taken over after lease expiry (redelivery, not a new job)
|
||||
ok - S3 objects from the crashed attempt reused: still exactly 3 variants
|
||||
events: ['01:30:56.132 accepted 1 api', '01:30:56.135 claimed 1 worker-fault-1', '01:31:18.241 lease_takeover 1 worker-fault-1', '01:31:18.745 ready 1 worker-fault']
|
||||
[crash-s3: 27s]
|
||||
== crash после COMMIT ready, до ack ==
|
||||
ok - commit before crash: status ready, error_code null
|
||||
ok - commit before crash: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - commit before crash: original byte-for-byte
|
||||
ok - commit before crash: thumbnail 256x160, sha256 matches
|
||||
ok - unacked message redelivered after restart and acked as no-op
|
||||
ok - ready recorded once, 3 variants
|
||||
[crash-ack: 8s]
|
||||
== crash API между COMMIT и publish (transactional outbox) ==
|
||||
ok - client did NOT get 202: API died after COMMIT (got 502)
|
||||
ok - attachment and job were committed before the crash (1 new)
|
||||
ok - relay-published job: status ready, error_code null
|
||||
ok - relay-published job: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - relay-published job: original byte-for-byte
|
||||
ok - relay-published job: thumbnail 256x160, sha256 matches
|
||||
ok - outbox row published by relay (no new job created)
|
||||
[outbox: 4s]
|
||||
== временные ошибки → backoff+jitter → failed → replay ==
|
||||
ok - after 23s: failed, error_code=retry_exhausted
|
||||
ok - bounded: 5 attempts, 4 scheduled retries, delays [0.6, 1.8, 3.8, 6.4]
|
||||
ok - backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1)
|
||||
ok - content of failed → 409 (got 409)
|
||||
ok - failed attachment cannot be attached → 409 (got 409)
|
||||
ok - failed job is in the registry (python -m app.replay --list)
|
||||
ok - explicit replay accepted
|
||||
ok - after replay: status ready, error_code null
|
||||
ok - after replay: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after replay: original byte-for-byte
|
||||
ok - after replay: thumbnail 256x160, sha256 matches
|
||||
ok - same job_id, attempt 6, replays=1 (audit)
|
||||
ok - replay of a ready job is refused (no duplicate execution)
|
||||
[retry-replay: 31s]
|
||||
== долгая, но живая задача: heartbeat продлевает аренду ==
|
||||
ok - job ran 31s > lease 20s, no takeover, attempt 1
|
||||
== замёрзший worker: аренда истекает, задачу забирают ==
|
||||
ok - claimed by worker-fault, then it froze (processing)
|
||||
recovered in 40.8s
|
||||
ok - after frozen worker: status ready, error_code null
|
||||
ok - after frozen worker: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized'])
|
||||
ok - after frozen worker: original byte-for-byte
|
||||
ok - after frozen worker: thumbnail 256x160, sha256 matches
|
||||
ok - reaper: lease expired → attempt 2 → ready
|
||||
ok - woken stale worker could not commit (lease_token fencing), result unchanged
|
||||
[lease: 104s]
|
||||
|
||||
66 passed, 0 failed
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,5 @@
|
||||
failed jobs: 0
|
||||
1794436317733-0 {"job": "{\"attachment_id\":\"e8a81276-b5df-45cc-8362-f38abe00a557\",\"job_id\":\"2db55464-1734-40c4-9ce1-05b1db9fb834\"}", "reason": "injected_transient", "source_id": "1794436317728-0", "at": "2026-11-11T22:31:57.733543Z"}
|
||||
1794436083241-0 {"job": "{\"attachment_id\":\"abc0488f-8870-4a6b-96de-299a6571d281\",\"job_id\":\"900a5962-c5a8-4680-8cfd-d3033df2e16d\"}", "reason": "injected_transient", "source_id": "1794436083233-0", "at": "2026-11-11T22:28:03.241832Z"}
|
||||
1794435590706-0 {"job": "{\"attachment_id\":\"92896e61-6806-4f6e-b4ac-31c8c7aa1c86\",\"job_id\":\"fae19d4e-898f-4d18-bc22-5dd0dbe3c53a\"}", "reason": "injected_transient", "source_id": "1794435590697-0", "at": "2026-11-11T22:19:50.706703Z"}
|
||||
{"stream_length": 0, "groups": [{"name": "workers", "consumers": 4, "pending": 0, "lag": 0}], "dlq_length": 3, "outbox_unpublished": 0, "jobs_by_status": {"ready": 70}}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,59 @@
|
||||
== аутентификация ==
|
||||
ok - wrong password -> 401 + Basic challenge
|
||||
ok - unknown user -> 401 (same as wrong password)
|
||||
ok - no token -> 401 + Bearer challenge
|
||||
ok - X-User-Id alone gives no access
|
||||
ok - unknown/forged token -> 401
|
||||
ok - Basic on business endpoint -> 401
|
||||
ok - X-User-Id cannot override bearer identity
|
||||
ok - password shorter than 12 -> 400
|
||||
== logout ==
|
||||
ok - logout -> 204
|
||||
ok - token after logout -> 401
|
||||
ok - second logout -> 401
|
||||
== гонка регистрации одного username ==
|
||||
ok - 20 parallel registrations -> one 201, nineteen 409 (got [201, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409])
|
||||
== ошибки и доступ (из lab1) ==
|
||||
ok - malformed JSON -> 400
|
||||
ok - duplicate username -> 409
|
||||
ok - invalid username 'review123\n' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'review123\r\n' -> 400, not 500 (got 400)
|
||||
ok - invalid username ' review123' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'Review123' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'ab' -> 400, not 500 (got 400)
|
||||
ok - blank message -> 400
|
||||
ok - sender_id in body is rejected
|
||||
ok - outsider POST to foreign chat -> 404
|
||||
ok - outsider GET foreign history -> 404
|
||||
ok - invalid chat uuid -> 400
|
||||
== пагинация ==
|
||||
ok - 5 messages / limit 2 -> 3 pages, no duplicates
|
||||
ok - cursor from another chat -> 400
|
||||
== Content-Type ==
|
||||
ok - valid JSON with text/plain -> 415 (got 415)
|
||||
ok - message with text/plain -> 415 (got 415)
|
||||
ok - text/plain without token -> 401 first (got 401)
|
||||
ok - application/json; charset=utf-8 accepted (got 201)
|
||||
|
||||
30 passed, 0 failed
|
||||
== 20 параллельных запросов с одним ключом ==
|
||||
ok - all parallel repeats -> 201 (got [201])
|
||||
ok - all responses carry the same message id (1 distinct)
|
||||
ok - requests were served by >=2 replicas (2)
|
||||
ok - exactly one message in history
|
||||
ok - every replay returns the identical Message
|
||||
== семантика ключа ==
|
||||
ok - same key, different body -> 409 (got 409)
|
||||
ok - other JSON formatting = same request (got 201)
|
||||
ok - same key in another chat is independent
|
||||
ok - same key from another user is independent
|
||||
ok - missing Idempotency-Key -> 400 (got 400)
|
||||
ok - invalid key 'has space'... -> 400 (got 400)
|
||||
ok - invalid key 'xxxxxxxxxxxx'... -> 400 (got 400)
|
||||
ok - invalid key 'slash/key'... -> 400 (got 400)
|
||||
ok - replay without session -> 401, access re-checked (got 401)
|
||||
== неопределённый результат: клиент повторяет с тем же ключом ==
|
||||
ok - retry after uncertain result -> 201
|
||||
ok - still exactly one message
|
||||
|
||||
16 passed, 0 failed
|
||||
@@ -0,0 +1,7 @@
|
||||
ok normal 320x200 with alpha: ok (0.4s)
|
||||
ok pixel limit (36 Mp > limit 10 Mp): invalid_image: pixel limit (0.4s)
|
||||
ok memory limit (36 Mp, RLIMIT_AS 120 MiB): invalid_image: memory limit (0.3s)
|
||||
ok corrupt/truncated PNG: invalid_image: cannot decode: OSError (0.3s)
|
||||
ok time limit (hangs 10 s, timeout 2 s): timeout(killed) (2.0s)
|
||||
|
||||
5 passed, 0 failed
|
||||
@@ -0,0 +1,19 @@
|
||||
# no-local-file
|
||||
# date: 2026-11-12T00:01:21 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env exec -T --index 1 api python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
{"files_scanned": 64, "copies_found": []}
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T --index 2 api python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
{"files_scanned": 57, "copies_found": []}
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T worker python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
{"files_scanned": 58, "copies_found": []}
|
||||
[exit 0]
|
||||
|
||||
$ docker inspect -f {{.Name}} mounts={{json .Mounts}} 858c692aadd78f68ceaadb4d8ee90ab48f93af82b843580a957dfb564d076383 96a8e9076e16f4080345c85bcd5f2267e5abe6a8b0e8b47e56d200fbe29ec603
|
||||
/mtusi-lab5-api-1 mounts=[]
|
||||
/mtusi-lab5-api-2 mounts=[]
|
||||
[exit 0]
|
||||
@@ -0,0 +1,44 @@
|
||||
# orphans
|
||||
# date: 2026-11-12T00:01:29 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env --profile fault up -d api-fault
|
||||
Container mtusi-lab5-postgres-1 Running
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-api-fault-1 Starting
|
||||
Container mtusi-lab5-api-fault-1 Started
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082
|
||||
ok - fault replica (profile fault) is ready on --fault-url
|
||||
ok - committed attachment that must survive every cleanup
|
||||
ok - it is processed to ready (lab5)
|
||||
baseline: {'objects': 29, 'attachments': 29, 'upload_intents': 0}
|
||||
== 1. прерванная загрузка ==
|
||||
ok - no object, no metadata, no journal row after interrupted upload ({'objects': 29, 'attachments': 29, 'upload_intents': 0})
|
||||
== 2. отказ БД после PUT: немедленная компенсация ==
|
||||
ok - upload answered 503, not 202 (got 503)
|
||||
ok - object deleted by compensation, journal clean ({'objects': 29, 'attachments': 29, 'upload_intents': 0})
|
||||
== 3. процесс убит между PUT и COMMIT ==
|
||||
ok - client did NOT get 202 (got 502)
|
||||
ok - orphan object + journal row left behind, no metadata ({'objects': 30, 'attachments': 29, 'upload_intents': 1})
|
||||
ok - default grace 900 s: fresh upload is NOT touched
|
||||
ok - dry-run reports 1 object, changes nothing
|
||||
ok - janitor removed orphan + journal row ({'grace_seconds': 5, 'dry_run': False, 'locked': True, 'intents_expired': 1, 'intents_removed': 1, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 0, 'errors': 0, 'duration_ms': 30.5})
|
||||
ok - second run is a no-op (idempotent)
|
||||
== 4. объект без журнала (страховочный обход bucket) ==
|
||||
ok - untracked object planted
|
||||
ok - scan with grace 1 h ignores a fresh object
|
||||
ok - scan dry-run finds 1
|
||||
ok - scan deletes the untracked object ({'grace_seconds': 5, 'dry_run': False, 'locked': True, 'intents_expired': 0, 'intents_removed': 0, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 1, 'errors': 0, 'objects_scanned': 30, 'duration_ms': 126.7})
|
||||
== 5. готовые вложения не тронуты ==
|
||||
ok - committed attachment still downloads, sha256 matches
|
||||
ok - published original + metadata, staging cleaned, no journal row ({'staging_object_exists': False, 'original_object_exists': True, 'attachment_row': True, 'attachment_status': 'ready', 'upload_intent': False})
|
||||
|
||||
18 passed, 0 failed
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env --profile fault stop api-fault
|
||||
Container mtusi-lab5-api-fault-1 Stopping
|
||||
Container mtusi-lab5-api-fault-1 Stopped
|
||||
[exit 0]
|
||||
@@ -0,0 +1,31 @@
|
||||
# persistence
|
||||
# date: 2026-11-12T15:08:19 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 15 hours ago Up 5 minutes (healthy) 6379/tcp
|
||||
mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 15 hours ago Up 5 minutes (healthy) 127.0.0.1:8080-8082->8080-8082/tcp
|
||||
mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 15 hours ago Up 5 minutes (healthy) 5432/tcp
|
||||
mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 15 hours ago Up 5 minutes (healthy) 6379/tcp
|
||||
mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 15 hours ago Up 3 minutes (healthy) 9000/tcp
|
||||
mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 4 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode persistence --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\restart.cmd --base-url http://localhost:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 5 seconds (health: starting) 8080/tcp
|
||||
mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 5 seconds (health: starting) 8080/tcp
|
||||
mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 15 hours ago Up 6 seconds (healthy) 6379/tcp
|
||||
mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 15 hours ago Up 4 seconds (health: starting) 127.0.0.1:8080-8082->8080-8082/tcp
|
||||
mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 15 hours ago Up 6 seconds (health: starting) 5432/tcp
|
||||
mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 15 hours ago Up 6 seconds (health: starting) 6379/tcp
|
||||
mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 15 hours ago Up 8 seconds (healthy) 9000/tcp
|
||||
mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 5 minutes ago Up 5 seconds (health: starting) 8080/tcp
|
||||
[exit 0]
|
||||
@@ -0,0 +1,7 @@
|
||||
# queue-drain
|
||||
# date: 2026-11-12T15:06:34 commit: 276dbda
|
||||
queue drained in 102.6 s: {'ready': 332}
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.tools queue
|
||||
{"stream_length": 0, "groups": [{"name": "workers", "consumers": 6, "pending": 0, "lag": 0}], "dlq_length": 1, "outbox_unpublished": 0, "jobs_by_status": {"ready": 332}}
|
||||
[exit 0]
|
||||
@@ -0,0 +1,30 @@
|
||||
# s3-bootstrap
|
||||
# date: 2026-11-12T00:01:07 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env run --rm s3-init
|
||||
Container mtusi-lab5-s3-1 Running
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-s3-init-run-a68a74b86415 Creating
|
||||
Container mtusi-lab5-s3-init-run-a68a74b86415 Created
|
||||
Bucket created successfully `root/messenger-attachments`.
|
||||
Access permission for `root/messenger-attachments` is set to `private`
|
||||
policy lab4-api-rw attached to API user
|
||||
verified: API user has policy lab4-api-rw
|
||||
Access permission for `root/messenger-attachments` is `private`
|
||||
bootstrap done: bucket=messenger-attachments (private), API user restricted by policy lab4-api-rw
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env run --rm s3-init
|
||||
Container mtusi-lab5-s3-1 Running
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-s3-init-run-17a0af6b8ff2 Creating
|
||||
Container mtusi-lab5-s3-init-run-17a0af6b8ff2 Created
|
||||
Bucket created successfully `root/messenger-attachments`.
|
||||
Access permission for `root/messenger-attachments` is set to `private`
|
||||
policy lab4-api-rw attached to API user
|
||||
verified: API user has policy lab4-api-rw
|
||||
Access permission for `root/messenger-attachments` is `private`
|
||||
bootstrap done: bucket=messenger-attachments (private), API user restricted by policy lab4-api-rw
|
||||
[exit 0]
|
||||
@@ -0,0 +1,22 @@
|
||||
# s3-outage
|
||||
# date: 2026-11-12T15:04:18 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/s3_outage.py --base-url http://localhost:8080
|
||||
ok - upload before outage
|
||||
ok - it is processed to ready before the outage (lab5: 202 → worker)
|
||||
== MinIO остановлен ==
|
||||
ok - upload -> 503 unavailable (got 503)
|
||||
ok - bounded by timeouts, not a hang (6.6 s)
|
||||
ok - download -> 503 (got 503)
|
||||
ok - metadata still served from PostgreSQL
|
||||
ok - text messages still work
|
||||
ok - message referencing an existing attachment still works (metadata only)
|
||||
ok - /health/ready = 200 (S3 is not critical for text path)
|
||||
== MinIO снова работает ==
|
||||
ok - file uploaded before outage downloads unchanged
|
||||
ok - uploads work again on all replicas: 4 x 202 in a row after 4.7 s (codes during recovery: [503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 202, 202, 202])
|
||||
stats: {'objects': 242, 'attachments': 242, 'upload_intents': 1}
|
||||
ok - janitor cleaned journal rows left by failed uploads ({'grace_seconds': 0, 'dry_run': False, 'locked': True, 'intents_expired': 1, 'intents_removed': 1, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 0, 'errors': 0, 'duration_ms': 39.5})
|
||||
|
||||
12 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,17 @@
|
||||
# s3-private
|
||||
# date: 2026-11-12T00:01:16 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.tools private-check
|
||||
ok API credentials: GET own object -> 200
|
||||
ok anonymous GET object -> 403
|
||||
ok anonymous LIST bucket -> 403
|
||||
ok anonymous PUT object -> 403
|
||||
ok API credentials: ListBuckets (no foreign buckets) -> only messenger-attachments
|
||||
ok API credentials: CreateBucket -> AccessDenied
|
||||
ok API credentials: PutBucketPolicy (make public) -> AccessDenied
|
||||
ok API credentials: DeleteBucket -> AccessDenied
|
||||
|
||||
8 passed, 0 failed
|
||||
[exit 0]
|
||||
127.0.0.1:9000 closed (TimeoutError) ok
|
||||
127.0.0.1:9001 closed (TimeoutError) ok
|
||||
@@ -0,0 +1,6 @@
|
||||
# smoke
|
||||
# date: 2026-11-11T23:51:45 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
[exit 0]
|
||||
@@ -0,0 +1,30 @@
|
||||
# stand
|
||||
# date: 2026-11-11T23:51:42 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 2 minutes ago Up 2 minutes (healthy) 6379/tcp
|
||||
mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080-8082->8080-8082/tcp
|
||||
mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp
|
||||
mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp
|
||||
mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 2 minutes ago Up 2 minutes (healthy) 9000/tcp
|
||||
mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env config --images
|
||||
redis:7.4.1-alpine
|
||||
haproxy:3.0.5-alpine
|
||||
mtusi-messenger:lab5
|
||||
postgres:16.4-alpine
|
||||
mtusi-messenger:lab5
|
||||
pgsty/mc:RELEASE.2026-09-16T00-00-00Z
|
||||
mtusi-messenger:lab5
|
||||
redis:7.4.1-alpine
|
||||
pgsty/minio:RELEASE.2026-08-04T00-00-00Z
|
||||
[exit 0]
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.migrate
|
||||
applied: nothing (up to date)
|
||||
[exit 0]
|
||||
@@ -0,0 +1,67 @@
|
||||
# student-tests
|
||||
# date: 2026-11-11T23:52:05 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/run.py --base-url http://localhost:8080
|
||||
== аутентификация ==
|
||||
ok - wrong password -> 401 + Basic challenge
|
||||
ok - unknown user -> 401 (same as wrong password)
|
||||
ok - no token -> 401 + Bearer challenge
|
||||
ok - X-User-Id alone gives no access
|
||||
ok - unknown/forged token -> 401
|
||||
ok - Basic on business endpoint -> 401
|
||||
ok - X-User-Id cannot override bearer identity
|
||||
ok - password shorter than 12 -> 400
|
||||
== logout ==
|
||||
ok - logout -> 204
|
||||
ok - token after logout -> 401
|
||||
ok - second logout -> 401
|
||||
== гонка регистрации одного username ==
|
||||
ok - 20 parallel registrations -> one 201, nineteen 409 (got [201, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409])
|
||||
== ошибки и доступ (из lab1) ==
|
||||
ok - malformed JSON -> 400
|
||||
ok - duplicate username -> 409
|
||||
ok - invalid username 'review123\n' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'review123\r\n' -> 400, not 500 (got 400)
|
||||
ok - invalid username ' review123' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'Review123' -> 400, not 500 (got 400)
|
||||
ok - invalid username 'ab' -> 400, not 500 (got 400)
|
||||
ok - blank message -> 400
|
||||
ok - sender_id in body is rejected
|
||||
ok - outsider POST to foreign chat -> 404
|
||||
ok - outsider GET foreign history -> 404
|
||||
ok - invalid chat uuid -> 400
|
||||
== пагинация ==
|
||||
ok - 5 messages / limit 2 -> 3 pages, no duplicates
|
||||
ok - cursor from another chat -> 400
|
||||
== Content-Type ==
|
||||
ok - valid JSON with text/plain -> 415 (got 415)
|
||||
ok - message with text/plain -> 415 (got 415)
|
||||
ok - text/plain without token -> 401 first (got 401)
|
||||
ok - application/json; charset=utf-8 accepted (got 201)
|
||||
|
||||
30 passed, 0 failed
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/idempotency.py --base-url http://localhost:8080
|
||||
== 20 параллельных запросов с одним ключом ==
|
||||
ok - all parallel repeats -> 201 (got [201])
|
||||
ok - all responses carry the same message id (1 distinct)
|
||||
ok - requests were served by >=2 replicas (2)
|
||||
ok - exactly one message in history
|
||||
ok - every replay returns the identical Message
|
||||
== семантика ключа ==
|
||||
ok - same key, different body -> 409 (got 409)
|
||||
ok - other JSON formatting = same request (got 201)
|
||||
ok - same key in another chat is independent
|
||||
ok - same key from another user is independent
|
||||
ok - missing Idempotency-Key -> 400 (got 400)
|
||||
ok - invalid key 'has space'... -> 400 (got 400)
|
||||
ok - invalid key 'xxxxxxxxxxxx'... -> 400 (got 400)
|
||||
ok - invalid key 'slash/key'... -> 400 (got 400)
|
||||
ok - replay without session -> 401, access re-checked (got 401)
|
||||
== неопределённый результат: клиент повторяет с тем же ключом ==
|
||||
ok - retry after uncertain result -> 201
|
||||
ok - still exactly one message
|
||||
|
||||
16 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,391 @@
|
||||
{
|
||||
"file_bytes": 10026876,
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"uploads": 30,
|
||||
"status_counts": {
|
||||
"202": 30
|
||||
},
|
||||
"sha256_ok": 30,
|
||||
"instances": [
|
||||
"215466c2b510",
|
||||
"e6cf2ec7aa3b"
|
||||
],
|
||||
"elapsed_s": 13.39,
|
||||
"throughput_MiB_s": 21.4,
|
||||
"latency_s": {
|
||||
"p50": 4.36,
|
||||
"max": 4.68
|
||||
},
|
||||
"memory_MiB": {
|
||||
"idle": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 73.8,
|
||||
"anon": 68.5,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 73.0,
|
||||
"anon": 68.3,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
"sampled_peak": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 139.6,
|
||||
"anon": 90.5
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 141.4,
|
||||
"anon": 93.7
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 84.1,
|
||||
"anon": 78.2,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.3,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
"max_growth_usage": 68.4,
|
||||
"max_growth_anon": 25.4,
|
||||
"cgroup_peak_after": {
|
||||
"e6cf2ec7aa3b": 146.6,
|
||||
"215466c2b510": 151.6
|
||||
}
|
||||
},
|
||||
"container_limit_MiB": 256,
|
||||
"samples": 25,
|
||||
"raw_samples": [
|
||||
{
|
||||
"t": 0.35,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 77.7,
|
||||
"anon": 72.3,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 75.4,
|
||||
"anon": 70.8,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1.03,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 88.3,
|
||||
"anon": 81.9,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 76.6,
|
||||
"anon": 71.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1.82,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 89.1,
|
||||
"anon": 83.2,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 90.2,
|
||||
"anon": 75.7,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2.55,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 88.2,
|
||||
"anon": 80.5,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 130.1,
|
||||
"anon": 81.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3.22,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 96.7,
|
||||
"anon": 72.2,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.6,
|
||||
"anon": 81.2,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3.8,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 108.5,
|
||||
"anon": 75.9,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 86.7,
|
||||
"anon": 81.2,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4.31,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 90.8,
|
||||
"anon": 75.8,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 86.6,
|
||||
"anon": 81.2,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5.23,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 83.2,
|
||||
"anon": 78.9,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 93.2,
|
||||
"anon": 87.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6.23,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 84.6,
|
||||
"anon": 79.3,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 96.5,
|
||||
"anon": 91.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7.1,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 99.1,
|
||||
"anon": 75.9,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 100.5,
|
||||
"anon": 93.7,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7.73,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 119.8,
|
||||
"anon": 78.0,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 100.2,
|
||||
"anon": 81.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8.32,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 139.6,
|
||||
"anon": 87.8,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 122.5,
|
||||
"anon": 81.4,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8.81,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 123.9,
|
||||
"anon": 90.5,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 132.4,
|
||||
"anon": 81.5,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9.53,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 87.1,
|
||||
"anon": 81.0,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.0,
|
||||
"anon": 81.6,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10.22,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 89.1,
|
||||
"anon": 82.9,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 88.0,
|
||||
"anon": 82.1,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10.83,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 89.4,
|
||||
"anon": 82.8,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 88.4,
|
||||
"anon": 82.1,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11.42,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 90.0,
|
||||
"anon": 82.9,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 88.6,
|
||||
"anon": 82.0,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12.12,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 107.6,
|
||||
"anon": 82.9,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 107.8,
|
||||
"anon": 82.1,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12.73,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 134.3,
|
||||
"anon": 82.8,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 141.4,
|
||||
"anon": 91.3,
|
||||
"peak": 149.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13.21,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 134.4,
|
||||
"anon": 82.9,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 135.1,
|
||||
"anon": 93.2,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13.61,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 83.9,
|
||||
"anon": 78.0,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.4,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14.0,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 87.6,
|
||||
"anon": 81.7,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.2,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14.51,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 94.4,
|
||||
"anon": 88.6,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.5,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14.91,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 84.2,
|
||||
"anon": 78.1,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.4,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15.28,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 83.9,
|
||||
"anon": 78.1,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.3,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
{
|
||||
"file_bytes": 10026613,
|
||||
"concurrency": 20,
|
||||
"rounds": 3,
|
||||
"uploads": 60,
|
||||
"status_counts": {
|
||||
"202": 60
|
||||
},
|
||||
"sha256_ok": 60,
|
||||
"instances": [
|
||||
"215466c2b510",
|
||||
"e6cf2ec7aa3b"
|
||||
],
|
||||
"elapsed_s": 25.05,
|
||||
"throughput_MiB_s": 22.9,
|
||||
"latency_s": {
|
||||
"p50": 7.18,
|
||||
"max": 8.63
|
||||
},
|
||||
"memory_MiB": {
|
||||
"idle": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 83.9,
|
||||
"anon": 78.1,
|
||||
"peak": 147.7
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 99.6,
|
||||
"anon": 94.4,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
"sampled_peak": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 153.1,
|
||||
"anon": 107.1
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 146.2,
|
||||
"anon": 101.9
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.7,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.5,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
"max_growth_usage": 69.2,
|
||||
"max_growth_anon": 29.0,
|
||||
"cgroup_peak_after": {
|
||||
"e6cf2ec7aa3b": 203.8,
|
||||
"215466c2b510": 212.6
|
||||
}
|
||||
},
|
||||
"container_limit_MiB": 256,
|
||||
"samples": 40,
|
||||
"raw_samples": [
|
||||
{
|
||||
"t": 0.6,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 87.7,
|
||||
"anon": 82.9,
|
||||
"peak": 147.7
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 100.1,
|
||||
"anon": 94.4,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1.39,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 100.4,
|
||||
"anon": 94.0,
|
||||
"peak": 147.7
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 89.9,
|
||||
"anon": 84.6,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2.34,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 103.4,
|
||||
"anon": 96.9,
|
||||
"peak": 147.7
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 91.3,
|
||||
"anon": 85.0,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3.2,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 106.0,
|
||||
"anon": 99.4,
|
||||
"peak": 153.3
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 92.2,
|
||||
"anon": 85.1,
|
||||
"peak": 155.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3.79,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.4,
|
||||
"anon": 87.0,
|
||||
"peak": 166.3
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 91.2,
|
||||
"anon": 85.4,
|
||||
"peak": 183.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4.39,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 94.8,
|
||||
"anon": 86.8,
|
||||
"peak": 186.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 97.8,
|
||||
"anon": 81.0,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5.0,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 98.6,
|
||||
"anon": 82.7,
|
||||
"peak": 190.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 116.1,
|
||||
"anon": 81.1,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5.71,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 128.4,
|
||||
"anon": 86.8,
|
||||
"peak": 192.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 119.8,
|
||||
"anon": 81.0,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6.63,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 153.1,
|
||||
"anon": 86.8,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 146.2,
|
||||
"anon": 93.0,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7.2,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 137.7,
|
||||
"anon": 86.9,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 133.1,
|
||||
"anon": 94.5,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7.59,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 138.2,
|
||||
"anon": 86.8,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 115.8,
|
||||
"anon": 83.2,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8.05,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 92.7,
|
||||
"anon": 86.7,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 88.4,
|
||||
"anon": 83.2,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8.91,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 94.9,
|
||||
"anon": 88.3,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 93.5,
|
||||
"anon": 87.8,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9.8,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 107.2,
|
||||
"anon": 100.4,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 90.6,
|
||||
"anon": 84.3,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10.62,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 112.1,
|
||||
"anon": 105.3,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 92.0,
|
||||
"anon": 85.4,
|
||||
"peak": 189.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11.42,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 101.0,
|
||||
"anon": 92.8,
|
||||
"peak": 194.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 93.9,
|
||||
"anon": 85.7,
|
||||
"peak": 190.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12.24,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 96.4,
|
||||
"anon": 87.3,
|
||||
"peak": 196.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 112.5,
|
||||
"anon": 86.1,
|
||||
"peak": 191.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12.92,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 117.1,
|
||||
"anon": 87.5,
|
||||
"peak": 196.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 121.2,
|
||||
"anon": 86.8,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13.72,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 139.4,
|
||||
"anon": 87.5,
|
||||
"peak": 196.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 140.0,
|
||||
"anon": 97.5,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14.39,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 150.9,
|
||||
"anon": 89.5,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 135.7,
|
||||
"anon": 99.2,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14.99,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 131.9,
|
||||
"anon": 89.4,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 133.7,
|
||||
"anon": 100.6,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15.37,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 131.9,
|
||||
"anon": 89.5,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 130.2,
|
||||
"anon": 88.3,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15.86,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 132.2,
|
||||
"anon": 89.6,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 130.4,
|
||||
"anon": 88.3,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16.21,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 123.1,
|
||||
"anon": 90.1,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 130.0,
|
||||
"anon": 88.3,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17.13,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 105.8,
|
||||
"anon": 99.8,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 93.6,
|
||||
"anon": 88.2,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18.01,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 113.1,
|
||||
"anon": 106.9,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 93.5,
|
||||
"anon": 87.6,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18.83,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 114.2,
|
||||
"anon": 107.1,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 96.2,
|
||||
"anon": 90.2,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19.73,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 101.4,
|
||||
"anon": 94.7,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 97.1,
|
||||
"anon": 90.1,
|
||||
"peak": 193.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20.63,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 102.9,
|
||||
"anon": 94.7,
|
||||
"peak": 198.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 105.2,
|
||||
"anon": 99.3,
|
||||
"peak": 195.3
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21.5,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 114.6,
|
||||
"anon": 94.7,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 108.6,
|
||||
"anon": 100.3,
|
||||
"peak": 207.7
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22.21,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 128.8,
|
||||
"anon": 94.6,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 126.7,
|
||||
"anon": 101.9,
|
||||
"peak": 209.1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22.9,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 146.2,
|
||||
"anon": 94.8,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 131.3,
|
||||
"anon": 90.2,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23.7,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 147.4,
|
||||
"anon": 87.8,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 136.4,
|
||||
"anon": 90.1,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24.43,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 113.2,
|
||||
"anon": 97.6,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 129.8,
|
||||
"anon": 90.2,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25.04,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 94.1,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 122.8,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25.38,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.8,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.6,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25.8,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.7,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.9,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26.19,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.8,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.6,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26.56,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.8,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.6,
|
||||
"anon": 90.2,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26.97,
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.9,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.6,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
# upload-memory
|
||||
# date: 2026-11-12T15:04:54 commit: 276dbda
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10 --output evidence/upload-memory-10.json
|
||||
file: 10026876 bytes (9.56 MiB), sha256 4582d3a7982b…
|
||||
idle MiB: {'e6cf2ec7aa3b': {'usage': 73.8, 'anon': 68.5, 'peak': 141.4}, '215466c2b510': {'usage': 73.0, 'anon': 68.3, 'peak': 149.5}}
|
||||
{
|
||||
"file_bytes": 10026876,
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"uploads": 30,
|
||||
"status_counts": {
|
||||
"202": 30
|
||||
},
|
||||
"sha256_ok": 30,
|
||||
"instances": [
|
||||
"215466c2b510",
|
||||
"e6cf2ec7aa3b"
|
||||
],
|
||||
"elapsed_s": 13.39,
|
||||
"throughput_MiB_s": 21.4,
|
||||
"latency_s": {
|
||||
"p50": 4.36,
|
||||
"max": 4.68
|
||||
},
|
||||
"memory_MiB": {
|
||||
"idle": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 73.8,
|
||||
"anon": 68.5,
|
||||
"peak": 141.4
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 73.0,
|
||||
"anon": 68.3,
|
||||
"peak": 149.5
|
||||
}
|
||||
},
|
||||
"sampled_peak": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 139.6,
|
||||
"anon": 90.5
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 141.4,
|
||||
"anon": 93.7
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 84.1,
|
||||
"anon": 78.2,
|
||||
"peak": 146.6
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 87.3,
|
||||
"anon": 82.1,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
"max_growth_usage": 68.4,
|
||||
"max_growth_anon": 25.4,
|
||||
"cgroup_peak_after": {
|
||||
"e6cf2ec7aa3b": 146.6,
|
||||
"215466c2b510": 151.6
|
||||
}
|
||||
},
|
||||
"container_limit_MiB": 256
|
||||
}
|
||||
ok - download c8359ca8 sha256 matches
|
||||
ok - download fcda2c0f sha256 matches
|
||||
ok - download 6c733ea4 sha256 matches
|
||||
ok - all 30 uploads -> 202
|
||||
ok - sha256 of every upload matches
|
||||
ok - peak process memory (anon) growth per replica 25.4 MiB < 100 MiB (naive buffering: ≥ 5 × 9.6 MiB × 2)
|
||||
saved evidence/upload-memory-10.json
|
||||
|
||||
6 passed, 0 failed
|
||||
[exit 0]
|
||||
|
||||
$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 20 --output evidence/upload-memory-20.json
|
||||
file: 10026613 bytes (9.56 MiB), sha256 8bb6a49a0703…
|
||||
idle MiB: {'e6cf2ec7aa3b': {'usage': 83.9, 'anon': 78.1, 'peak': 147.7}, '215466c2b510': {'usage': 99.6, 'anon': 94.4, 'peak': 151.6}}
|
||||
{
|
||||
"file_bytes": 10026613,
|
||||
"concurrency": 20,
|
||||
"rounds": 3,
|
||||
"uploads": 60,
|
||||
"status_counts": {
|
||||
"202": 60
|
||||
},
|
||||
"sha256_ok": 60,
|
||||
"instances": [
|
||||
"215466c2b510",
|
||||
"e6cf2ec7aa3b"
|
||||
],
|
||||
"elapsed_s": 25.05,
|
||||
"throughput_MiB_s": 22.9,
|
||||
"latency_s": {
|
||||
"p50": 7.18,
|
||||
"max": 8.63
|
||||
},
|
||||
"memory_MiB": {
|
||||
"idle": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 83.9,
|
||||
"anon": 78.1,
|
||||
"peak": 147.7
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 99.6,
|
||||
"anon": 94.4,
|
||||
"peak": 151.6
|
||||
}
|
||||
},
|
||||
"sampled_peak": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 153.1,
|
||||
"anon": 107.1
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 146.2,
|
||||
"anon": 101.9
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"e6cf2ec7aa3b": {
|
||||
"usage": 93.7,
|
||||
"anon": 87.9,
|
||||
"peak": 203.8
|
||||
},
|
||||
"215466c2b510": {
|
||||
"usage": 95.5,
|
||||
"anon": 90.3,
|
||||
"peak": 212.6
|
||||
}
|
||||
},
|
||||
"max_growth_usage": 69.2,
|
||||
"max_growth_anon": 29.0,
|
||||
"cgroup_peak_after": {
|
||||
"e6cf2ec7aa3b": 203.8,
|
||||
"215466c2b510": 212.6
|
||||
}
|
||||
},
|
||||
"container_limit_MiB": 256
|
||||
}
|
||||
ok - download 56cafe16 sha256 matches
|
||||
ok - download 6a3c88ac sha256 matches
|
||||
ok - download 795dd71a sha256 matches
|
||||
ok - all 60 uploads -> 202
|
||||
ok - sha256 of every upload matches
|
||||
ok - peak process memory (anon) growth per replica 29.0 MiB < 100 MiB (naive buffering: ≥ 10 × 9.6 MiB × 2)
|
||||
saved evidence/upload-memory-20.json
|
||||
|
||||
6 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,12 @@
|
||||
# worker-limits
|
||||
# date: 2026-11-11T23:54:14 commit: 276dbda
|
||||
|
||||
$ docker compose --env-file .env exec -T api python -m app.tools limits-selftest
|
||||
ok normal 320x200 with alpha: ok (3.3s)
|
||||
ok pixel limit (36 Mp > limit 10 Mp): invalid_image: pixel limit (3.8s)
|
||||
ok memory limit (36 Mp, RLIMIT_AS 120 MiB): invalid_image: memory limit (2.0s)
|
||||
ok corrupt/truncated PNG: invalid_image: cannot decode: OSError (2.9s)
|
||||
ok time limit (hangs 10 s, timeout 2 s): timeout(killed) (2.0s)
|
||||
|
||||
5 passed, 0 failed
|
||||
[exit 0]
|
||||
@@ -0,0 +1,84 @@
|
||||
# Балансировщик lab3/lab4: единственная публичная точка входа (127.0.0.1:8080).
|
||||
# Реплики API находятся через DNS Docker по имени сервиса "api"
|
||||
# (server-template + resolvers): добавление/удаление контейнеров подхватывается
|
||||
# без перезапуска LB. Активная проверка /health/ready исключает недоступную реплику.
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 5s
|
||||
hold other 5s
|
||||
hold refused 5s
|
||||
hold nx 5s
|
||||
hold timeout 5s
|
||||
hold obsolete 5s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
# Короткий connect: остановленный контейнер не отвечает на SYN, и без этого
|
||||
# запрос ждал бы 2 c до повтора на другой реплике.
|
||||
timeout connect 500ms
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
# Повтор только при ошибке УСТАНОВКИ соединения (запрос ещё не отправлен) —
|
||||
# безопасно даже для POST; redispatch отправляет повтор на другую реплику.
|
||||
retries 3
|
||||
option redispatch
|
||||
|
||||
frontend fe_api
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
# Отдельный вход для теста истечения сессии (профиль ttl): тоже только через LB.
|
||||
frontend fe_api_ttl
|
||||
bind :8081
|
||||
default_backend api_ttl
|
||||
|
||||
# lab4: вход в реплику с внедрением отказов (профиль fault) — только для тестов
|
||||
# компенсации/очистки (student-tests/orphans.py). Обычный трафик сюда не идёт.
|
||||
frontend fe_api_fault
|
||||
bind :8082
|
||||
default_backend api_fault
|
||||
|
||||
# Проверка живости самого LB для healthcheck контейнера.
|
||||
frontend fe_health
|
||||
bind :8404
|
||||
monitor-uri /healthz
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
# Проверка раз в 0,5 c; 2 неудачи подряд → реплика исключается (≈1–1,5 c),
|
||||
# 2 успеха подряд → возвращается.
|
||||
default-server inter 500ms fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 6 api:8080 check
|
||||
# Запасной путь, только если НИ ОДНА реплика не готова (например, упала общая БД):
|
||||
# тогда LB всё равно передаёт запрос в живой процесс, и API отвечает по контракту
|
||||
# (401 анониму, 503 unavailable в формате ошибки, /health/live 200), а не голым 503 LB.
|
||||
# Остановленный контейнер исчезает из DNS и сюда не попадает.
|
||||
option allbackups
|
||||
# allow-dup-ip: те же IP, что у основных серверов (по умолчанию HAProxy не даёт
|
||||
# одному IP достаться двум серверам бэкенда, и резервные остались бы без адреса).
|
||||
server-template fallback 6 api:8080 backup resolve-opts allow-dup-ip
|
||||
|
||||
backend api_ttl
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 500ms fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template ttl 2 api-ttl:8080
|
||||
|
||||
backend api_fault
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 500ms fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template fault 1 api-fault:8080
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
# Идемпотентный bootstrap MinIO (сервис s3-init, запускается при каждом `up`).
|
||||
# 1. bucket (если уже есть — не ошибка);
|
||||
# 2. bucket приватный: анонимный доступ «none»;
|
||||
# 3. политика lab4-api-rw: только Get/Put/Delete объектов и List этого bucket;
|
||||
# 4. отдельный пользователь API (S3_ACCESS_KEY_ID) с этой политикой.
|
||||
# Root-пользователь MinIO нужен только здесь. Секреты не печатаются.
|
||||
set -eu
|
||||
: "${MINIO_ROOT_USER:?}" "${MINIO_ROOT_PASSWORD:?}" "${S3_BUCKET:?}"
|
||||
: "${S3_ACCESS_KEY_ID:?}" "${S3_SECRET_ACCESS_KEY:?}"
|
||||
if [ "$S3_ACCESS_KEY_ID" = "$MINIO_ROOT_USER" ]; then
|
||||
echo "S3_ACCESS_KEY_ID must differ from MINIO_ROOT_USER (least privilege)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc alias set root http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
|
||||
mc mb --ignore-existing "root/$S3_BUCKET"
|
||||
mc anonymous set none "root/$S3_BUCKET"
|
||||
|
||||
POLICY_NAME=lab4-api-rw
|
||||
cat > /tmp/policy.json <<POLICY
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::${S3_BUCKET}/*"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::${S3_BUCKET}"]
|
||||
}
|
||||
]
|
||||
}
|
||||
POLICY
|
||||
# create с тем же именем перезаписывает политику — повтор безопасен.
|
||||
mc admin policy create root "$POLICY_NAME" /tmp/policy.json >/dev/null
|
||||
# add для существующего пользователя обновляет секрет — повтор безопасен.
|
||||
mc admin user add root "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
|
||||
# attach для уже привязанной политики возвращает ошибку — это не сбой, проверяем итог ниже.
|
||||
if mc admin policy attach root "$POLICY_NAME" --user "$S3_ACCESS_KEY_ID" >/dev/null 2>&1; then
|
||||
echo "policy $POLICY_NAME attached to API user"
|
||||
else
|
||||
echo "policy $POLICY_NAME was already attached (attach is a no-op)"
|
||||
fi
|
||||
info=$(mc admin user info root "$S3_ACCESS_KEY_ID" --json)
|
||||
case "$info" in
|
||||
*\"policyName\"*"$POLICY_NAME"*) echo "verified: API user has policy $POLICY_NAME" ;;
|
||||
*) echo "API user has NO policy $POLICY_NAME: $info" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
mc anonymous get "root/$S3_BUCKET"
|
||||
echo "bootstrap done: bucket=$S3_BUCKET (private), API user restricted by policy $POLICY_NAME"
|
||||
@@ -0,0 +1,54 @@
|
||||
-- 0001: начальная схема мессенджера (lab2).
|
||||
-- Источник истины для пользователей, чатов, участников, сообщений и сессий.
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT users_username_key UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
-- Участники чата. position сохраняет порядок member_ids из запроса.
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
-- «Мои чаты»: найти чаты пользователя без полного скана.
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
-- Keyset-пагинация истории: WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
-- ORDER BY created_at, id LIMIT n — индекс отдаёт страницу без скана всей истории.
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Сессии. Храним только SHA-256 от токена: утечка таблицы не даёт рабочих токенов.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- 0002: общая идемпотентность POST /chats/{chat_id}/messages (lab3).
|
||||
-- Область уникальности ключа: пользователь + чат (метод и путь фиксированы) + ключ.
|
||||
-- Миграция только добавляет таблицу: старая версия API (lab2) с ней совместима.
|
||||
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL, -- SHA-256 канонического тела запроса
|
||||
-- Ключ вставляется ДО сообщения в той же транзакции: проверка FK — при COMMIT.
|
||||
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, idem_key),
|
||||
CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$')
|
||||
);
|
||||
-- Для очистки ключей старше IDEMPOTENCY_TTL_SECONDS.
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,56 @@
|
||||
-- 0003: вложения (lab4). Метаданные — здесь, байты — в приватном S3 bucket.
|
||||
-- Миграция только добавляет таблицы и ослабляет CHECK текста сообщения:
|
||||
-- работающая версия lab3 с новой схемой совместима (rolling update возможен).
|
||||
|
||||
-- Текст может быть пустым, если у сообщения есть вложения (проверяет API).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
-- Только для отображения; в путь/ключ S3 не попадает.
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
error_code text,
|
||||
-- Ключ объекта назначает сервер (originals/<uuid>); уникален.
|
||||
object_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_dimensions CHECK (width > 0 AND height > 0),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed'))
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
-- Связь сообщение ↔ вложения. position сохраняет порядок attachment_ids из запроса.
|
||||
-- Одно вложение чата можно приложить к нескольким сообщениям этого чата.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
-- Журнал загрузок (компенсация отказа между S3 и PostgreSQL).
|
||||
-- Строка пишется и коммитится ДО PUT в S3, удаляется в одной транзакции
|
||||
-- с INSERT в attachments. Строка старше grace period = брошенная загрузка:
|
||||
-- janitor удаляет её объект в S3 (если он есть) и саму строку.
|
||||
CREATE TABLE upload_intents (
|
||||
object_key text PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX upload_intents_created_idx ON upload_intents (created_at);
|
||||
@@ -0,0 +1,88 @@
|
||||
-- 0004: фоновая обработка вложений (lab5).
|
||||
-- Только добавляет таблицы/колонки: старая версия API (lab4) с этой схемой работает.
|
||||
|
||||
-- Где лежит принятый, но ещё не опубликованный оригинал (приватный prefix staging/).
|
||||
-- object_key — КОНЕЧНЫЙ ключ originals/<id>: объект там появляется, когда worker его опубликует.
|
||||
-- После безопасной фиксации результата staging удаляется и колонка обнуляется.
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key);
|
||||
|
||||
-- Одна логическая работа на (вложение, версия pipeline). job_id не меняется при redelivery.
|
||||
-- attempt — номер управляемой попытки (растёт при retry, lease expiry и replay).
|
||||
-- lease_token — «забор» (fencing): зафиксировать результат может только владелец
|
||||
-- текущей аренды; устаревший worker после пробуждения ничего не перезапишет.
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
attempt integer NOT NULL DEFAULT 1,
|
||||
max_attempts integer NOT NULL,
|
||||
lease_token uuid,
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
next_attempt_at timestamptz,
|
||||
error_code text,
|
||||
last_error text,
|
||||
replays integer NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
finished_at timestamptz,
|
||||
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND attempt <= max_attempts),
|
||||
CONSTRAINT attachment_jobs_pipeline CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_jobs_lease CHECK (status <> 'processing' OR (lease_token IS NOT NULL AND lease_until IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (finished_at) WHERE status = 'failed';
|
||||
|
||||
-- Transactional outbox: строка пишется в ТОЙ ЖЕ транзакции, что вложение и job.
|
||||
-- Пока published_at IS NULL, relay опубликует её в брокер (повтор безопасен: worker идемпотентен).
|
||||
-- available_at в будущем — отложенная попытка (backoff с jitter).
|
||||
CREATE TABLE job_outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
|
||||
attempt integer NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
published_at timestamptz,
|
||||
stream_id text,
|
||||
publish_attempts integer NOT NULL DEFAULT 0,
|
||||
last_error text
|
||||
);
|
||||
CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at) WHERE published_at IS NULL;
|
||||
|
||||
-- Результаты. Ключ эффекта — (attachment_id, pipeline_version, variant):
|
||||
-- повтор job делает ON CONFLICT DO NOTHING и не создаёт второй набор вариантов.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
variant text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant),
|
||||
CONSTRAINT attachment_variants_object_key UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_name CHECK (variant IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0 AND size_bytes > 0),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
-- Журнал событий job: реестр для оператора (failed, replay, takeover) без секретов.
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE,
|
||||
at timestamptz NOT NULL DEFAULT now(),
|
||||
event text NOT NULL,
|
||||
attempt integer,
|
||||
actor text,
|
||||
detail text
|
||||
);
|
||||
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
|
||||
@@ -0,0 +1,12 @@
|
||||
# Рантайм-зависимости, версии закреплены для воспроизводимой сборки.
|
||||
fastapi==0.141.1
|
||||
uvicorn[standard]==0.46.0
|
||||
pydantic==2.13.3
|
||||
psycopg[binary]==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
argon2-cffi==25.1.0
|
||||
redis==8.1.0
|
||||
# lab4: S3-клиент (MinIO) и проверка изображений
|
||||
boto3==1.38.0
|
||||
botocore==1.38.0
|
||||
pillow==11.2.1
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem Windows: same as check-s3-private.sh
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env exec -T api python -m app.tools private-check %*
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Проверка приватности bucket и ограниченных прав API ВНЕ API-приложения:
|
||||
# прямые HTTP-запросы к MinIO без подписи и с ключами API-пользователя.
|
||||
# Запросы идут из контейнера api (он в сети storage), но НЕ через код эндпоинтов.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env exec -T api python -m app.tools private-check "$@"
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem Restart hook (Windows): same as restart.sh, volumes are kept.
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env restart postgres redis broker s3 api worker lb
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Restart hook для make test-persistence: перезапускает ВСЕ реплики API, LB и
|
||||
# хранилища (PostgreSQL, Redis, брокер, MinIO) и worker. Volumes сохраняются; ничего не удаляется и не сидится.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres redis broker s3 api worker lb
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem Restore stopped api replicas after the failover test.
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env start api
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Вернуть остановленные реплики API после failover-теста.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env start api
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem lab5: start the worker again; accepted jobs are processed to ready.
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env start worker
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# lab5: запустить worker; принятые задачи доходят до ready.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env start worker
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Failover hook (Windows): stop ONE api container whose short ID = TARGET_INSTANCE_ID.
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
set CID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.service=api"') do set CID=%%i
|
||||
if "%CID%"=="" exit /b 1
|
||||
docker stop -t 8 %CID% >nul
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Failover hook для make test-failover.
|
||||
# X-Instance-Id реплики = hostname контейнера = короткий ID контейнера Docker.
|
||||
# Останавливаем РОВНО этот контейнер, и только если он относится к сервису api
|
||||
# (БД, Redis и LB этот скрипт не трогает). Реплика остаётся остановленной;
|
||||
# вернуть её: scripts/restore-api.sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
cid=$(docker ps -q --filter "id=${TARGET_INSTANCE_ID}" --filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "no running api container for ${TARGET_INSTANCE_ID}" >&2; exit 1; }
|
||||
docker stop -t 8 "$cid" >/dev/null
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
rem lab5: stop the worker (demo: upload is still accepted with 202 queued).
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env stop worker
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# lab5: остановить worker (демонстрация: upload всё равно 202 queued).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env stop worker
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Вложения (lab4, адаптировано под lab5): позитивный сценарий и негативный набор.
|
||||
С lab5 успешный upload — 202 queued; байты и варианты доступны после ready.
|
||||
|
||||
python student-tests/attachments.py --base-url http://localhost:8080
|
||||
|
||||
Проверяет (каждый пользовательский промах — ожидаемый 4xx, НИКОГДА не 500):
|
||||
- PNG и JPEG: 202 queued → ready, sha256/size, скачивание побайтно вторым участником,
|
||||
истинный Content-Type, Unicode filename, 404 постороннему (метаданные и байты);
|
||||
- слишком большой файл: по Content-Length (тело не отправляется), chunked без
|
||||
Content-Length больше лимита, файл ровно в лимит проходит;
|
||||
- «бомба» по пикселям (маленький файл, > 40 млн пикселей) → 413;
|
||||
- повреждённые изображения (усечённый PNG/JPEG, битый CRC, мусор) → 415;
|
||||
- подмена MIME (PNG как image/jpeg, JPEG как image/png, текст/GIF/HTML как png) → 415;
|
||||
- опасные filename (/, \\, NUL, управляющие, bidi, .., пусто, > 255) → 400;
|
||||
- чужое attachment_id / вложение другого чата / несуществующее → 404;
|
||||
дубликаты, > 10 id, не-UUID → 400; пустой текст без вложений → 400;
|
||||
- идемпотентность с attachment_ids: отсутствие поля == [], другой набор → 409.
|
||||
"""
|
||||
import argparse
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from common import (SAMPLE_JPG, SAMPLE_PNG, Api, Report, make_png, raw_http, sha256,
|
||||
upload, wait_ready)
|
||||
|
||||
LIMIT = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
statuses: list[int] = []
|
||||
|
||||
def expect(st, want, name):
|
||||
statuses.append(st)
|
||||
want = want if isinstance(want, tuple) else (want,)
|
||||
r.check(st in want, f"{name} -> {'/'.join(map(str, want))} (got {st})")
|
||||
|
||||
alice, bob, eve = api.new_user("att_a"), api.new_user("att_b"), api.new_user("att_e")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "files", "member_ids": [alice["id"], bob["id"]]},
|
||||
token=alice["token"])
|
||||
st, other, _ = api.call("POST", "/api/v1/chats", {"title": "alice second", "member_ids": [alice["id"]]},
|
||||
token=alice["token"])
|
||||
st, evechat, _ = api.call("POST", "/api/v1/chats", {"title": "eve", "member_ids": [eve["id"]]},
|
||||
token=eve["token"])
|
||||
A, B, E = alice["token"], bob["token"], eve["token"]
|
||||
mpath = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
print("== позитивный сценарий ==")
|
||||
st, png, _ = upload(api, A, chat["id"], SAMPLE_PNG, "sample.png")
|
||||
expect(st, 202, "upload PNG")
|
||||
r.check(png and png["status"] == "queued" and png["variants"] == [] and png["error_code"] is None,
|
||||
"202 queued, variants=[], error_code=null")
|
||||
r.check(png and png["sha256"] == sha256(SAMPLE_PNG) and png["size_bytes"] == len(SAMPLE_PNG),
|
||||
"sha256/size describe the original")
|
||||
r.check(png and "object_key" not in png and "bucket" not in str(png), "no S3 key/bucket in public JSON")
|
||||
st, jpg, _ = upload(api, B, chat["id"], SAMPLE_JPG, "фото 📷 отпуск.jpg", "image/jpeg")
|
||||
expect(st, 202, "upload JPEG with Unicode filename")
|
||||
r.check(jpg and jpg["filename"] == "фото 📷 отпуск.jpg" and jpg["content_type"] == "image/jpeg",
|
||||
"filename returned as is (display only)")
|
||||
png_ready, jpg_ready = wait_ready(api, B, png["id"]), wait_ready(api, A, jpg["id"])
|
||||
r.check(png_ready["status"] == jpg_ready["status"] == "ready", "both become ready (worker)")
|
||||
st, raw, rh = api.raw("GET", f"/api/v1/attachments/{png['id']}/content", headers={"Authorization": "Bearer " + B})
|
||||
expect(st, 200, "member downloads PNG")
|
||||
r.check(raw == SAMPLE_PNG and sha256(raw) == png["sha256"], "downloaded bytes are identical (sha256)")
|
||||
r.check(rh.get("Content-Type", "").startswith("image/png"), "true Content-Type image/png")
|
||||
r.check(rh.get("X-Content-Type-Options") == "nosniff", "nosniff header")
|
||||
st, raw, rh = api.raw("GET", f"/api/v1/attachments/{jpg['id']}/content?variant=original",
|
||||
headers={"Authorization": "Bearer " + A})
|
||||
expect(st, 200, "member downloads JPEG (variant=original)")
|
||||
r.check(raw == SAMPLE_JPG and rh.get("Content-Type", "").startswith("image/jpeg"), "JPEG bytes and type")
|
||||
st, msg, _ = api.call("POST", mpath, {"text": "", "attachment_ids": [png["id"], jpg["id"]]}, token=A)
|
||||
expect(st, 201, "message with empty text + 2 attachments")
|
||||
r.check(msg and msg["attachment_ids"] == [png["id"], jpg["id"]] and msg["text"] == "",
|
||||
"attachment_ids kept in request order")
|
||||
st, page, _ = api.call("GET", mpath, token=B)
|
||||
got = [m for m in (page or {}).get("items", []) if m["id"] == (msg or {}).get("id")]
|
||||
r.check(len(got) == 1 and got[0]["attachment_ids"] == [png["id"], jpg["id"]], "history shows attachment_ids")
|
||||
st, plain, _ = api.call("POST", mpath, {"text": "просто текст"}, token=B)
|
||||
r.check(st == 201 and plain["attachment_ids"] == [], "text-only POST (lab3) still valid, attachment_ids=[]")
|
||||
st, meta, _ = api.call("GET", f"/api/v1/attachments/{png['id']}", token=B)
|
||||
expect(st, 200, "member reads metadata")
|
||||
st, raw, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content?variant=thumbnail",
|
||||
headers={"Authorization": "Bearer " + B})
|
||||
expect(st, 200, "thumbnail variant exists in lab5")
|
||||
st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content?variant=bogus",
|
||||
headers={"Authorization": "Bearer " + B})
|
||||
expect(st, 400, "unknown variant name")
|
||||
|
||||
print("== доступ ==")
|
||||
for suffix in ("", "/content"):
|
||||
st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}{suffix}", headers={"Authorization": "Bearer " + E})
|
||||
expect(st, 404, f"outsider GET attachment{suffix or ' metadata'}")
|
||||
st, _, _ = api.raw("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", headers={"Authorization": "Bearer " + A})
|
||||
expect(st, 404, f"unknown attachment{suffix or ' metadata'}")
|
||||
st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content")
|
||||
expect(st, 401, "anonymous download")
|
||||
st, _, _ = api.raw("GET", "/api/v1/attachments/not-a-uuid", headers={"Authorization": "Bearer " + A})
|
||||
expect(st, 400, "attachment id is not a UUID")
|
||||
st, _, _ = upload(api, E, chat["id"], SAMPLE_PNG)
|
||||
expect(st, 404, "outsider uploads into foreign chat")
|
||||
st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", SAMPLE_PNG,
|
||||
{"Content-Type": "image/png"})
|
||||
expect(st, 401, "anonymous upload")
|
||||
|
||||
print("== привязка к сообщению ==")
|
||||
st, eve_att, _ = upload(api, E, evechat["id"], SAMPLE_PNG)
|
||||
expect(st, 202, "eve uploads into her own chat")
|
||||
st, other_att, _ = upload(api, A, other["id"], SAMPLE_PNG)
|
||||
expect(st, 202, "alice uploads into her second chat")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [eve_att["id"]]}, token=A)
|
||||
expect(st, 404, "foreign attachment_id (eve's chat) in message")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [other_att["id"]]}, token=A)
|
||||
expect(st, 404, "attachment of ANOTHER chat of the same user")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, token=A)
|
||||
expect(st, 404, "unknown attachment_id")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"], eve_att["id"]]}, token=A)
|
||||
expect(st, 404, "one valid + one foreign -> whole message rejected")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"], png["id"]]}, token=A)
|
||||
expect(st, 400, "duplicate attachment_ids")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"].upper(), png["id"]]}, token=A)
|
||||
expect(st, 400, "same UUID in different case is a duplicate")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, token=A)
|
||||
expect(st, 400, "more than 10 attachment_ids")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": ["nope"]}, token=A)
|
||||
expect(st, 400, "attachment_id is not a UUID")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": None}, token=A)
|
||||
expect(st, 400, "attachment_ids: null")
|
||||
st, _, _ = api.call("POST", mpath, {"text": " ", "attachment_ids": []}, token=A)
|
||||
expect(st, 400, "blank text without attachments")
|
||||
st, _, _ = api.call("POST", mpath, {"attachment_ids": [png["id"]]}, token=A)
|
||||
expect(st, 400, "text field is required")
|
||||
st, m2, _ = api.call("POST", mpath, {"text": "ещё раз", "attachment_ids": [png["id"]]}, token=B)
|
||||
expect(st, 201, "any member may reference a chat attachment again")
|
||||
st, _, _ = api.call("POST", f"/api/v1/chats/{evechat['id']}/messages",
|
||||
{"text": "x", "attachment_ids": [png["id"]]}, token=E)
|
||||
expect(st, 404, "eve attaches alice's file to her own chat")
|
||||
|
||||
print("== идемпотентность с вложениями ==")
|
||||
key = uuid.uuid4().hex
|
||||
st, first, _ = api.call("POST", mpath, {"text": "idem"}, token=A, headers={"Idempotency-Key": key})
|
||||
st2, again, _ = api.call("POST", mpath, {"attachment_ids": [], "text": "idem"}, token=A,
|
||||
headers={"Idempotency-Key": key})
|
||||
r.check(st == st2 == 201 and first == again, "missing attachment_ids == [] (same Message)")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "idem", "attachment_ids": [png["id"]]}, token=A,
|
||||
headers={"Idempotency-Key": key})
|
||||
expect(st, 409, "same key, different attachment_ids")
|
||||
key2 = uuid.uuid4().hex
|
||||
st, a1, _ = api.call("POST", mpath, {"text": "", "attachment_ids": [jpg["id"], png["id"]]}, token=A,
|
||||
headers={"Idempotency-Key": key2})
|
||||
st2, a2, _ = api.call("POST", mpath, {"attachment_ids": [jpg["id"], png["id"]], "text": ""}, token=A,
|
||||
headers={"Idempotency-Key": key2})
|
||||
r.check(st == st2 == 201 and a1 == a2, "replay with attachments returns the same Message")
|
||||
|
||||
print("== размер ==")
|
||||
base = f"/api/v1/chats/{chat['id']}/attachments?filename=big.png"
|
||||
auth = {"Authorization": "Bearer " + A, "Content-Type": "image/png"}
|
||||
st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(LIMIT + 1)})
|
||||
expect(st, 413, "Content-Length = 10 MiB + 1, body never sent (limit before reading)")
|
||||
st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(50 * LIMIT)})
|
||||
expect(st, 413, "Content-Length = 500 MiB")
|
||||
st, _ = raw_http(args.base_url, "POST", base, auth, chunks=(b"\0" * 65536 for _ in range(200)), chunked=True)
|
||||
expect(st, 413, "chunked body > 10 MiB without Content-Length (stream cut at limit)")
|
||||
big = make_png(1900, 1900, noise=True) # ≈10,8 МБ настоящего PNG
|
||||
# Шлём кусками и следим за ранним ответом: сервер отвечает 413 по Content-Length,
|
||||
# не дочитывая тело, и urllib получил бы обрыв соединения вместо ответа.
|
||||
st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(len(big))},
|
||||
chunks=(big[i:i + 65536] for i in range(0, len(big), 65536)))
|
||||
expect(st, 413, f"real PNG of {len(big)} bytes")
|
||||
fit = make_png(1860, 1870, noise=True)
|
||||
if len(fit) <= LIMIT:
|
||||
st, body, _ = upload(api, A, chat["id"], fit, "fits.png", timeout=120)
|
||||
expect(st, 202, f"real PNG of {len(fit)} bytes (≤ 10 MiB) accepted")
|
||||
if st == 202:
|
||||
r.check(body["size_bytes"] == len(fit) and body["sha256"] == sha256(fit), "big file sha256/size")
|
||||
st, _, _ = upload(api, A, chat["id"], b"")
|
||||
expect(st, 400, "empty body")
|
||||
|
||||
print("== пиксели ==")
|
||||
bomb = make_png(8000, 8000, color_type=0, rows=(b"\0" * 8001) * 8000) # 64 млн пикс., ~60 КБ
|
||||
st, _, _ = upload(api, A, chat["id"], bomb, "bomb.png")
|
||||
expect(st, 413, f"decompression bomb 8000x8000 in {len(bomb)} bytes")
|
||||
wide = make_png(40001, 1001, color_type=0, rows=(b"\0" * 40002) * 1001) # 40 040 001 пикс.
|
||||
st, _, _ = upload(api, A, chat["id"], wide, "wide.png")
|
||||
expect(st, 413, "40 040 001 pixels (just over the limit)")
|
||||
|
||||
print("== повреждённые файлы ==")
|
||||
corrupt = bytearray(SAMPLE_PNG)
|
||||
corrupt[len(corrupt) // 2] ^= 0xFF
|
||||
cases = [
|
||||
("truncated PNG", SAMPLE_PNG[: len(SAMPLE_PNG) // 2], "image/png"),
|
||||
("PNG with flipped byte (CRC)", bytes(corrupt), "image/png"),
|
||||
("PNG signature only", SAMPLE_PNG[:8], "image/png"),
|
||||
("PNG header + garbage", SAMPLE_PNG[:33] + b"\xde\xad\xbe\xef" * 100, "image/png"),
|
||||
("truncated JPEG", SAMPLE_JPG[: len(SAMPLE_JPG) // 2], "image/jpeg"),
|
||||
("JPEG SOI + garbage", b"\xff\xd8\xff\xe0" + b"\x00" * 300, "image/jpeg"),
|
||||
]
|
||||
for name, data, ctype in cases:
|
||||
st, body, _ = upload(api, A, chat["id"], data, "x", ctype)
|
||||
expect(st, 415, name)
|
||||
r.check(not body or body["error"]["code"] == "unsupported_media_type", f"{name}: error code")
|
||||
|
||||
print("== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) ==")
|
||||
import random
|
||||
rnd = random.Random(2026)
|
||||
fuzz_codes = []
|
||||
for data, ctype in ((SAMPLE_PNG, "image/png"), (SAMPLE_JPG, "image/jpeg")):
|
||||
for _ in range(15):
|
||||
d = bytearray(data)
|
||||
for _ in range(rnd.randint(1, 4)):
|
||||
d[rnd.randrange(8, len(d))] ^= 1 << rnd.randrange(8)
|
||||
if rnd.random() < 0.3:
|
||||
d = d[: rnd.randrange(16, len(d))]
|
||||
st, _, _ = upload(api, A, chat["id"], bytes(d), "fuzz", ctype)
|
||||
statuses.append(st)
|
||||
fuzz_codes.append(st)
|
||||
r.check(set(fuzz_codes) <= {202, 415},
|
||||
f"30 corrupted files -> only 202/415 (got {sorted(set(fuzz_codes), key=str)})")
|
||||
|
||||
print("== подмена MIME ==")
|
||||
spoof = [
|
||||
("PNG bytes sent as image/jpeg", SAMPLE_PNG, "image/jpeg"),
|
||||
("JPEG bytes sent as image/png", SAMPLE_JPG, "image/png"),
|
||||
("text renamed to .png", b"just text, not an image\n" * 20, "image/png"),
|
||||
("GIF sent as image/png", b"GIF89a\x01\x00\x01\x00\x80\x00\x00" + b"\x00" * 40, "image/png"),
|
||||
("HTML sent as image/jpeg", b"<html><script>alert(1)</script></html>", "image/jpeg"),
|
||||
("PNG with Content-Type image/gif", SAMPLE_PNG, "image/gif"),
|
||||
("PNG with Content-Type application/octet-stream", SAMPLE_PNG, "application/octet-stream"),
|
||||
("PNG with Content-Type text/html", SAMPLE_PNG, "text/html"),
|
||||
]
|
||||
for name, data, ctype in spoof:
|
||||
st, _, _ = upload(api, A, chat["id"], data, "evil.png", ctype)
|
||||
expect(st, 415, name)
|
||||
|
||||
print("== опасные filename ==")
|
||||
bad_names = ["../../etc/passwd", "dir/a.png", "C:\\Windows\\a.png", "a\x00.png", "a\nb.png",
|
||||
"tab\t.png", "\x1b[31mred.png", "photo\u202egnp.exe", "..", ".", " ", "x" * 256]
|
||||
for name in bad_names:
|
||||
st, _, _ = upload(api, A, chat["id"], SAMPLE_PNG, name)
|
||||
expect(st, 400, f"filename {name[:20]!r}")
|
||||
st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments", SAMPLE_PNG,
|
||||
{"Authorization": "Bearer " + A, "Content-Type": "image/png"})
|
||||
expect(st, 400, "filename missing")
|
||||
st, ok, _ = upload(api, A, chat["id"], SAMPLE_PNG, "x" * 255)
|
||||
expect(st, 202, "filename of exactly 255 chars")
|
||||
st, ok, _ = upload(api, A, chat["id"], SAMPLE_PNG, "отчёт (финал) v2 — 100%.png")
|
||||
expect(st, 202, "filename with spaces, Unicode and punctuation")
|
||||
st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={quote('a.png')}", SAMPLE_PNG,
|
||||
{"Authorization": "Bearer " + A, "Content-Type": "image/png; charset=binary"})
|
||||
expect(st, 202, "media type parameters are ignored")
|
||||
|
||||
r.check(500 not in statuses and all(s is not None for s in statuses),
|
||||
f"no 500 and no dropped connection among {len(statuses)} checked responses")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Прогон ВСЕХ проверок lab5 (и унаследованных lab2–4) с сохранением вывода в evidence/ (Windows/macOS/Linux).
|
||||
Стенд должен быть запущен: docker compose --env-file .env up -d --build
|
||||
|
||||
python student-tests/collect_evidence.py --base-url http://localhost:8080
|
||||
python student-tests/collect_evidence.py --only smoke,attachments # выборочно
|
||||
|
||||
Каждый шаг пишет evidence/<имя>.txt (команда, время, stdout/stderr, код выхода).
|
||||
Ошибка одного шага не останавливает остальные; в конце — сводка PASS/FAIL.
|
||||
Hook-и: на Windows scripts/*.cmd, иначе scripts/*.sh. Секреты не печатаются.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
PY = sys.executable
|
||||
WIN = os.name == "nt"
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
SAMPLE_SHA = hashlib.sha256((ROOT / "tests/fixtures/sample.png").read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def hook(name: str) -> str:
|
||||
return str(ROOT / "scripts" / (name + (".cmd" if WIN else ".sh")))
|
||||
|
||||
|
||||
# Публичные tests/ читают JSON без явной кодировки: на Windows (cp1251) это падает.
|
||||
# UTF-8 mode Python для всех дочерних процессов, сами тесты не меняем.
|
||||
ENV = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"}
|
||||
|
||||
|
||||
def run(cmd: list[str], log, timeout: int = 900) -> int:
|
||||
log.write(f"\n$ {' '.join(cmd)}\n")
|
||||
log.flush()
|
||||
try:
|
||||
p = subprocess.run(cmd, cwd=ROOT, env=ENV, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, encoding="utf-8", errors="replace", timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.write("TIMEOUT\n")
|
||||
return 124
|
||||
log.write(p.stdout)
|
||||
log.write(f"[exit {p.returncode}]\n")
|
||||
return p.returncode
|
||||
|
||||
|
||||
def ports_closed(log) -> int:
|
||||
"""MinIO (9000 API, 9001 консоль) не опубликован на хосте."""
|
||||
bad = 0
|
||||
for port in (9000, 9001):
|
||||
s = socket.socket()
|
||||
s.settimeout(1)
|
||||
try:
|
||||
s.connect(("127.0.0.1", port))
|
||||
log.write(f"127.0.0.1:{port} OPEN <- не должно быть\n")
|
||||
bad += 1
|
||||
except OSError as exc:
|
||||
log.write(f"127.0.0.1:{port} closed ({type(exc).__name__}) ok\n")
|
||||
finally:
|
||||
s.close()
|
||||
return bad
|
||||
|
||||
|
||||
def drain(log, limit: int = 600) -> int:
|
||||
"""Ждать, пока в очереди не останется queued/processing (не дольше limit секунд)."""
|
||||
import json
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < limit:
|
||||
out = subprocess.run(COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"], cwd=ROOT,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace").stdout
|
||||
try:
|
||||
jobs = json.loads(out.strip().splitlines()[-1])["jobs_by_status"]
|
||||
except (ValueError, IndexError, KeyError):
|
||||
jobs = {}
|
||||
busy = jobs.get("queued", 0) + jobs.get("processing", 0)
|
||||
if jobs and busy == 0:
|
||||
log.write(f"queue drained in {time.monotonic() - t0:.1f} s: {jobs}\n")
|
||||
return 0
|
||||
time.sleep(3)
|
||||
log.write(f"queue NOT drained in {limit} s\n")
|
||||
return 1
|
||||
|
||||
|
||||
def steps(base: str):
|
||||
t = ["--base-url", base]
|
||||
yield "check", [[PY, "scripts/check_contract.py"]]
|
||||
yield "stand", [COMPOSE + ["ps"], COMPOSE + ["config", "--images"],
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-m", "app.migrate"]]
|
||||
yield "smoke", [[PY, "tests/smoke.py", *t]]
|
||||
yield "student-tests", [[PY, "student-tests/run.py", *t], [PY, "student-tests/idempotency.py", *t]]
|
||||
yield "attachments", [[PY, "student-tests/attachments.py", *t]]
|
||||
# lab5: брокер, worker, отказы и восстановление
|
||||
yield "broker-config", [COMPOSE + ["exec", "-T", "broker", "redis-cli", "CONFIG", "GET", "append*"],
|
||||
COMPOSE + ["exec", "-T", "broker", "redis-cli", "CONFIG", "GET", "maxmemory-policy"],
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"]]
|
||||
yield "worker-limits", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "limits-selftest"]]
|
||||
for sc in ("worker-down", "broker-down", "redelivery", "crash-s3", "crash-ack", "outbox", "retry-replay", "lease"):
|
||||
yield f"jobs-{sc}", [[PY, "student-tests/jobs.py", *t, "--scenario", sc]]
|
||||
yield "jobs-registry", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.replay", "--list"],
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-m", "app.replay", "--dlq"],
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"],
|
||||
COMPOSE + ["--profile", "fault", "stop", "api-fault", "worker-fault"]]
|
||||
yield "s3-bootstrap", [COMPOSE + ["run", "--rm", "s3-init"], COMPOSE + ["run", "--rm", "s3-init"]]
|
||||
yield "s3-private", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "private-check"],
|
||||
"PORTS"]
|
||||
yield "no-local-file", [COMPOSE + ["exec", "-T", "--index", str(i), "api", "python", "-m", "app.tools",
|
||||
"find-local", SAMPLE_SHA] for i in (1, 2)] + [
|
||||
COMPOSE + ["exec", "-T", "worker", "python", "-m", "app.tools", "find-local", SAMPLE_SHA]] + [
|
||||
["docker", "inspect", "-f", "{{.Name}} mounts={{json .Mounts}}"] + "API_IDS".split()]
|
||||
yield "orphans", [COMPOSE + ["--profile", "fault", "up", "-d", "api-fault"], "SLEEP",
|
||||
[PY, "student-tests/orphans.py", *t, "--fault-url", "http://localhost:8082"],
|
||||
COMPOSE + ["--profile", "fault", "stop", "api-fault"]]
|
||||
yield "s3-outage", [[PY, "student-tests/s3_outage.py", *t]]
|
||||
yield "upload-memory", [[PY, "student-tests/upload_memory.py", *t, "--concurrency", "10",
|
||||
"--output", "evidence/upload-memory-10.json"],
|
||||
[PY, "student-tests/upload_memory.py", *t, "--concurrency", "20",
|
||||
"--output", "evidence/upload-memory-20.json"]]
|
||||
# upload-memory ставит в очередь 90 больших картинок: ждём, пока worker их обработает,
|
||||
# иначе вложение следующих тестов встанет в конец очереди (это и есть queue lag).
|
||||
yield "queue-drain", ["DRAIN", COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"]]
|
||||
yield "persistence", [COMPOSE + ["ps"], [PY, "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", hook("restart"), *t], COMPOSE + ["ps"]]
|
||||
yield "failover", [[PY, "tests/resilience.py", "--mode", "failover", "--action-script", hook("stop-one"), *t],
|
||||
[hook("restore-api")], "SLEEP",
|
||||
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", hook("stop-one"), *t],
|
||||
[hook("restore-api")]]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--only", default="")
|
||||
args = ap.parse_args()
|
||||
only = {x.strip() for x in args.only.split(",") if x.strip()}
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip() or "no-git"
|
||||
summary = []
|
||||
for name, cmds in steps(args.base_url):
|
||||
if only and name not in only:
|
||||
continue
|
||||
path = EVIDENCE / f"{name}.txt"
|
||||
print(f"== {name} -> {path.relative_to(ROOT)}")
|
||||
with open(path, "w", encoding="utf-8") as log:
|
||||
log.write(f"# {name}\n# date: {dt.datetime.now().isoformat(timespec='seconds')} commit: {commit}\n")
|
||||
failed = 0
|
||||
for cmd in cmds:
|
||||
if cmd == "SLEEP":
|
||||
time.sleep(8)
|
||||
continue
|
||||
if cmd == "DRAIN":
|
||||
failed += drain(log)
|
||||
continue
|
||||
if cmd == "PORTS":
|
||||
failed += ports_closed(log)
|
||||
continue
|
||||
if "API_IDS" in cmd:
|
||||
ids = subprocess.run(COMPOSE + ["ps", "-q", "api"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.split()
|
||||
cmd = [c for c in cmd if c != "API_IDS"] + ids
|
||||
code = run(cmd, log)
|
||||
failed += 1 if code else 0
|
||||
status = "PASS" if not failed else f"FAIL ({failed})"
|
||||
summary.append((name, status))
|
||||
print(" ", status)
|
||||
print("\nSummary:")
|
||||
for name, status in summary:
|
||||
print(f" {name:18} {status}")
|
||||
sys.exit(0 if all(s == "PASS" for _, s in summary) else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
"""Общий HTTP-клиент для собственных тестов (только стандартная библиотека)."""
|
||||
import base64
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
PASSWORD = "Test-password-" + uuid.uuid4().hex[:8]
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, base: str):
|
||||
self.base = base.rstrip("/")
|
||||
|
||||
def raw(self, method, path, data=None, headers=None, timeout=10):
|
||||
req = urllib.request.Request(self.base + path, data=data, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, r.read(), r.headers
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(), e.headers
|
||||
|
||||
def call(self, method, path, body=None, token=None, headers=None, timeout=10):
|
||||
h = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
# С lab3 POST сообщения требует Idempotency-Key; если тест не задал свой — новый ключ.
|
||||
if method == "POST" and path.split("?")[0].endswith("/messages"):
|
||||
h.setdefault("Idempotency-Key", uuid.uuid4().hex)
|
||||
st, raw, rh = self.raw(method, path, data, h, timeout=timeout)
|
||||
try:
|
||||
return st, json.loads(raw or b"null"), rh
|
||||
except ValueError:
|
||||
return st, None, rh
|
||||
|
||||
def register(self, prefix: str, password: str = PASSWORD) -> dict:
|
||||
name = (prefix + "_" + uuid.uuid4().hex[:10])[:32]
|
||||
st, user, _ = self.call("POST", "/api/v1/users",
|
||||
{"username": name, "display_name": prefix, "password": password})
|
||||
assert st == 201, f"register {name}: {st} {user}"
|
||||
user["password"] = password
|
||||
return user
|
||||
|
||||
def login(self, username: str, password: str):
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
st, body, rh = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
return st, body, rh
|
||||
|
||||
def new_user(self, prefix: str) -> dict:
|
||||
user = self.register(prefix)
|
||||
st, sess, _ = self.login(user["username"], user["password"])
|
||||
assert st == 201, f"login: {st}"
|
||||
user["token"] = sess["token"]
|
||||
return user
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self):
|
||||
self.passed = self.failed = 0
|
||||
|
||||
def check(self, cond, name):
|
||||
if cond:
|
||||
self.passed += 1
|
||||
print(f" ok - {name}")
|
||||
else:
|
||||
self.failed += 1
|
||||
print(f" FAIL - {name}")
|
||||
|
||||
def finish(self):
|
||||
print(f"\n{self.passed} passed, {self.failed} failed")
|
||||
raise SystemExit(1 if self.failed else 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- lab4: файлы
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import select
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
SAMPLE_PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
SAMPLE_JPG = (FIXTURES / "sample.jpg").read_bytes()
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def sha256(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def make_png(width: int, height: int, *, noise: bool = False, color_type: int = 2,
|
||||
rows: bytes | None = None) -> bytes:
|
||||
"""Синтетический PNG только средствами stdlib.
|
||||
noise=True — случайные пиксели (почти не сжимаются: так получаем файл нужного размера).
|
||||
rows — готовые распакованные данные (для «бомб» и битых файлов)."""
|
||||
channels = {0: 1, 2: 3, 6: 4}[color_type]
|
||||
if rows is None:
|
||||
line = width * channels
|
||||
rows = b"".join(b"\x00" + (os.urandom(line) if noise else bytes(line)) for _ in range(height))
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, color_type, 0, 0, 0)
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr)
|
||||
+ chunk(b"IDAT", zlib.compress(rows, 1 if noise else 9)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def upload(api, token: str, chat_id: str, data: bytes, filename: str = "image.png",
|
||||
content_type: str = "image/png", headers: dict | None = None, timeout: int = 60):
|
||||
from urllib.parse import quote
|
||||
h = {"Authorization": "Bearer " + token, "Content-Type": content_type, **(headers or {})}
|
||||
path = f"/api/v1/chats/{chat_id}/attachments?filename={quote(filename, safe='')}"
|
||||
st, raw, rh = api.raw("POST", path, data, h, timeout=timeout)
|
||||
try:
|
||||
body = json.loads(raw or b"null")
|
||||
except ValueError:
|
||||
body = None
|
||||
return st, body, rh
|
||||
|
||||
|
||||
def wait_ready(api, token: str, attachment_id: str, seconds: float = 90) -> dict:
|
||||
"""lab5: upload отвечает 202 queued; ждём, пока worker доведёт до ready/failed."""
|
||||
import time
|
||||
deadline = time.monotonic() + seconds
|
||||
att = {}
|
||||
while time.monotonic() < deadline:
|
||||
st, att, _ = api.call("GET", f"/api/v1/attachments/{attachment_id}", token=token)
|
||||
if st == 200 and att["status"] in ("ready", "failed"):
|
||||
return att
|
||||
time.sleep(0.3)
|
||||
return att or {}
|
||||
|
||||
|
||||
def raw_http(base: str, method: str, path: str, headers: dict, chunks=(), chunked: bool = False,
|
||||
close_after_send: bool = False, timeout: float = 30):
|
||||
"""HTTP/1.1 «руками» через сокет — для сценариев, которые urllib не умеет:
|
||||
заявить Content-Length и не прислать тело, прислать chunked больше лимита,
|
||||
оборвать загрузку на середине. Пока шлём тело, следим, не ответил ли сервер
|
||||
раньше (413 до конца тела) — тогда прекращаем отправку и читаем ответ.
|
||||
Возвращает (status | None, body_bytes)."""
|
||||
u = urlparse(base)
|
||||
sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout)
|
||||
try:
|
||||
lines = [f"{method} {path} HTTP/1.1", f"Host: {u.hostname}:{u.port or 80}", "Connection: close"]
|
||||
lines += [f"{k}: {v}" for k, v in headers.items()]
|
||||
if chunked:
|
||||
lines.append("Transfer-Encoding: chunked")
|
||||
sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
|
||||
for part in chunks:
|
||||
ready, _, _ = select.select([sock], [], [], 0)
|
||||
if ready:
|
||||
break # сервер уже ответил — дальше не шлём
|
||||
try:
|
||||
sock.sendall(f"{len(part):x}\r\n".encode() + part + b"\r\n" if chunked else part)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
break
|
||||
else:
|
||||
if chunked and not close_after_send:
|
||||
try:
|
||||
sock.sendall(b"0\r\n\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
if close_after_send:
|
||||
return None, b"" # обрыв загрузки: ответ не ждём
|
||||
data = b""
|
||||
try:
|
||||
while True:
|
||||
piece = sock.recv(65536)
|
||||
if not piece:
|
||||
break
|
||||
data += piece
|
||||
except (ConnectionResetError, socket.timeout):
|
||||
pass
|
||||
finally:
|
||||
sock.close()
|
||||
m = re.match(rb"HTTP/1\.[01] (\d{3})", data)
|
||||
return (int(m.group(1)) if m else None), data.split(b"\r\n\r\n", 1)[-1]
|
||||
|
||||
|
||||
def compose(*args: str, check: bool = True) -> str:
|
||||
out = subprocess.run(COMPOSE + list(args), cwd=ROOT, capture_output=True, text=True)
|
||||
if check and out.returncode != 0:
|
||||
raise RuntimeError(f"docker compose {' '.join(args)} failed: {out.stderr.strip()[-300:]}")
|
||||
return out.stdout
|
||||
|
||||
|
||||
def tool(*args: str) -> dict:
|
||||
"""python -m app.tools … внутри контейнера api → последняя строка JSON."""
|
||||
out = compose("exec", "-T", "api", "python", "-m", "app.tools", *args)
|
||||
return json.loads(out.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def janitor(*args: str, attempts: int = 10) -> dict:
|
||||
"""Один проход очистки внутри api. Если advisory lock занят фоновым
|
||||
проходом другой реплики — повторяем."""
|
||||
import time
|
||||
for _ in range(attempts):
|
||||
out = compose("exec", "-T", "api", "python", "-m", "app.janitor", "--once", *args)
|
||||
stats = json.loads(out.strip().splitlines()[-1])
|
||||
if stats.get("locked"):
|
||||
return stats
|
||||
time.sleep(1)
|
||||
raise RuntimeError("janitor lock is busy")
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Истечение сессии при коротком SESSION_TTL_SECONDS (=2).
|
||||
Запускается против отдельного экземпляра API с коротким TTL:
|
||||
docker compose --env-file .env --profile ttl up -d api-ttl
|
||||
python student-tests/expiry.py --base-url http://localhost:8081
|
||||
"""
|
||||
import argparse
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from common import Api, Report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8081")
|
||||
ap.add_argument("--ttl", type=float, default=2)
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
# Экземпляр с коротким TTL мог только что стартовать: ждём готовности до 60 с.
|
||||
deadline = time.time() + 60
|
||||
while True:
|
||||
try:
|
||||
if api.raw("GET", "/health/ready", timeout=2)[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.time() > deadline:
|
||||
raise SystemExit("API is not ready at " + args.base_url)
|
||||
time.sleep(1)
|
||||
|
||||
user = api.register("ttl")
|
||||
st, sess, _ = api.login(user["username"], user["password"])
|
||||
r.check(st == 201, "login -> 201")
|
||||
exp = datetime.fromisoformat(sess["expires_at"].replace("Z", "+00:00"))
|
||||
left = (exp - datetime.now(timezone.utc)).total_seconds()
|
||||
r.check(0 < left <= args.ttl + 1, f"expires_at is about {args.ttl}s ahead (got {left:.2f}s)")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=sess["token"])
|
||||
r.check(st == 200, "token works before expiry") # заодно кладёт сессию в кэш Redis
|
||||
time.sleep(args.ttl + 1.5)
|
||||
st, _, rh = api.call("GET", "/api/v1/users/me", token=sess["token"])
|
||||
r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Bearer"),
|
||||
"expired token -> 401 (cache does not extend the session)")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
-- План запроса истории на ≥10 000 сообщениях (уровень «4»).
|
||||
-- Всё внутри транзакции с ROLLBACK: учебные данные не остаются в базе.
|
||||
-- Запуск (одинаково в PowerShell и sh):
|
||||
-- docker compose cp student-tests/explain.sql postgres:/tmp/explain.sql
|
||||
-- docker compose exec postgres psql -U messenger -d messenger -f /tmp/explain.sql
|
||||
BEGIN;
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ('00000000-0000-4000-8000-000000000001', 'explain_seed', 'Seed', 'x', now());
|
||||
|
||||
-- Чат, историю которого читаем, и 9 «соседних» чатов, чтобы в таблице были чужие сообщения.
|
||||
INSERT INTO chats (id, title, created_by, created_at)
|
||||
SELECT ('00000000-0000-4000-8000-0000000001' || lpad(g::text, 2, '0'))::uuid, 'seed ' || g,
|
||||
'00000000-0000-4000-8000-000000000001', now()
|
||||
FROM generate_series(0, 9) g;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT ('00000000-0000-4000-8000-0000000001' || lpad(g::text, 2, '0'))::uuid,
|
||||
'00000000-0000-4000-8000-000000000001', 0
|
||||
FROM generate_series(0, 9) g;
|
||||
|
||||
-- 12 000 сообщений в целевом чате + 48 000 в соседних = 60 000 строк.
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(),
|
||||
('00000000-0000-4000-8000-0000000001' || lpad((CASE WHEN g <= 12000 THEN 0 ELSE g % 9 + 1 END)::text, 2, '0'))::uuid,
|
||||
'00000000-0000-4000-8000-000000000001',
|
||||
'msg ' || g,
|
||||
now() - interval '1 day' + g * interval '1 millisecond'
|
||||
FROM generate_series(1, 60000) g;
|
||||
|
||||
ANALYZE messages;
|
||||
|
||||
SELECT count(*) AS messages_in_target_chat
|
||||
FROM messages WHERE chat_id = '00000000-0000-4000-8000-000000000100';
|
||||
|
||||
\echo '=== 1. Первая страница (limit 50 + 1) ==='
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-000000000100'
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo '=== 2. Страница из середины истории (keyset-курсор после ~6000-го сообщения) ==='
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-000000000100'
|
||||
AND (created_at, id) > (now() - interval '1 day' + 6000 * interval '1 millisecond',
|
||||
'00000000-0000-0000-0000-000000000000')
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo '=== 3. Для сравнения: та же страница, если запретить индекс ==='
|
||||
SET LOCAL enable_indexscan = off;
|
||||
SET LOCAL enable_bitmapscan = off;
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-000000000100'
|
||||
AND (created_at, id) > (now() - interval '1 day' + 6000 * interval '1 millisecond',
|
||||
'00000000-0000-0000-0000-000000000000')
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Поочерёдная остановка КАЖДОЙ реплики API под нагрузкой (lab3, уровень «4»).
|
||||
|
||||
Для каждой реплики: дождаться, что LB обслуживает все реплики → запустить
|
||||
нагрузку записи → через --before секунд остановить реплику (docker stop, SIGTERM)
|
||||
→ ещё --after секунд нагрузки → собрать RPS, ошибки, время восстановления →
|
||||
проверить сохранность и отсутствие дублей → вернуть реплику и дождаться,
|
||||
что LB снова её обслуживает.
|
||||
|
||||
python student-tests/failover_each.py --base-url http://localhost:8080 --output evidence/failover-each.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from common import Api, Report
|
||||
from loadgen import LoadGen
|
||||
|
||||
COMPOSE = "docker compose --env-file .env"
|
||||
|
||||
|
||||
def run(cmd: str, **fmt) -> str:
|
||||
out = subprocess.run(shlex.split(cmd.format(**fmt)), check=True, capture_output=True, text=True)
|
||||
return out.stdout
|
||||
|
||||
|
||||
def served_instances(api: Api, n: int = 30) -> set:
|
||||
seen = set()
|
||||
for _ in range(n):
|
||||
try:
|
||||
st, _, rh = api.call("GET", "/health/live", timeout=2)
|
||||
if st == 200:
|
||||
seen.add(rh.get("X-Instance-Id", ""))
|
||||
except OSError:
|
||||
pass
|
||||
return seen
|
||||
|
||||
|
||||
def wait_served(api: Api, iid: str, seconds: float = 60) -> float:
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < seconds:
|
||||
if iid in served_instances(api, 10):
|
||||
return round(time.monotonic() - start, 2)
|
||||
time.sleep(0.5)
|
||||
return -1
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--workers", type=int, default=4)
|
||||
ap.add_argument("--before", type=float, default=4, help="секунд нагрузки до остановки")
|
||||
ap.add_argument("--after", type=float, default=12, help="секунд нагрузки после остановки")
|
||||
ap.add_argument("--list-cmd", default=f"{COMPOSE} ps -q api")
|
||||
ap.add_argument("--stop-cmd", default="docker stop -t 8 {id}")
|
||||
ap.add_argument("--start-cmd", default="docker start {id}")
|
||||
ap.add_argument("--output", default=None)
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
|
||||
replicas = [line.strip() for line in run(args.list_cmd).splitlines() if line.strip()]
|
||||
r.check(len(replicas) >= 2, f"at least 2 API replicas running ({len(replicas)})")
|
||||
results = []
|
||||
user = api.new_user("failover")
|
||||
for cid in replicas:
|
||||
iid = cid[:12] # X-Instance-Id = hostname контейнера = короткий ID
|
||||
print(f"== реплика {iid} ==")
|
||||
r.check(wait_served(api, iid) >= 0, f"LB serves {iid} before the test")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": f"failover {iid}", "member_ids": [user["id"]]},
|
||||
token=user["token"])
|
||||
gen = LoadGen(api, user["token"], chat["id"], workers=args.workers)
|
||||
gen.start()
|
||||
time.sleep(args.before)
|
||||
t_stop = time.monotonic()
|
||||
run(args.stop_cmd, id=cid)
|
||||
stop_cmd_seconds = round(time.monotonic() - t_stop, 2)
|
||||
time.sleep(args.after)
|
||||
gen.stop()
|
||||
rel = t_stop - gen.t0
|
||||
later = gen.instances_after(t_stop + 3)
|
||||
res = {
|
||||
"stopped_instance": iid,
|
||||
"stop_command_seconds": stop_cmd_seconds,
|
||||
"before": gen.window(0, rel),
|
||||
"first_10s_after_stop": gen.window(rel, rel + 10),
|
||||
"first_success_after_stop_s": gen.first_success_after(t_stop),
|
||||
"max_gap_without_success_after_stop_s": gen.max_gap_after(t_stop),
|
||||
"stopped_instance_served_after_3s": later.count(iid),
|
||||
"instances_after_stop": sorted(set(later)),
|
||||
**gen.verify(),
|
||||
}
|
||||
t_start = time.monotonic()
|
||||
run(args.start_cmd, id=cid)
|
||||
res["rejoin_seconds"] = wait_served(api, iid)
|
||||
res["rejoin_including_start_s"] = round(time.monotonic() - t_start, 2)
|
||||
results.append(res)
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
r.check(res["max_gap_without_success_after_stop_s"] <= 10, f"writes resumed within 10 s after stopping {iid}")
|
||||
r.check(res["stopped_instance_served_after_3s"] == 0, f"stopped {iid} not served after detection")
|
||||
r.check(res["lost_or_changed"] == 0, "confirmed messages preserved")
|
||||
r.check(res["duplicate_texts"] == 0, "no duplicates despite retries with the same key")
|
||||
r.check(res["rejoin_seconds"] >= 0, f"{iid} rejoined the LB after restart")
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump({"workload": f"{args.workers} closed-loop writers, retry same key on failure, "
|
||||
f"{args.before}s before + {args.after}s after stop",
|
||||
"replicas": results}, f, ensure_ascii=False, indent=2)
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
# Собственные fixtures
|
||||
|
||||
- `sample.jpg` — синтетический JPEG 240×160 (линии и рамка), создан программно, без EXIF и персональных данных.
|
||||
|
||||
Остальные файлы для негативных тестов (oversized, битые, «бомба» по пикселям,
|
||||
подмена MIME, шум ~9,5 МБ для эксперимента памяти) генерируются тестами
|
||||
на лету (`student-tests/common.py: make_png`) — большие файлы в Git не храним.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Общая идемпотентность POST сообщения через LB (lab3).
|
||||
|
||||
python student-tests/idempotency.py --base-url http://localhost:8080
|
||||
|
||||
Проверяет: параллельные запросы с ОДНИМ ключом через разные реплики дают одно
|
||||
сообщение; другой body с тем же ключом → 409; ключ без/неверного формата → 400;
|
||||
порядок/пробелы в JSON не важны; область ключа — пользователь + чат.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import Api, Report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--parallel", type=int, default=20)
|
||||
api, r = Api(ap.parse_args().base_url), Report()
|
||||
args = ap.parse_args()
|
||||
|
||||
alice, bob = api.new_user("idem_a"), api.new_user("idem_b")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "idem", "member_ids": [alice["id"], bob["id"]]},
|
||||
token=alice["token"])
|
||||
st, chat2, _ = api.call("POST", "/api/v1/chats", {"title": "idem2", "member_ids": [alice["id"]]},
|
||||
token=alice["token"])
|
||||
path, path2 = f"/api/v1/chats/{chat['id']}/messages", f"/api/v1/chats/{chat2['id']}/messages"
|
||||
|
||||
def history(p, token):
|
||||
st, page, _ = api.call("GET", p + "?limit=100", token=token)
|
||||
return page["items"]
|
||||
|
||||
print(f"== {args.parallel} параллельных запросов с одним ключом ==")
|
||||
key, text = uuid.uuid4().hex, "одно сообщение " + uuid.uuid4().hex[:6]
|
||||
|
||||
def send(_):
|
||||
return api.call("POST", path, {"text": text}, token=alice["token"], headers={"Idempotency-Key": key})
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
||||
res = list(pool.map(send, range(args.parallel)))
|
||||
codes = [x[0] for x in res]
|
||||
ids = {x[1]["id"] for x in res if x[0] == 201}
|
||||
instances = {x[2].get("X-Instance-Id") for x in res}
|
||||
r.check(codes.count(201) == args.parallel, f"all parallel repeats -> 201 (got {sorted(set(codes))})")
|
||||
r.check(len(ids) == 1, f"all responses carry the same message id ({len(ids)} distinct)")
|
||||
r.check(len(instances) >= 2, f"requests were served by >=2 replicas ({len(instances)})")
|
||||
r.check(sum(m["text"] == text for m in history(path, alice["token"])) == 1, "exactly one message in history")
|
||||
first = res[0][1]
|
||||
r.check(all(x[1] == first for x in res), "every replay returns the identical Message")
|
||||
|
||||
print("== семантика ключа ==")
|
||||
st, _, _ = api.call("POST", path, {"text": "другой текст"}, token=alice["token"], headers={"Idempotency-Key": key})
|
||||
r.check(st == 409, f"same key, different body -> 409 (got {st})")
|
||||
raw = ('{ "text" : ' + json.dumps(text, ensure_ascii=False) + ' }').encode()
|
||||
st, body, _ = api.raw("POST", path, raw, {"Content-Type": "application/json", "Idempotency-Key": key,
|
||||
"Authorization": "Bearer " + alice["token"]})
|
||||
r.check(st == 201 and json.loads(body)["id"] == first["id"], f"other JSON formatting = same request (got {st})")
|
||||
st, other, _ = api.call("POST", path2, {"text": text}, token=alice["token"], headers={"Idempotency-Key": key})
|
||||
r.check(st == 201 and other["id"] != first["id"], "same key in another chat is independent")
|
||||
st, bobs, _ = api.call("POST", path, {"text": text}, token=bob["token"], headers={"Idempotency-Key": key})
|
||||
r.check(st == 201 and bobs["id"] != first["id"] and bobs["sender_id"] == bob["id"],
|
||||
"same key from another user is independent")
|
||||
st, _, _ = api.raw("POST", path, json.dumps({"text": "без ключа"}).encode(),
|
||||
{"Content-Type": "application/json", "Authorization": "Bearer " + alice["token"]})
|
||||
r.check(st == 400, f"missing Idempotency-Key -> 400 (got {st})")
|
||||
for bad in ("has space", "x" * 129, "slash/key"):
|
||||
st, _, _ = api.call("POST", path, {"text": "bad key"}, token=alice["token"],
|
||||
headers={"Idempotency-Key": bad})
|
||||
r.check(st == 400, f"invalid key {bad[:12]!r}... -> 400 (got {st})")
|
||||
st, _, _ = api.call("POST", path, {"text": text}, headers={"Idempotency-Key": key})
|
||||
r.check(st == 401, f"replay without session -> 401, access re-checked (got {st})")
|
||||
|
||||
print("== неопределённый результат: клиент повторяет с тем же ключом ==")
|
||||
key2 = uuid.uuid4().hex
|
||||
try:
|
||||
api.call("POST", path, {"text": "таймаут?"}, token=alice["token"], headers={"Idempotency-Key": key2},
|
||||
timeout=0.001)
|
||||
except OSError:
|
||||
pass # ответ не дождались: записано или нет — неизвестно
|
||||
st, again, _ = api.call("POST", path, {"text": "таймаут?"}, token=alice["token"], headers={"Idempotency-Key": key2})
|
||||
r.check(st == 201, "retry after uncertain result -> 201")
|
||||
r.check(sum(m["text"] == "таймаут?" for m in history(path, alice["token"])) == 1, "still exactly one message")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,415 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Фоновая обработка (lab5): отказы, повторная доставка, outbox, retry/DLQ/replay, аренда.
|
||||
|
||||
python student-tests/jobs.py --base-url http://localhost:8080 [--scenario NAME|all]
|
||||
|
||||
Сценарии (каждый печатает ok/FAIL и итог):
|
||||
worker-down worker остановлен → 202 queued, content 409, сообщения работают;
|
||||
рестарт брокера и API с volumes; worker запущен → ready (уровень 3)
|
||||
broker-down брокер недоступен при upload → всё равно 202 (намерение в outbox),
|
||||
брокер вернулся → relay публикует → ready
|
||||
redelivery дубль сообщения уже ready-задачи → no-op, второго набора вариантов нет
|
||||
crash-s3 worker-fault падает ПОСЛЕ записи в S3, ДО COMMIT/ack → та же задача доходит до ready
|
||||
crash-ack worker-fault падает ПОСЛЕ COMMIT ready, ДО ack → повторная доставка = no-op
|
||||
outbox api-fault падает ПОСЛЕ COMMIT, ДО publish → relay публикует, ready без нового upload
|
||||
retry-replay временная ошибка на каждой попытке → backoff с jitter → failed retry_exhausted
|
||||
→ явный replay → ready; повторный replay ready-задачи отклонён
|
||||
lease «долгая, но живая» задача (heartbeat) не перехватывается;
|
||||
«замёрзший» worker теряет аренду → reaper → другая попытка → ready,
|
||||
проснувшийся старый worker ничего не перезаписывает
|
||||
|
||||
Docker-команды (по умолчанию) можно заменить файлом --hooks-file (JSON: имя → argv).
|
||||
Тест сам переключает worker ↔ worker-fault и в конце возвращает обычный worker.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from common import Api, Report, upload
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() # 320×200
|
||||
C = ["docker", "compose", "--env-file", ".env"]
|
||||
F = C + ["--profile", "fault"]
|
||||
HOOKS = {
|
||||
"stop_worker": C + ["stop", "worker"],
|
||||
"start_worker": C + ["start", "worker"],
|
||||
"start_fault_worker": F + ["up", "-d", "worker-fault"],
|
||||
"stop_fault_worker": F + ["stop", "worker-fault"],
|
||||
"start_fault_api": F + ["up", "-d", "api-fault"],
|
||||
"restart_api": C + ["restart", "api"],
|
||||
"restart_broker": C + ["restart", "broker"],
|
||||
"stop_broker": C + ["stop", "broker"],
|
||||
"start_broker": C + ["start", "broker"],
|
||||
"tool": C + ["exec", "-T", "api", "python", "-m", "app.tools"],
|
||||
"replay": C + ["exec", "-T", "api", "python", "-m", "app.replay"],
|
||||
"redis": C + ["exec", "-T", "broker", "redis-cli"],
|
||||
}
|
||||
FAULT_BASE = "http://localhost:8082"
|
||||
|
||||
|
||||
def sh(name: str, *args: str, check: bool = True) -> str:
|
||||
out = subprocess.run(HOOKS[name] + list(args), cwd=ROOT, capture_output=True, text=True)
|
||||
if check and out.returncode != 0:
|
||||
raise RuntimeError(f"{name} {' '.join(args)}: {out.stderr.strip()[-300:]}")
|
||||
return out.stdout
|
||||
|
||||
|
||||
def tool(*args: str):
|
||||
out = sh("tool", *args)
|
||||
text = out.strip()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except ValueError:
|
||||
return json.loads(text.splitlines()[-1])
|
||||
|
||||
|
||||
def fault(*args: str) -> None:
|
||||
sh("redis", *args)
|
||||
|
||||
|
||||
class Ctx:
|
||||
def __init__(self, base: str):
|
||||
self.api = Api(base)
|
||||
self.r = Report()
|
||||
self.alice = self.api.new_user("jobs_a")
|
||||
st, chat, _ = self.api.call("POST", "/api/v1/chats", {"title": "jobs", "member_ids": [self.alice["id"]]},
|
||||
token=self.alice["token"])
|
||||
assert st == 201, st
|
||||
self.chat = chat
|
||||
|
||||
def upload(self, data: bytes = FIXTURE, base: str | None = None, headers: dict | None = None):
|
||||
api = Api(base) if base else self.api
|
||||
return upload(api, self.alice["token"], self.chat["id"], data, "sample.png", headers=headers)
|
||||
|
||||
def get(self, att_id: str) -> dict:
|
||||
st, att, _ = self.api.call("GET", f"/api/v1/attachments/{att_id}", token=self.alice["token"])
|
||||
assert st == 200, st
|
||||
return att
|
||||
|
||||
def wait(self, att_id: str, statuses=("ready",), seconds: float = 90) -> dict:
|
||||
deadline = time.monotonic() + seconds
|
||||
att = self.get(att_id)
|
||||
while time.monotonic() < deadline and att["status"] not in statuses:
|
||||
time.sleep(0.5)
|
||||
att = self.get(att_id)
|
||||
return att
|
||||
|
||||
def content(self, att_id: str, variant: str = "original"):
|
||||
return self.api.raw("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}",
|
||||
headers={"Authorization": "Bearer " + self.alice["token"]})
|
||||
|
||||
def check_ready_result(self, att: dict, label: str) -> None:
|
||||
r = self.r
|
||||
names = [v["name"] for v in att["variants"]]
|
||||
r.check(att["status"] == "ready" and att["error_code"] is None, f"{label}: status ready, error_code null")
|
||||
r.check(names == ["thumbnail", "crop", "optimized"], f"{label}: variants thumbnail+crop+optimized ({names})")
|
||||
st, body, rh = self.content(att["id"])
|
||||
r.check(st == 200 and body == FIXTURE, f"{label}: original byte-for-byte")
|
||||
th = {v["name"]: v for v in att["variants"]}.get("thumbnail")
|
||||
st, tb, rh = self.content(att["id"], "thumbnail")
|
||||
r.check(st == 200 and th and (th["width"], th["height"]) == (256, 160)
|
||||
and hashlib.sha256(tb).hexdigest() == th["sha256"], f"{label}: thumbnail 256x160, sha256 matches")
|
||||
|
||||
def job(self, att_id: str) -> dict:
|
||||
return tool("job", att_id)
|
||||
|
||||
def use_fault_worker(self) -> None:
|
||||
sh("stop_worker")
|
||||
sh("start_fault_worker")
|
||||
time.sleep(3)
|
||||
|
||||
def use_normal_worker(self) -> None:
|
||||
sh("stop_fault_worker", check=False)
|
||||
sh("start_worker")
|
||||
|
||||
|
||||
# ======================================================================== сценарии
|
||||
def worker_down(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== worker остановлен: приём не зависит от обработки ==")
|
||||
sh("stop_worker")
|
||||
t0 = time.monotonic()
|
||||
st, att, _ = c.upload()
|
||||
took = time.monotonic() - t0
|
||||
r.check(st == 202 and att["status"] == "queued" and att["variants"] == [],
|
||||
f"upload → 202 queued without worker ({took:.2f}s)")
|
||||
aid = att["id"]
|
||||
time.sleep(3)
|
||||
r.check(c.get(aid)["status"] == "queued", "still queued after 3 s: API did not resize synchronously")
|
||||
st, _, _ = c.content(aid)
|
||||
r.check(st == 409, f"content of queued attachment → 409 (got {st})")
|
||||
st, _, _ = c.content(aid, "thumbnail")
|
||||
r.check(st == 409, f"thumbnail of queued attachment → 409 (got {st})")
|
||||
st, msg, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages",
|
||||
{"text": "", "attachment_ids": [aid]}, token=c.alice["token"])
|
||||
r.check(st == 201, f"message with queued attachment → 201 (got {st})")
|
||||
st, _, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages", {"text": "другие сообщения работают"},
|
||||
token=c.alice["token"])
|
||||
r.check(st == 201, "other messages keep working")
|
||||
q = tool("queue")
|
||||
r.check(q["stream_length"] >= 1 and q["outbox_unpublished"] == 0,
|
||||
f"job is in the broker stream (len={q['stream_length']}), outbox published")
|
||||
|
||||
print("== рестарт брокера и API с сохранением volumes ==")
|
||||
sh("restart_broker")
|
||||
sh("restart_api")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if c.api.raw("GET", "/health/ready")[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
q = tool("queue")
|
||||
r.check(q["stream_length"] >= 1, f"after broker restart the message is still there (len={q['stream_length']})")
|
||||
st, same, _ = c.api.call("GET", f"/api/v1/attachments/{aid}", token=c.alice["token"])
|
||||
r.check(st == 200 and same["status"] == "queued", "same session, same attachment, still queued")
|
||||
|
||||
print("== worker запущен ==")
|
||||
sh("start_worker")
|
||||
att = c.wait(aid, ("ready", "failed"), 60)
|
||||
c.check_ready_result(att, "after restart")
|
||||
time.sleep(3)
|
||||
j = c.job(aid)
|
||||
r.check(not j["staging_object_exists"] and not j["attachment"]["staging_present"],
|
||||
"staging removed after ready (original lives in originals/)")
|
||||
print(f" attachment {aid}, job {j['job']['job_id']}, events: {j['events']}")
|
||||
|
||||
|
||||
def broker_down(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== брокер недоступен: приём не зависит от брокера (transactional outbox) ==")
|
||||
sh("stop_broker")
|
||||
try:
|
||||
r.check(c.api.raw("GET", "/health/ready")[0] == 200, "/health/ready = 200: broker is not critical for API")
|
||||
t0 = time.monotonic()
|
||||
st, att, _ = c.upload()
|
||||
r.check(st == 202 and att["status"] == "queued",
|
||||
f"upload → 202 queued while broker is down ({time.monotonic() - t0:.1f}s, publish timeout ≤ 2s)")
|
||||
aid = att["id"]
|
||||
j = c.job(aid)
|
||||
r.check(len(j["outbox"]) == 1 and not j["outbox"][0]["published"], "intent kept in outbox, not published yet")
|
||||
finally:
|
||||
sh("start_broker")
|
||||
att = c.wait(aid, ("ready", "failed"), 60)
|
||||
c.check_ready_result(att, "after broker is back")
|
||||
j = c.job(aid)
|
||||
r.check(j["outbox"][0]["published"], "relay published the outbox row when broker came back")
|
||||
|
||||
|
||||
def redelivery(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== повторная доставка уже ready-задачи ==")
|
||||
st, att, _ = c.upload()
|
||||
att = c.wait(att["id"])
|
||||
before = c.job(att["id"])
|
||||
for _ in range(2):
|
||||
print(" ", sh("tool", "republish", att["id"]).strip()[:120])
|
||||
time.sleep(4)
|
||||
after = c.job(att["id"])
|
||||
same = c.get(att["id"])
|
||||
r.check(same["status"] == "ready" and same["variants"] == att["variants"],
|
||||
"duplicates: status stays ready, variants unchanged (same sha256)")
|
||||
r.check(len(after["variant_rows"]) == 3 and after["variant_objects_in_s3"] == 3,
|
||||
f"no second set of variants: rows={len(after['variant_rows'])}, S3 objects={after['variant_objects_in_s3']}")
|
||||
r.check(sum("ready" in e.split()[1:2] for e in after["events"]) == 1
|
||||
and after["job"]["attempt"] == before["job"]["attempt"],
|
||||
"job finished exactly once, attempt unchanged")
|
||||
q = tool("queue")
|
||||
r.check(all(g["pending"] == 0 for g in q["groups"]), "duplicates acked (no pending messages)")
|
||||
|
||||
|
||||
def crash_s3(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== crash после записи в S3, до COMMIT и ack ==")
|
||||
c.use_fault_worker()
|
||||
try:
|
||||
fault("SET", "fault:crash-after-s3-write", "1")
|
||||
st, att, _ = c.upload()
|
||||
t0 = time.monotonic()
|
||||
att = c.wait(att["id"], ("ready", "failed"), 120)
|
||||
print(f" recovered in {time.monotonic() - t0:.1f}s (worker restart + lease expiry)")
|
||||
c.check_ready_result(att, "after crash before ack")
|
||||
j = c.job(att["id"])
|
||||
r.check(any("lease_takeover" in e for e in j["events"]) and j["job"]["attempt"] == 1,
|
||||
"same attempt taken over after lease expiry (redelivery, not a new job)")
|
||||
r.check(len(j["variant_rows"]) == 3 and j["variant_objects_in_s3"] == 3,
|
||||
"S3 objects from the crashed attempt reused: still exactly 3 variants")
|
||||
print(f" events: {j['events']}")
|
||||
finally:
|
||||
fault("DEL", "fault:crash-after-s3-write")
|
||||
c.use_normal_worker()
|
||||
|
||||
|
||||
def crash_ack(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== crash после COMMIT ready, до ack ==")
|
||||
c.use_fault_worker()
|
||||
try:
|
||||
fault("SET", "fault:crash-after-commit-before-ack", "1")
|
||||
st, att, _ = c.upload()
|
||||
att = c.wait(att["id"], ("ready", "failed"), 60)
|
||||
c.check_ready_result(att, "commit before crash")
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
q = tool("queue")
|
||||
if all(g["pending"] == 0 for g in q["groups"]):
|
||||
break
|
||||
time.sleep(2)
|
||||
r.check(all(g["pending"] == 0 for g in q["groups"]),
|
||||
"unacked message redelivered after restart and acked as no-op")
|
||||
j = c.job(att["id"])
|
||||
r.check(sum(e.split()[1] == "ready" for e in j["events"]) == 1 and len(j["variant_rows"]) == 3,
|
||||
"ready recorded once, 3 variants")
|
||||
finally:
|
||||
fault("DEL", "fault:crash-after-commit-before-ack")
|
||||
c.use_normal_worker()
|
||||
|
||||
|
||||
def outbox(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== crash API между COMMIT и publish (transactional outbox) ==")
|
||||
sh("start_fault_api")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if Api(FAULT_BASE).raw("GET", "/health/ready")[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
before = {a["id"] for a in tool("chat-attachments", c.chat["id"])}
|
||||
try:
|
||||
st, body, _ = c.upload(base=FAULT_BASE, headers={"X-Fault-Inject": "crash-after-commit-before-publish"})
|
||||
except OSError as exc:
|
||||
st, body = None, str(exc)
|
||||
r.check(st != 202, f"client did NOT get 202: API died after COMMIT (got {st})")
|
||||
new = [a for a in tool("chat-attachments", c.chat["id"]) if a["id"] not in before]
|
||||
r.check(len(new) == 1, f"attachment and job were committed before the crash ({len(new)} new)")
|
||||
if new:
|
||||
aid = new[0]["id"]
|
||||
att = c.wait(aid, ("ready", "failed"), 60)
|
||||
c.check_ready_result(att, "relay-published job")
|
||||
j = c.job(aid)
|
||||
r.check(all(o["published"] for o in j["outbox"]) and len(j["outbox"]) == 1,
|
||||
"outbox row published by relay (no new job created)")
|
||||
|
||||
|
||||
def retry_replay(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== временные ошибки → backoff+jitter → failed → replay ==")
|
||||
c.use_fault_worker()
|
||||
try:
|
||||
fault("SET", "fault:transient-error", "100")
|
||||
st, att, _ = c.upload()
|
||||
aid = att["id"]
|
||||
t0 = time.monotonic()
|
||||
att = c.wait(aid, ("failed", "ready"), 150)
|
||||
r.check(att["status"] == "failed" and att["error_code"] == "retry_exhausted" and att["variants"] == [],
|
||||
f"after {time.monotonic() - t0:.0f}s: failed, error_code=retry_exhausted")
|
||||
j = c.job(aid)
|
||||
delays = [float(e.split(" in ")[-1].rstrip("s")) for e in j["events"] if "retry_scheduled" in e]
|
||||
r.check(j["job"]["attempt"] == j["job"]["max_attempts"] == 5 and len(delays) == 4,
|
||||
f"bounded: 5 attempts, 4 scheduled retries, delays {delays}")
|
||||
base = [1, 2, 4, 8]
|
||||
r.check(all(b / 2 <= d <= b * 1.01 for b, d in zip(base, delays)),
|
||||
"backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1)")
|
||||
st, _, _ = c.content(aid)
|
||||
r.check(st == 409, f"content of failed → 409 (got {st})")
|
||||
st, _, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages",
|
||||
{"text": "x", "attachment_ids": [aid]}, token=c.alice["token"])
|
||||
r.check(st == 409, f"failed attachment cannot be attached → 409 (got {st})")
|
||||
listed = sh("replay", "--list")
|
||||
r.check(aid in listed, "failed job is in the registry (python -m app.replay --list)")
|
||||
|
||||
fault("DEL", "fault:transient-error")
|
||||
out = sh("replay", "--attachment", aid, "--actor", "jobs-test")
|
||||
r.check("replayed" in out, "explicit replay accepted")
|
||||
att = c.wait(aid, ("ready", "failed"), 60)
|
||||
c.check_ready_result(att, "after replay")
|
||||
j = c.job(aid)
|
||||
r.check(j["job"]["attempt"] == 6 and j["job"]["replays"] == 1, "same job_id, attempt 6, replays=1 (audit)")
|
||||
out = sh("replay", "--attachment", aid)
|
||||
r.check("refused" in out, "replay of a ready job is refused (no duplicate execution)")
|
||||
finally:
|
||||
fault("DEL", "fault:transient-error")
|
||||
c.use_normal_worker()
|
||||
|
||||
|
||||
def lease(c: Ctx) -> None:
|
||||
r = c.r
|
||||
print("== долгая, но живая задача: heartbeat продлевает аренду ==")
|
||||
c.use_fault_worker()
|
||||
try:
|
||||
fault("SET", "fault:slow-seconds", "30")
|
||||
t0 = time.monotonic()
|
||||
st, att, _ = c.upload()
|
||||
att = c.wait(att["id"], ("ready", "failed"), 90)
|
||||
took = time.monotonic() - t0
|
||||
fault("DEL", "fault:slow-seconds")
|
||||
j = c.job(att["id"])
|
||||
r.check(att["status"] == "ready" and took > 25 and j["job"]["attempt"] == 1
|
||||
and not any("takeover" in e or "lease_expired" in e for e in j["events"]),
|
||||
f"job ran {took:.0f}s > lease 20s, no takeover, attempt 1")
|
||||
|
||||
print("== замёрзший worker: аренда истекает, задачу забирают ==")
|
||||
fault("SET", "fault:freeze-after-claim", "1")
|
||||
st, att, _ = c.upload()
|
||||
aid = att["id"]
|
||||
time.sleep(3)
|
||||
r.check(c.get(aid)["status"] == "processing", "claimed by worker-fault, then it froze (processing)")
|
||||
sh("start_worker") # второй, здоровый worker
|
||||
t0 = time.monotonic()
|
||||
att = c.wait(aid, ("ready", "failed"), 120)
|
||||
print(f" recovered in {time.monotonic() - t0:.1f}s")
|
||||
c.check_ready_result(att, "after frozen worker")
|
||||
j = c.job(aid)
|
||||
r.check(j["job"]["attempt"] == 2 and any("lease_expired" in e for e in j["events"]),
|
||||
"reaper: lease expired → attempt 2 → ready")
|
||||
time.sleep(max(0, 65 - (time.monotonic() - t0))) # frozen worker wakes up (3×lease after claim)
|
||||
j2 = c.job(aid)
|
||||
r.check(sum(e.split()[1] == "ready" for e in j2["events"]) == 1 and len(j2["variant_rows"]) == 3
|
||||
and c.get(aid)["variants"] == att["variants"],
|
||||
"woken stale worker could not commit (lease_token fencing), result unchanged")
|
||||
finally:
|
||||
fault("DEL", "fault:slow-seconds")
|
||||
fault("DEL", "fault:freeze-after-claim")
|
||||
c.use_normal_worker()
|
||||
|
||||
|
||||
SCENARIOS = {"worker-down": worker_down, "broker-down": broker_down, "redelivery": redelivery, "crash-s3": crash_s3,
|
||||
"crash-ack": crash_ack, "outbox": outbox, "retry-replay": retry_replay, "lease": lease}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--scenario", default="all", choices=["all", *SCENARIOS])
|
||||
ap.add_argument("--hooks-file", help="JSON {hook: argv} вместо docker compose")
|
||||
args = ap.parse_args()
|
||||
if args.hooks_file:
|
||||
HOOKS.update(json.loads(Path(args.hooks_file).read_text(encoding="utf-8")))
|
||||
c = Ctx(args.base_url)
|
||||
names = list(SCENARIOS) if args.scenario == "all" else [args.scenario]
|
||||
for name in names:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
SCENARIOS[name](c)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
c.r.check(False, f"{name}: crashed with {type(exc).__name__}: {exc}")
|
||||
print(f" [{name}: {time.monotonic() - t0:.0f}s]")
|
||||
c.r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
main()
|
||||
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
Генератор записей для экспериментов отказа и rolling update (lab3, уровень «4»).
|
||||
|
||||
Несколько потоков в замкнутом цикле отправляют сообщения через LB. У каждого
|
||||
сообщения свой Idempotency-Key; если ответ не получен (ошибка/таймаут/5xx),
|
||||
поток повторяет ТОТ ЖЕ запрос с ТЕМ ЖЕ ключом — так проверяется, что
|
||||
неопределённая запись безопасно повторяется и не даёт дублей.
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from common import Api
|
||||
|
||||
|
||||
class LoadGen:
|
||||
def __init__(self, api: Api, token: str, chat_id: str, workers: int = 4, timeout: float = 2.0):
|
||||
self.api, self.token, self.chat_id = api, token, chat_id
|
||||
self.path = f"/api/v1/chats/{chat_id}/messages"
|
||||
self.workers, self.timeout = workers, timeout
|
||||
self.stop_event = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.ok = [] # (t, instance, message)
|
||||
self.errors = [] # (t, reason)
|
||||
self.retried = 0 # сколько раз повторяли тот же ключ
|
||||
self.threads = []
|
||||
self.t0 = None
|
||||
|
||||
def _worker(self, w: int):
|
||||
i = 0
|
||||
while not self.stop_event.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
text = f"load-{w}-{i}-{key[:8]}"
|
||||
first = True
|
||||
while not self.stop_event.is_set():
|
||||
if not first:
|
||||
with self.lock:
|
||||
self.retried += 1
|
||||
first = False
|
||||
try:
|
||||
st, body, rh = self.api.call("POST", self.path, {"text": text}, token=self.token,
|
||||
headers={"Idempotency-Key": key}, timeout=self.timeout)
|
||||
except OSError as exc:
|
||||
st, body, rh = None, None, None
|
||||
reason = type(exc).__name__
|
||||
else:
|
||||
reason = f"HTTP {st}"
|
||||
now = time.monotonic()
|
||||
if st == 201:
|
||||
with self.lock:
|
||||
self.ok.append((now, rh.get("X-Instance-Id", ""), body))
|
||||
break
|
||||
with self.lock:
|
||||
self.errors.append((now, reason))
|
||||
time.sleep(0.1) # повтор с тем же ключом
|
||||
i += 1
|
||||
|
||||
def start(self):
|
||||
self.t0 = time.monotonic()
|
||||
for w in range(self.workers):
|
||||
t = threading.Thread(target=self._worker, args=(w,), daemon=True)
|
||||
t.start()
|
||||
self.threads.append(t)
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=self.timeout + 2)
|
||||
|
||||
# ---------------------------------------------------------------- метрики
|
||||
def window(self, a: float, b: float) -> dict:
|
||||
"""Успехи/ошибки в окне [a, b) секунд от старта нагрузки."""
|
||||
ok = [x for x in self.ok if a <= x[0] - self.t0 < b]
|
||||
err = [x for x in self.errors if a <= x[0] - self.t0 < b]
|
||||
dur = max(b - a, 1e-9)
|
||||
return {"successful": len(ok), "errors": len(err), "successful_rps": round(len(ok) / dur, 1)}
|
||||
|
||||
def max_gap_after(self, t_abs: float) -> float:
|
||||
"""Самый длинный промежуток без успешной записи после момента t_abs (сек)."""
|
||||
times = [t_abs] + sorted(x[0] for x in self.ok if x[0] >= t_abs)
|
||||
if len(times) == 1:
|
||||
return float("inf")
|
||||
return round(max(b - a for a, b in zip(times, times[1:])), 3)
|
||||
|
||||
def first_success_after(self, t_abs: float):
|
||||
later = sorted(x[0] for x in self.ok if x[0] >= t_abs)
|
||||
return round(later[0] - t_abs, 3) if later else None
|
||||
|
||||
def instances_after(self, t_abs: float) -> list:
|
||||
return [x[1] for x in sorted(self.ok) if x[0] >= t_abs]
|
||||
|
||||
def verify(self) -> dict:
|
||||
"""Все подтверждённые сообщения есть в истории без изменений, дублей нет."""
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
q = "?limit=100" + (f"&cursor={quote(cursor, safe='')}" if cursor else "")
|
||||
st, page, _ = self.api.call("GET", self.path + q, token=self.token)
|
||||
assert st == 200, f"history read failed: {st}"
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
by_id = {m["id"]: m for m in items}
|
||||
texts = [m["text"] for m in items]
|
||||
confirmed = [x[2] for x in self.ok]
|
||||
lost = [m for m in confirmed if by_id.get(m["id"]) != m]
|
||||
return {"history_messages": len(items), "confirmed": len(confirmed),
|
||||
"lost_or_changed": len(lost), "duplicate_texts": len(texts) - len(set(texts)),
|
||||
"retries_with_same_key": self.retried}
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Logout при приостановленном Redis (замечание к app/sessions.py).
|
||||
|
||||
Сценарий преподавателя: прогреть сессию в кэше, приостановить Redis, сделать
|
||||
logout, восстановить Redis и проверить старый токен.
|
||||
Требование: 204 только при гарантированном отзыве; после 204 старый токен
|
||||
никогда не принимается, в том числе после восстановления Redis.
|
||||
|
||||
python student-tests/logout_cache.py --base-url http://localhost:8080
|
||||
(по умолчанию сам делает docker compose pause/unpause redis)
|
||||
"""
|
||||
import argparse
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from common import Api, Report
|
||||
|
||||
COMPOSE = "docker compose --env-file .env"
|
||||
|
||||
|
||||
def sh(cmd: str):
|
||||
print(f" $ {cmd}")
|
||||
subprocess.run(shlex.split(cmd), check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--pause-redis", default=f"{COMPOSE} pause redis")
|
||||
ap.add_argument("--unpause-redis", default=f"{COMPOSE} unpause redis")
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
|
||||
user = api.new_user("logoutcache")
|
||||
token = user["token"]
|
||||
for _ in range(2): # первый запрос кладёт сессию в Redis, второй читает из кэша
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=token)
|
||||
r.check(st == 200, "session is warmed up in cache")
|
||||
|
||||
print("== Redis приостановлен ==")
|
||||
sh(args.pause_redis)
|
||||
try:
|
||||
st_logout, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=token)
|
||||
r.check(st_logout != 204,
|
||||
f"logout is NOT confirmed while cache cannot be invalidated (got {st_logout}, expected 503)")
|
||||
finally:
|
||||
sh(args.unpause_redis)
|
||||
time.sleep(0.5)
|
||||
|
||||
print("== Redis восстановлен ==")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=token)
|
||||
if st_logout == 204:
|
||||
r.check(st == 401, f"after a 204 the old token is rejected after Redis recovery (got {st})")
|
||||
else:
|
||||
r.check(st == 200, f"unconfirmed logout left the session consistent and valid (got {st})")
|
||||
st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=token)
|
||||
r.check(st == 204, f"retry logout after Redis recovery -> 204 (got {st})")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=token)
|
||||
r.check(st == 401, f"old token after confirmed logout -> 401, no stale cache copy (got {st})")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=token)
|
||||
r.check(st == 401, f"old token still rejected on repeat (got {st})")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Согласованность S3 ↔ PostgreSQL при сбоях (lab4, уровень «4»).
|
||||
Сам управляет контейнерами через docker compose; запускать из корня репозитория:
|
||||
|
||||
docker compose --env-file .env --profile fault up -d api-fault
|
||||
python student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082
|
||||
|
||||
Сценарии:
|
||||
1. Прерванная загрузка (клиент оборвал тело) → в S3 и журнале ничего не появилось.
|
||||
2. PUT в S3 прошёл, запись в БД упала (db-error-after-s3-put) → 503, API сразу
|
||||
компенсирует: объект и строка журнала удалены, вложения нет.
|
||||
3. PUT прошёл, процесс убит до COMMIT (crash-after-s3-put, os._exit) → остаётся
|
||||
объект + строка upload_intents (известная сирота). Janitor:
|
||||
- с рабочим grace period (900 c) её НЕ трогает (может быть живая загрузка);
|
||||
- dry-run показывает, что удалит, и ничего не меняет;
|
||||
- после истечения grace удаляет объект и строку; повтор — без изменений.
|
||||
4. Объект без журнала и без метаданных (страховка) → находится обходом bucket
|
||||
(--scan-bucket) и удаляется только после grace.
|
||||
5. Готовые вложения не удаляются никогда: после всех проходов скачиваются, sha256 совпадает.
|
||||
"""
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from common import SAMPLE_PNG, Api, Report, compose, janitor, raw_http, sha256, tool, upload, wait_ready as att_ready
|
||||
|
||||
|
||||
def wait_ready(api: Api, seconds: int = 60) -> bool:
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if api.raw("GET", "/health/ready", timeout=3)[0] == 200:
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--fault-url", default="http://localhost:8082")
|
||||
ap.add_argument("--grace", type=int, default=5, help="маленький grace для теста (с)")
|
||||
args = ap.parse_args()
|
||||
api, fault, r = Api(args.base_url), Api(args.fault_url), Report()
|
||||
r.check(wait_ready(fault, 60), "fault replica (profile fault) is ready on --fault-url")
|
||||
|
||||
user = api.new_user("orphan")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "orphans", "member_ids": [user["id"]]},
|
||||
token=user["token"])
|
||||
st, keep, _ = upload(api, user["token"], chat["id"], SAMPLE_PNG, "keep.png")
|
||||
r.check(st == 202, "committed attachment that must survive every cleanup")
|
||||
r.check(att_ready(api, user["token"], keep["id"]).get("status") == "ready", "it is processed to ready (lab5)")
|
||||
# Сначала дочищаем всё старое, чтобы считать объекты от нуля.
|
||||
janitor("--grace-seconds", "0", "--scan-bucket")
|
||||
base = tool("stats")
|
||||
print(" baseline:", base)
|
||||
|
||||
print("== 1. прерванная загрузка ==")
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=cut.png"
|
||||
raw_http(args.base_url, "POST", path,
|
||||
{"Authorization": "Bearer " + user["token"], "Content-Type": "image/png",
|
||||
"Content-Length": str(len(SAMPLE_PNG))},
|
||||
chunks=[SAMPLE_PNG[:20000]], close_after_send=True)
|
||||
time.sleep(2)
|
||||
now = tool("stats")
|
||||
r.check(now == base, f"no object, no metadata, no journal row after interrupted upload ({now})")
|
||||
|
||||
print("== 2. отказ БД после PUT: немедленная компенсация ==")
|
||||
st, body, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "dbfail.png",
|
||||
headers={"X-Fault-Inject": "db-error-after-s3-put"})
|
||||
r.check(st == 503, f"upload answered 503, not 202 (got {st})")
|
||||
now = tool("stats")
|
||||
r.check(now == base, f"object deleted by compensation, journal clean ({now})")
|
||||
|
||||
print("== 3. процесс убит между PUT и COMMIT ==")
|
||||
try:
|
||||
st, _, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "crash.png",
|
||||
headers={"X-Fault-Inject": "crash-after-s3-put"}, timeout=20)
|
||||
except OSError:
|
||||
st = None
|
||||
r.check(st != 202, f"client did NOT get 202 (got {st})")
|
||||
time.sleep(1)
|
||||
now = tool("stats")
|
||||
r.check(now["objects"] == base["objects"] + 1 and now["upload_intents"] == base["upload_intents"] + 1
|
||||
and now["attachments"] == base["attachments"],
|
||||
f"orphan object + journal row left behind, no metadata ({now})")
|
||||
s = janitor() # рабочий grace period
|
||||
r.check(s["objects_deleted"] == 0 and tool("stats") == now,
|
||||
f"default grace {s['grace_seconds']} s: fresh upload is NOT touched")
|
||||
time.sleep(args.grace + 1)
|
||||
s = janitor("--grace-seconds", str(args.grace), "--dry-run")
|
||||
r.check(s["would_delete_objects"] == 1 and tool("stats") == now, "dry-run reports 1 object, changes nothing")
|
||||
s = janitor("--grace-seconds", str(args.grace))
|
||||
after = tool("stats")
|
||||
r.check(s["objects_deleted"] == 1 and after == base, f"janitor removed orphan + journal row ({s})")
|
||||
s = janitor("--grace-seconds", str(args.grace))
|
||||
r.check(s["objects_deleted"] == 0 and tool("stats") == base, "second run is a no-op (idempotent)")
|
||||
|
||||
print("== 4. объект без журнала (страховочный обход bucket) ==")
|
||||
compose("exec", "-T", "api", "python", "-c",
|
||||
"import uuid; from app import storage; storage.init(); "
|
||||
"storage.client().put_object(Bucket=storage.settings.s3_bucket, "
|
||||
"Key=storage.object_key(uuid.uuid4()), Body=b'lost')")
|
||||
r.check(tool("stats")["objects"] == base["objects"] + 1, "untracked object planted")
|
||||
s = janitor("--scan-bucket", "--grace-seconds", "3600")
|
||||
r.check(s["orphans_found"] == 0, "scan with grace 1 h ignores a fresh object")
|
||||
time.sleep(args.grace + 1)
|
||||
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace), "--dry-run")
|
||||
r.check(s["orphans_found"] == 1 and tool("stats")["objects"] == base["objects"] + 1, "scan dry-run finds 1")
|
||||
s = janitor("--scan-bucket", "--grace-seconds", str(args.grace))
|
||||
r.check(s["objects_deleted"] == 1 and tool("stats") == base, f"scan deletes the untracked object ({s})")
|
||||
|
||||
print("== 5. готовые вложения не тронуты ==")
|
||||
st, raw, _ = api.raw("GET", f"/api/v1/attachments/{keep['id']}/content",
|
||||
headers={"Authorization": "Bearer " + user["token"]})
|
||||
r.check(st == 200 and sha256(raw) == keep["sha256"], "committed attachment still downloads, sha256 matches")
|
||||
state = tool("state", keep["id"])
|
||||
r.check(state == {"staging_object_exists": False, "original_object_exists": True,
|
||||
"attachment_row": True, "attachment_status": "ready", "upload_intent": False},
|
||||
f"published original + metadata, staging cleaned, no journal row ({state})")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Недоступность зависимостей (уровень «4»). Сам останавливает и запускает контейнеры:
|
||||
python student-tests/outage.py --base-url http://localhost:8080
|
||||
|
||||
1. Redis остановлен → API работает в обход кэша (сессии читаются из PostgreSQL).
|
||||
2. PostgreSQL остановлен → защищённые запросы получают ограниченную ошибку 503,
|
||||
неизвестный токен НЕ принимается (никогда 200), /health/ready = 503.
|
||||
3. PostgreSQL запущен снова → тот же токен снова работает, данные на месте.
|
||||
"""
|
||||
import argparse
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import Api, Report
|
||||
|
||||
COMPOSE = "docker compose --env-file .env"
|
||||
|
||||
|
||||
def sh(cmd: str):
|
||||
print(f" $ {cmd}")
|
||||
subprocess.run(shlex.split(cmd), check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_ready(api: Api, seconds=90):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if api.raw("GET", "/health/ready", timeout=3)[0] == 200:
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--stop-redis", default=f"{COMPOSE} stop redis")
|
||||
ap.add_argument("--start-redis", default=f"{COMPOSE} start redis")
|
||||
ap.add_argument("--stop-db", default=f"{COMPOSE} stop postgres")
|
||||
ap.add_argument("--start-db", default=f"{COMPOSE} start postgres")
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
|
||||
user = api.new_user("outage")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "outage", "member_ids": [user["id"]]}, token=user["token"])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
api.call("POST", path, {"text": "before outage"}, token=user["token"])
|
||||
|
||||
print("== 1. Redis недоступен ==")
|
||||
sh(args.stop_redis)
|
||||
try:
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=user["token"])
|
||||
r.check(st == 200, "valid token still works without Redis (cache bypassed)")
|
||||
st, _, _ = api.call("POST", path, {"text": "without redis"}, token=user["token"])
|
||||
r.check(st == 201, "writes still work without Redis")
|
||||
fresh = api.register("noredis")
|
||||
st, sess, _ = api.login(fresh["username"], fresh["password"])
|
||||
r.check(st == 201, "login works without Redis")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex)
|
||||
r.check(st == 401, "forged token still rejected without Redis")
|
||||
finally:
|
||||
sh(args.start_redis)
|
||||
|
||||
print("== 2. PostgreSQL недоступен ==")
|
||||
sh(args.stop_db)
|
||||
try:
|
||||
time.sleep(1)
|
||||
st, _, _ = api.call("GET", path, token=user["token"])
|
||||
r.check(st == 503, f"history without DB -> 503, not a hang (got {st})")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex)
|
||||
r.check(st in (401, 503), f"unknown token is never trusted without DB (got {st})")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me")
|
||||
r.check(st == 401, "anonymous request -> 401")
|
||||
st = api.raw("GET", "/health/ready")[0]
|
||||
r.check(st == 503, "/health/ready -> 503")
|
||||
st = api.raw("GET", "/health/live")[0]
|
||||
r.check(st == 200, "/health/live -> 200 (process is alive)")
|
||||
finally:
|
||||
sh(args.start_db)
|
||||
|
||||
print("== 3. восстановление ==")
|
||||
r.check(wait_ready(api), "/health/ready back to 200")
|
||||
st, page, _ = api.call("GET", path, token=user["token"])
|
||||
texts = [m["text"] for m in (page or {}).get("items", [])]
|
||||
r.check(st == 200 and "before outage" in texts and "without redis" in texts,
|
||||
"same token reads the same history after DB is back")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Конкурентная запись: 10 клиентов параллельно отправляют по 10 сообщений.
|
||||
Ожидаем ровно 100 уникальных сообщений в истории.
|
||||
python student-tests/race.py --base-url http://localhost:8080
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import Api
|
||||
|
||||
CLIENTS, PER_CLIENT = 10, 10
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
api = Api(ap.parse_args().base_url)
|
||||
user = api.new_user("race")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "race", "member_ids": [user["id"]]}, token=user["token"])
|
||||
mpath = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def worker(w):
|
||||
return [api.call("POST", mpath, {"text": f"c{w}-m{i}"}, token=user["token"])[0] for i in range(PER_CLIENT)]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
codes = [c for batch in pool.map(worker, range(CLIENTS)) for c in batch]
|
||||
|
||||
ids, texts, cursor = [], [], None
|
||||
while True:
|
||||
st, page, _ = api.call("GET", f"{mpath}?limit=50" + (f"&cursor={cursor}" if cursor else ""), token=user["token"])
|
||||
ids += [m["id"] for m in page["items"]]
|
||||
texts += [m["text"] for m in page["items"]]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
expected = {f"c{w}-m{i}" for w in range(CLIENTS) for i in range(PER_CLIENT)}
|
||||
ok = codes.count(201) == 100 and len(ids) == 100 and len(set(ids)) == 100 and set(texts) == expected
|
||||
print(f"sent=100 accepted={codes.count(201)} received={len(ids)} unique_ids={len(set(ids))} "
|
||||
f"all_texts_present={set(texts) == expected}")
|
||||
print("PASS" if ok else "FAIL")
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Rollback составной операции «создать чат + участников» (уровень «4»).
|
||||
|
||||
Создаём чат, где последний участник — несуществующий пользователь. Проверку
|
||||
приложения нарочно обходим и вызываем запись напрямую: первый INSERT в chats
|
||||
проходит, а вставка участника падает на внешнем ключе. Ожидаем, что транзакция
|
||||
откатилась целиком: ни чата, ни участников в базе нет.
|
||||
|
||||
Запуск внутри контейнера API (у него есть доступ к БД и коду):
|
||||
docker compose cp student-tests/rollback_check.py api:/tmp/rollback_check.py
|
||||
docker compose exec api python /tmp/rollback_check.py
|
||||
Локально без Docker: python student-tests/rollback_check.py
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path.cwd()))
|
||||
import psycopg # noqa: E402
|
||||
|
||||
from app import repo # noqa: E402
|
||||
from app.config import settings # noqa: E402
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
title = "rollback-" + uuid.uuid4().hex[:8]
|
||||
async with await psycopg.AsyncConnection.connect(settings.database_url) as conn:
|
||||
creator = await repo.create_user(conn, "rb_" + uuid.uuid4().hex[:10], "rb", "x")
|
||||
await conn.commit()
|
||||
ghost = uuid.uuid4() # такого пользователя нет
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await repo.insert_chat(conn, title, [uuid.UUID(creator["id"]), ghost], creator["id"])
|
||||
print("FAIL - insert unexpectedly succeeded")
|
||||
return 1
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
print(" ok - insert of unknown member failed on foreign key")
|
||||
cur = await conn.execute(
|
||||
"SELECT count(*) FROM chats WHERE title = %s", (title,))
|
||||
chats = (await cur.fetchone())[0]
|
||||
cur = await conn.execute(
|
||||
"SELECT count(*) FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE c.title = %s", (title,))
|
||||
members = (await cur.fetchone())[0]
|
||||
ok = chats == 0 and members == 0
|
||||
print(f" {'ok ' if ok else 'FAIL'} - after rollback: chats={chats}, members={members} (expected 0, 0)")
|
||||
print("PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Поочерёдная замена реплик API (rolling update) под нагрузкой (lab3, уровень «4»).
|
||||
|
||||
Compose не делает rolling update сам, поэтому замена выполняется по шагам:
|
||||
1. собрать новый образ (docker compose build api);
|
||||
2. для каждой старой реплики:
|
||||
поднять ОДНУ новую (up --scale api=N+1 --no-recreate) → дождаться healthy
|
||||
и того, что LB её обслуживает → остановить старую (SIGTERM, graceful) → удалить;
|
||||
3. всё это время нагрузка пишет сообщения через LB и повторяет неудачные
|
||||
запросы с тем же Idempotency-Key.
|
||||
Итог: ошибки/RPS во время замены, все реплики на новом образе, нет потерь и дублей.
|
||||
|
||||
Миграции: перед заменой выполняется сервис migrate (однократно). Новые миграции
|
||||
должны быть совместимы с ещё работающей старой версией (только добавление).
|
||||
|
||||
python student-tests/rolling_update.py --base-url http://localhost:8080 --output evidence/rolling-update.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from common import Api, Report
|
||||
from failover_each import served_instances
|
||||
from loadgen import LoadGen
|
||||
|
||||
COMPOSE = "docker compose --env-file .env"
|
||||
|
||||
|
||||
def sh(cmd: str) -> str:
|
||||
print(f" $ {cmd}")
|
||||
return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def api_ids() -> list:
|
||||
return [x.strip() for x in sh(f"{COMPOSE} ps -q api").splitlines() if x.strip()]
|
||||
|
||||
|
||||
def image_of(cid: str) -> str:
|
||||
return sh(f"docker inspect -f {{{{.Image}}}} {cid}").strip()
|
||||
|
||||
|
||||
def wait_healthy(cid: str, seconds: float = 90) -> bool:
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if sh(f"docker inspect -f {{{{.State.Health.Status}}}} {cid}").strip() == "healthy":
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def wait_served(api: Api, iid: str, seconds: float = 60) -> bool:
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if iid in served_instances(api, 10):
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--workers", type=int, default=4)
|
||||
ap.add_argument("--no-build", action="store_true", help="не пересобирать образ")
|
||||
ap.add_argument("--output", default=None)
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
|
||||
if not args.no_build:
|
||||
sh(f"{COMPOSE} build api")
|
||||
sh(f"{COMPOSE} run --rm migrate") # однократные миграции до замены
|
||||
new_image = sh("docker image inspect -f {{.Id}} mtusi-messenger:lab4").strip()
|
||||
old = api_ids()
|
||||
n = len(old)
|
||||
r.check(n >= 2, f"{n} replicas before update")
|
||||
old_images = {cid[:12]: image_of(cid) for cid in old}
|
||||
|
||||
user = api.new_user("rolling")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "rolling", "member_ids": [user["id"]]},
|
||||
token=user["token"])
|
||||
gen = LoadGen(api, user["token"], chat["id"], workers=args.workers)
|
||||
gen.start()
|
||||
time.sleep(3)
|
||||
t_begin = time.monotonic()
|
||||
steps = []
|
||||
for cid in old:
|
||||
before = set(api_ids())
|
||||
sh(f"{COMPOSE} up -d --no-deps --no-recreate --scale api={len(before) + 1} api")
|
||||
new = sorted(set(api_ids()) - before)
|
||||
r.check(len(new) == 1, f"one new replica started for {cid[:12]}")
|
||||
new_id = new[0]
|
||||
healthy = wait_healthy(new_id)
|
||||
served = wait_served(api, new_id[:12])
|
||||
t0 = time.monotonic()
|
||||
sh(f"docker stop -t 15 {cid}")
|
||||
sh(f"docker rm {cid}")
|
||||
steps.append({"old": cid[:12], "new": new_id[:12], "new_healthy": healthy, "new_served_by_lb": served,
|
||||
"old_stop_seconds": round(time.monotonic() - t0, 2)})
|
||||
r.check(healthy and served, f"new replica {new_id[:12]} healthy and in LB before old {cid[:12]} stops")
|
||||
t_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
gen.stop()
|
||||
|
||||
final = api_ids()
|
||||
rel_a, rel_b = t_begin - gen.t0, t_end - gen.t0
|
||||
result = {
|
||||
"replicas_before": sorted(old_images), "replicas_after": sorted(c[:12] for c in final),
|
||||
"all_on_new_image": all(image_of(c) == new_image for c in final),
|
||||
"steps": steps,
|
||||
"update_duration_s": round(t_end - t_begin, 1),
|
||||
"before_update": gen.window(0, rel_a),
|
||||
"during_update": gen.window(rel_a, rel_b),
|
||||
"after_update": gen.window(rel_b, rel_b + 3),
|
||||
"max_gap_without_success_s": gen.max_gap_after(t_begin),
|
||||
"instances_serving_at_end": sorted(set(gen.instances_after(t_end))),
|
||||
**gen.verify(),
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
r.check(len(final) == n, f"replica count restored to {n}")
|
||||
r.check(result["all_on_new_image"], "all replicas run the new image")
|
||||
r.check(not set(result["instances_serving_at_end"]) & set(old_images), "old replicas no longer serve traffic")
|
||||
r.check(result["lost_or_changed"] == 0 and result["duplicate_texts"] == 0, "no lost or duplicated writes")
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Собственные тесты lab2 (запуск одной командой):
|
||||
python student-tests/run.py --base-url http://localhost:8080
|
||||
|
||||
Негативные сценарии lab1 на новой аутентификации + проверки уровня «4»:
|
||||
неверный пароль, logout, X-User-Id/Basic не дают доступ, гонка регистрации.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import PASSWORD, Api, Report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
api = Api(ap.parse_args().base_url)
|
||||
r = Report()
|
||||
|
||||
alice, bob, eve = api.new_user("alice"), api.new_user("bob"), api.new_user("eve")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats",
|
||||
{"title": "own tests", "member_ids": [alice["id"], bob["id"]]}, token=alice["token"])
|
||||
assert st == 201, st
|
||||
mpath = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
print("== аутентификация ==")
|
||||
st, _, rh = api.login(alice["username"], "Wrong-password-123")
|
||||
r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Basic"), "wrong password -> 401 + Basic challenge")
|
||||
st, _, _ = api.login("no_such_user_" + uuid.uuid4().hex[:8], PASSWORD)
|
||||
r.check(st == 401, "unknown user -> 401 (same as wrong password)")
|
||||
st, _, rh = api.call("GET", "/api/v1/users/me")
|
||||
r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Bearer"), "no token -> 401 + Bearer challenge")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]})
|
||||
r.check(st == 401, "X-User-Id alone gives no access")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex)
|
||||
r.check(st == 401, "unknown/forged token -> 401")
|
||||
basic = base64.b64encode(f"{alice['username']}:{alice['password']}".encode()).decode()
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", headers={"Authorization": "Basic " + basic})
|
||||
r.check(st == 401, "Basic on business endpoint -> 401")
|
||||
st, me, _ = api.call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": eve["id"]})
|
||||
r.check(st == 200 and me["id"] == alice["id"], "X-User-Id cannot override bearer identity")
|
||||
st, _, _ = api.call("POST", "/api/v1/users", {"username": "short_pw_" + uuid.uuid4().hex[:6],
|
||||
"display_name": "x", "password": "short"})
|
||||
r.check(st == 400, "password shorter than 12 -> 400")
|
||||
|
||||
print("== logout ==")
|
||||
tmp = api.new_user("logout")
|
||||
st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=tmp["token"])
|
||||
r.check(st == 204, "logout -> 204")
|
||||
st, _, _ = api.call("GET", "/api/v1/users/me", token=tmp["token"])
|
||||
r.check(st == 401, "token after logout -> 401")
|
||||
st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=tmp["token"])
|
||||
r.check(st == 401, "second logout -> 401")
|
||||
|
||||
print("== гонка регистрации одного username ==")
|
||||
name = "race_" + uuid.uuid4().hex[:10]
|
||||
|
||||
def reg(_):
|
||||
st, _, _ = api.call("POST", "/api/v1/users", {"username": name, "display_name": "r", "password": PASSWORD})
|
||||
return st
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
codes = list(pool.map(reg, range(20)))
|
||||
r.check(codes.count(201) == 1 and codes.count(409) == 19,
|
||||
f"20 parallel registrations -> one 201, nineteen 409 (got {sorted(codes)})")
|
||||
|
||||
print("== ошибки и доступ (из lab1) ==")
|
||||
st, _, _ = api.raw("POST", "/api/v1/users", b"{not json", {"Content-Type": "application/json"})
|
||||
r.check(st == 400, "malformed JSON -> 400")
|
||||
st, _, _ = api.call("POST", "/api/v1/users", {"username": alice["username"], "display_name": "d", "password": PASSWORD})
|
||||
r.check(st == 409, "duplicate username -> 409")
|
||||
for bad in ("review123\n", "review123\r\n", " review123", "Review123", "ab"):
|
||||
st, _, _ = api.call("POST", "/api/v1/users", {"username": bad, "display_name": "d", "password": PASSWORD})
|
||||
r.check(st == 400, f"invalid username {bad!r} -> 400, not 500 (got {st})")
|
||||
st, _, _ = api.call("POST", mpath, {"text": " "}, token=alice["token"])
|
||||
r.check(st == 400, "blank message -> 400")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "x", "sender_id": eve["id"]}, token=alice["token"])
|
||||
r.check(st == 400, "sender_id in body is rejected")
|
||||
st, _, _ = api.call("POST", mpath, {"text": "чужое"}, token=eve["token"])
|
||||
r.check(st == 404, "outsider POST to foreign chat -> 404")
|
||||
st, _, _ = api.call("GET", mpath, token=eve["token"])
|
||||
r.check(st == 404, "outsider GET foreign history -> 404")
|
||||
st, _, _ = api.call("GET", "/api/v1/chats/not-a-uuid", token=alice["token"])
|
||||
r.check(st == 400, "invalid chat uuid -> 400")
|
||||
|
||||
print("== пагинация ==")
|
||||
for i in range(5):
|
||||
api.call("POST", mpath, {"text": f"m{i}"}, token=bob["token"])
|
||||
seen, cursor, pages = [], None, 0
|
||||
while pages < 10:
|
||||
st, page, _ = api.call("GET", f"{mpath}?limit=2" + (f"&cursor={cursor}" if cursor else ""), token=alice["token"])
|
||||
pages += 1
|
||||
seen += [m["id"] for m in page["items"]]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
r.check(len(seen) == 5 and len(set(seen)) == 5 and pages == 3, "5 messages / limit 2 -> 3 pages, no duplicates")
|
||||
st, other, _ = api.call("POST", "/api/v1/chats", {"title": "other", "member_ids": [alice["id"]]}, token=alice["token"])
|
||||
st, p, _ = api.call("GET", f"{mpath}?limit=1", token=alice["token"])
|
||||
st, _, _ = api.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={p['next_cursor']}", token=alice["token"])
|
||||
r.check(st == 400, "cursor from another chat -> 400")
|
||||
|
||||
print("== Content-Type ==")
|
||||
body = json.dumps({"username": "ct_" + uuid.uuid4().hex[:8], "display_name": "c", "password": PASSWORD}).encode()
|
||||
st, _, _ = api.raw("POST", "/api/v1/users", body, {"Content-Type": "text/plain"})
|
||||
r.check(st == 415, f"valid JSON with text/plain -> 415 (got {st})")
|
||||
st, _, _ = api.raw("POST", mpath, json.dumps({"text": "x"}).encode(),
|
||||
{"Content-Type": "text/plain", "Authorization": "Bearer " + alice["token"]})
|
||||
r.check(st == 415, f"message with text/plain -> 415 (got {st})")
|
||||
st, _, _ = api.raw("POST", "/api/v1/chats", b"{}", {"Content-Type": "text/plain"})
|
||||
r.check(st == 401, f"text/plain without token -> 401 first (got {st})")
|
||||
st, _, _ = api.raw("POST", mpath, json.dumps({"text": "utf8"}).encode(),
|
||||
{"Content-Type": "application/json; charset=utf-8", "Authorization": "Bearer " + alice["token"],
|
||||
"Idempotency-Key": uuid.uuid4().hex})
|
||||
r.check(st == 201, f"application/json; charset=utf-8 accepted (got {st})")
|
||||
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Недоступность S3 (lab4). Сам останавливает и запускает MinIO; из корня репозитория:
|
||||
|
||||
python student-tests/s3_outage.py --base-url http://localhost:8080
|
||||
|
||||
Ожидание: пока MinIO остановлен, загрузка и скачивание дают быстрый 503 в формате
|
||||
контракта (не зависание и не 500), а текстовая часть мессенджера, метаданные
|
||||
вложений и /health/ready работают (S3 не критичен для обязательного текстового пути).
|
||||
После старта MinIO всё снова работает, ранее загруженный файл не изменился.
|
||||
Брошенная во время отказа запись журнала дочищается janitor после grace period.
|
||||
"""
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from common import wait_ready, SAMPLE_PNG, Api, Report, compose, janitor, sha256, tool, upload
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
u = api.new_user("s3down")
|
||||
T = u["token"]
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "s3 outage", "member_ids": [u["id"]]}, token=T)
|
||||
st, before, _ = upload(api, T, chat["id"], SAMPLE_PNG, "before.png")
|
||||
r.check(st == 202, "upload before outage")
|
||||
r.check(wait_ready(api, T, before["id"], seconds=120).get("status") == "ready",
|
||||
"it is processed to ready before the outage (lab5: 202 → worker)")
|
||||
content = f"/api/v1/attachments/{before['id']}/content"
|
||||
|
||||
print("== MinIO остановлен ==")
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
t0 = time.monotonic()
|
||||
st, body, _ = upload(api, T, chat["id"], SAMPLE_PNG, "during.png", timeout=30)
|
||||
took = time.monotonic() - t0
|
||||
r.check(st == 503 and body and body["error"]["code"] == "unavailable", f"upload -> 503 unavailable (got {st})")
|
||||
r.check(took < 15, f"bounded by timeouts, not a hang ({took:.1f} s)")
|
||||
st, _, _ = api.raw("GET", content, headers={"Authorization": "Bearer " + T}, timeout=30)
|
||||
r.check(st == 503, f"download -> 503 (got {st})")
|
||||
st, meta, _ = api.call("GET", f"/api/v1/attachments/{before['id']}", token=T)
|
||||
r.check(st == 200 and meta["sha256"] == before["sha256"], "metadata still served from PostgreSQL")
|
||||
st, _, _ = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "S3 лежит, а чат работает"}, token=T)
|
||||
r.check(st == 201, "text messages still work")
|
||||
st, _, _ = api.call("POST", f"/api/v1/chats/{chat['id']}/messages",
|
||||
{"text": "с вложением", "attachment_ids": [before["id"]]}, token=T)
|
||||
r.check(st == 201, "message referencing an existing attachment still works (metadata only)")
|
||||
r.check(api.raw("GET", "/health/ready")[0] == 200, "/health/ready = 200 (S3 is not critical for text path)")
|
||||
finally:
|
||||
compose("start", "s3")
|
||||
|
||||
print("== MinIO снова работает ==")
|
||||
# После старта MinIO каждая реплика может ещё до S3_BREAKER_SECONDS (5 c) отвечать 503:
|
||||
# предохранитель закрывается по таймеру. Поэтому ждём, пока ОБЕ реплики восстановятся:
|
||||
# несколько успешных загрузок подряд (LB чередует реплики).
|
||||
t0 = time.monotonic()
|
||||
ok = False
|
||||
for _ in range(40):
|
||||
st, raw, _ = api.raw("GET", content, headers={"Authorization": "Bearer " + T})
|
||||
if st == 200:
|
||||
ok = sha256(raw) == before["sha256"]
|
||||
break
|
||||
time.sleep(0.5)
|
||||
r.check(ok, "file uploaded before outage downloads unchanged")
|
||||
streak, codes = 0, []
|
||||
while time.monotonic() - t0 < 40 and streak < 4:
|
||||
st, _, rh = upload(api, T, chat["id"], SAMPLE_PNG, "after.png")
|
||||
codes.append(st)
|
||||
streak = streak + 1 if st == 202 else 0
|
||||
if st != 202:
|
||||
time.sleep(0.5)
|
||||
recovered = time.monotonic() - t0
|
||||
r.check(streak >= 4, f"uploads work again on all replicas: 4 x 202 in a row after {recovered:.1f} s "
|
||||
f"(codes during recovery: {codes})")
|
||||
stats = tool("stats")
|
||||
print(" stats:", stats)
|
||||
s = janitor("--grace-seconds", "0")
|
||||
r.check(s["errors"] == 0, f"janitor cleaned journal rows left by failed uploads ({s})")
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Эксперимент: пик памяти API при N одновременных загрузках (lab4, уровень «4»).
|
||||
|
||||
python student-tests/upload_memory.py --base-url http://localhost:8080 \
|
||||
--concurrency 10 --output evidence/upload-memory-10.json
|
||||
|
||||
Гипотеза: память реплики почти не зависит от размера и числа одновременных
|
||||
загрузок, потому что тело не копится в RAM (≤ UPLOAD_SPOOL_MEMORY_BYTES в памяти,
|
||||
остальное во временном файле), проверка изображения потоковая, в S3 уходит поток.
|
||||
Наивная реализация (`await request.body()` + `Image.open().load()`) держала бы
|
||||
≥ размер файла × 2…4 на каждую загрузку: 10 × 9,5 МБ → 200–400 МБ, что больше
|
||||
лимита контейнера 256 МБ (OOM).
|
||||
|
||||
Методика: генерируется настоящий PNG из шума ≈9,5 МБ (почти не сжимается),
|
||||
N потоков одновременно отправляют его через LB (несколько раундов), параллельно
|
||||
каждые ~0,3 с читается память cgroup v2 всех контейнеров api (usage как в docker stats, anon, memory.peak). Фиксируются память
|
||||
до нагрузки, пик во время и после, время ответа, коды ответов и sha256.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import ROOT, Api, Report, compose, make_png, sha256, upload, wait_ready
|
||||
|
||||
# Печатаем «имя значение»: cgroup v2, иначе v1 (старые ядра/Docker Desktop с cgroup v1).
|
||||
CGROUP = (
|
||||
"if [ -f /sys/fs/cgroup/memory.current ]; then "
|
||||
"echo current $(cat /sys/fs/cgroup/memory.current); "
|
||||
"grep -E '^(anon|inactive_file) ' /sys/fs/cgroup/memory.stat; "
|
||||
"[ -f /sys/fs/cgroup/memory.peak ] && echo peak $(cat /sys/fs/cgroup/memory.peak); "
|
||||
"else M=/sys/fs/cgroup/memory; "
|
||||
"echo current $(cat $M/memory.usage_in_bytes); "
|
||||
"grep -E '^total_(rss|inactive_file) ' $M/memory.stat | sed -e 's/^total_rss/anon/' -e 's/^total_//'; "
|
||||
"echo peak $(cat $M/memory.max_usage_in_bytes); fi")
|
||||
UNITS = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3}
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
return [c[:12] for c in compose("ps", "-q", "api").split()]
|
||||
|
||||
|
||||
def _docker_stats(cid: str) -> dict:
|
||||
"""Запасной путь: docker stats (usage без файлового кэша)."""
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", cid],
|
||||
capture_output=True, text=True, cwd=ROOT).stdout
|
||||
m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", out)
|
||||
usage = round(float(m.group(1)) * UNITS[m.group(2)] / 1024 ** 2, 1) if m else 0.0
|
||||
return {"usage": usage, "anon": usage, "peak": None}
|
||||
|
||||
|
||||
_warned: list = []
|
||||
|
||||
|
||||
def read_cgroup(cid: str) -> dict:
|
||||
"""Память контейнера (MiB): usage как в docker stats (current − inactive_file),
|
||||
anon — память процессов без файлового кэша, peak — максимум за жизнь cgroup."""
|
||||
p = subprocess.run(["docker", "exec", cid, "sh", "-c", CGROUP],
|
||||
capture_output=True, text=True, cwd=ROOT)
|
||||
vals = {}
|
||||
for line in p.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
vals[parts[0]] = int(parts[1])
|
||||
if "current" not in vals:
|
||||
if not _warned:
|
||||
_warned.append(1)
|
||||
print(f" cgroup read failed (exit {p.returncode}: {p.stderr.strip()[:200]!r}); using docker stats")
|
||||
return _docker_stats(cid)
|
||||
mib = 1024 ** 2
|
||||
return {"usage": round((vals["current"] - vals.get("inactive_file", 0)) / mib, 1),
|
||||
"anon": round(vals.get("anon", vals["current"]) / mib, 1),
|
||||
"peak": round(vals["peak"] / mib, 1) if "peak" in vals else None}
|
||||
|
||||
|
||||
def sample_mem(ids: list[str]) -> dict:
|
||||
with ThreadPoolExecutor(max_workers=len(ids) or 1) as pool:
|
||||
return dict(zip(ids, pool.map(read_cgroup, ids)))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--concurrency", type=int, default=10)
|
||||
ap.add_argument("--rounds", type=int, default=3)
|
||||
ap.add_argument("--width", type=int, default=1780, help="PNG width/height: 1780 → ≈9,5 МБ")
|
||||
ap.add_argument("--output", default=None)
|
||||
args = ap.parse_args()
|
||||
api, r = Api(args.base_url), Report()
|
||||
|
||||
data = make_png(args.width, args.width, noise=True)
|
||||
digest = sha256(data)
|
||||
print(f"file: {len(data)} bytes ({len(data) / 1024 ** 2:.2f} MiB), sha256 {digest[:12]}…")
|
||||
u = api.new_user("memexp")
|
||||
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "memory", "member_ids": [u["id"]]}, token=u["token"])
|
||||
|
||||
ids = api_containers()
|
||||
idle = sample_mem(ids)
|
||||
print("idle MiB:", idle)
|
||||
peak = {k: {"usage": v["usage"], "anon": v["anon"]} for k, v in idle.items()}
|
||||
samples = []
|
||||
stop = threading.Event()
|
||||
|
||||
def sampler():
|
||||
t0 = time.monotonic()
|
||||
while not stop.is_set():
|
||||
m = sample_mem(ids)
|
||||
samples.append({"t": round(time.monotonic() - t0, 2), **m})
|
||||
for k, v in m.items():
|
||||
for f in ("usage", "anon"):
|
||||
peak[k][f] = max(peak[k][f], v[f])
|
||||
time.sleep(0.2)
|
||||
|
||||
th = threading.Thread(target=sampler, daemon=True)
|
||||
th.start()
|
||||
results = []
|
||||
|
||||
def one(i):
|
||||
t = time.monotonic()
|
||||
try:
|
||||
st, body, rh = upload(api, u["token"], chat["id"], data, f"noise-{i}.png", timeout=120)
|
||||
except OSError as exc:
|
||||
return {"status": None, "error": type(exc).__name__, "seconds": round(time.monotonic() - t, 2)}
|
||||
return {"status": st, "seconds": round(time.monotonic() - t, 2),
|
||||
"sha_ok": bool(body) and body.get("sha256") == digest,
|
||||
"instance": rh.get("X-Instance-Id"), "id": (body or {}).get("id")}
|
||||
|
||||
t_start = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
for rnd in range(args.rounds):
|
||||
results += list(pool.map(one, range(rnd * args.concurrency, (rnd + 1) * args.concurrency)))
|
||||
elapsed = time.monotonic() - t_start
|
||||
time.sleep(2)
|
||||
stop.set()
|
||||
th.join()
|
||||
after = sample_mem(ids)
|
||||
|
||||
ok = [x for x in results if x["status"] == 202]
|
||||
lat = sorted(x["seconds"] for x in ok)
|
||||
summary = {
|
||||
"file_bytes": len(data), "concurrency": args.concurrency, "rounds": args.rounds,
|
||||
"uploads": len(results), "status_counts": {str(s): sum(1 for x in results if x["status"] == s)
|
||||
for s in sorted({x["status"] for x in results}, key=str)},
|
||||
"sha256_ok": sum(1 for x in ok if x["sha_ok"]),
|
||||
"instances": sorted({x["instance"] for x in ok if x["instance"]}),
|
||||
"elapsed_s": round(elapsed, 2),
|
||||
"throughput_MiB_s": round(len(ok) * len(data) / 1024 ** 2 / elapsed, 1) if elapsed else None,
|
||||
"latency_s": {"p50": lat[len(lat) // 2] if lat else None, "max": lat[-1] if lat else None},
|
||||
"memory_MiB": {"idle": idle, "sampled_peak": peak, "after": after,
|
||||
"max_growth_usage": round(max(peak[k]["usage"] - idle[k]["usage"] for k in peak), 1),
|
||||
"max_growth_anon": round(max(peak[k]["anon"] - idle[k]["anon"] for k in peak), 1),
|
||||
"cgroup_peak_after": {k: v["peak"] for k, v in after.items()}},
|
||||
"container_limit_MiB": 256, "samples": len(samples),
|
||||
}
|
||||
print(json.dumps({k: v for k, v in summary.items() if k != "samples"}, ensure_ascii=False, indent=2))
|
||||
|
||||
# Проверка, что сохранены настоящие байты (выборочно). С lab5 байты отдаются после ready:
|
||||
# 202 означает «принято», обработку делает worker (под нагрузкой — с очередью).
|
||||
for x in ok[:3]:
|
||||
wait_ready(api, u["token"], x["id"], seconds=300)
|
||||
st, raw, _ = api.raw("GET", f"/api/v1/attachments/{x['id']}/content",
|
||||
headers={"Authorization": "Bearer " + u["token"]}, timeout=60)
|
||||
r.check(st == 200 and sha256(raw) == digest, f"download {x['id'][:8]} sha256 matches")
|
||||
r.check(len(ok) == len(results), f"all {len(results)} uploads -> 202")
|
||||
r.check(summary["sha256_ok"] == len(ok), "sha256 of every upload matches")
|
||||
# Критерий — память ПРОЦЕССОВ (anon): именно туда попало бы накопление тела в RAM.
|
||||
# usage дополнительно включает файловый кэш ядра для временных spool-файлов (вытесняемый),
|
||||
# он печатается выше для полноты, но буферизацию не показывает.
|
||||
growth = summary["memory_MiB"]["max_growth_anon"]
|
||||
r.check(growth < 100,
|
||||
f"peak process memory (anon) growth per replica {growth} MiB < 100 MiB "
|
||||
f"(naive buffering: ≥ {args.concurrency // 2} × {len(data) / 1024 ** 2:.1f} MiB × 2)")
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump({**summary, "raw_samples": samples}, f, ensure_ascii=False, indent=2)
|
||||
print("saved", args.output)
|
||||
r.finish()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user