86 changed files with 10148 additions and 30 deletions
+38 -5
View File
@@ -1,20 +1,53 @@
# VCS / CI
.git
.github
# Secrets and local env
.env
.env.*
!.env.example
secrets
certs
volumes
data
backups
*.key
*.pem
*.p12
*.pfx
*.jks
# Local / runtime junk
.venv
__pycache__
venv
**/__pycache__
**/*.py[cod]
*.egg-info
.pytest_cache
.mypy_cache
.ruff_cache
node_modules
evidence
.DS_Store
Thumbs.db
.idea
.vscode
# Lab artifacts not needed at runtime
evidence
volumes
data
backups
docs
student-tests
tests
scripts
examples
contracts
REPORT.md
README.md
COURSE.md
CONTRIBUTING.md
MAINTAINERS.md
Dockerfile.example
compose.example.yaml
requirements-dev.txt
*.md
# Keep only what the image needs: requirements.txt + src/ + migrations/
+21 -4
View File
@@ -1,16 +1,17 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Secret VALUES are not here: run `python scripts/init_secrets.py` to create ./secrets/*.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
# Sessions live in Redis (AOF) since lab2 grade 4.
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
# Number of interchangeable API replicas behind HAProxy (grade 3 requires ≥2).
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Private S3 bucket (SeaweedFS; s3:9000 is the s3-gw sidecar on the internal storage network).
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
@@ -18,8 +19,24 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
MAX_CONCURRENT_UPLOADS=8
UPLOAD_IDLE_TIMEOUT_SECONDS=10
UPLOAD_DEADLINE_SECONDS=120
UPLOAD_GRACE_SECONDS=900
RECONCILE_INTERVAL_SECONDS=300
SHUTDOWN_DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=8
# Lab5: RabbitMQ broker + worker pipeline.
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_BACKOFF_BASE_SECONDS=1
JOB_BACKOFF_MAX_SECONDS=60
WORKER_PROCESS_TIMEOUT_SECONDS=45
OUTBOX_POLL_SECONDS=0.5
STAGING_RETENTION_SECONDS=86400
# Optimized WebP quality (EXIF always stripped). Method is fixed in code (4).
WEBP_QUALITY=80
# Fault injection (keep 0 for normal runs).
WORKER_CRASH_AFTER_SIDE_EFFECT=0
+3
View File
@@ -0,0 +1,3 @@
# Shell scripts run on Unix or inside Linux containers (infra/s3-gw-entrypoint.sh is
# bind-mounted into s3-gw): keep LF even on Windows checkouts with core.autocrlf=true.
*.sh text eol=lf
+41
View File
@@ -0,0 +1,41 @@
# Pinned base image (lab4 → lab5). Update digest when bumping Python.
FROM python:3.12.8-slim-bookworm AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONPATH=/app/src \
MIGRATIONS_DIR=/app/migrations \
UPLOAD_SPOOL_DIR=/tmp/messenger-spool
WORKDIR /app
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home /app --shell /usr/sbin/nologin app
# djpeg (libjpeg-turbo) verifies uploaded JPEGs; libwebp for optimized variants in the worker.
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
libjpeg-turbo-progs=1:2.1.5-2 \
libwebp7=1.2.4-0.2+deb12u1 \
libwebpdemux2=1.2.4-0.2+deb12u1 \
libwebpmux3=1.2.4-0.2+deb12u1 \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir --upgrade pip==24.3.1 \
&& pip install --no-cache-dir --retries 10 --timeout 60 -r requirements.txt
# Code stays root-owned and read-only for the runtime user.
COPY src ./src
COPY migrations ./migrations
USER app
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=5 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
CMD ["python", "-m", "messenger"]
+121 -16
View File
@@ -1,37 +1,142 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Болотов Илья, БВТ2303
- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не выбран)
- Ветка: `solution/bvt2303-bolotov`
- Использованные источники, библиотеки и помощь ИИ:
- FastAPI, Uvicorn, psycopg3 (+pool), redis-py, argon2-cffi, boto3 1.35.99, pika (RabbitMQ), Pillow, HAProxy 2.9, SeaweedFS 3.97, RabbitMQ 3.13.7, libjpeg-turbo (`djpeg`);
- спецификации PNG/JPEG, документация RabbitMQ (publisher confirms, durable queues, manual ack);
- материалы курса: README, contracts (`attachment-states.md`, OpenAPI), tests;
- перенос lab4 и реализация async-pipeline выполнены с помощью ИИ-ассистента (Cursor). Числа ниже взяты из `evidence/`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
**Prerequisites:**
- Docker Engine + Compose v2 (проверено: Docker Desktop/WSL2);
- Python 3.9+ на хосте для секретов и тестов. На Windows задайте `PYTHONUTF8=1`.
```sh
cp .env.example .env # Windows: copy .env.example .env
python scripts/init_secrets.py # ./secrets/* (идемпотентно)
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2
# Windows: scripts\up.cmd ; Unix: ./scripts/up.sh
```
Порядок старта:
1. `postgres`, `redis`, `broker` (RabbitMQ), `s3` → healthy.
2. One-shot `migrate` (`001`…`006`) и `s3-init` (приватный bucket) → exit 0.
3. `api` ×2, `worker`, `lb` (публикация только `127.0.0.1:8080`).
Готовность: `GET http://127.0.0.1:8080/health/ready` → 200. На Windows используйте `127.0.0.1`, не `localhost`.
Проверки (PowerShell; Unix — те же команды с `export`):
```powershell
$env:PYTHONUTF8=1
python scripts/check_contract.py
python tests/smoke.py --base-url http://127.0.0.1:8080
python student-tests/test_lab5_remarks.py
python student-tests/test_images_unit.py
$env:RUN_DOCKER_TESTS=1; python student-tests/test_lab5_pipeline.py
python scripts/lab5_defense_evidence.py
python scripts/lab5_outbox_replay_evidence.py
$env:RUN_DEPS_TEST=1; python -c "import sys; sys.path[:0]=['student-tests']; from test_deps import test_redis_stop_returns_503_or_unready; test_redis_stop_returns_503_or_unready()"
# операторский replay failed-задач:
# docker compose --env-file .env exec -T api python -m messenger.replay --list
# docker compose --env-file .env exec -T api python -m messenger.replay --dead-letter-id N
scripts\stop-worker.cmd # / scripts/stop-worker.sh
scripts\start-worker.cmd
```
- Остановка без потери данных: `docker compose --env-file .env down`.
- Удаление учебных данных: `docker compose --env-file .env down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
Клиент → HAProxy lb (127.0.0.1:8080)
├─ messenger_api (/health/ready) → API × N
└─ messenger_api_degraded (/health/live)
API → PostgreSQL users/chats/messages + attachments/jobs/outbox/DLQ/revocations
→ Redis сессии (AOF) + durable session_revocations в PG
→ s3:9000 staging/ + original/ + variants/
→ outbox poll → RabbitMQ (durable queue, confirms, manual ack)
Worker → lease job → decode → original + thumbnail + crop + WebP → ready → ack
Janitor/reconcile → intents + orphans; staging queued/processing не трогает
```
**Перенесено из lab4:** весь `src/messenger` (валидация PNG/JPEG, S3, HAProxy leastconn, drain shutdown, идемпотентность сообщений, студенческие тесты lab3/4). Upload больше не завершает объект как `ready` синхронно.
**Решения lab5 (уровни 3–4):**
- **Приём → 202 queued.** После потоковой валидации (≥ worker-декодера: PNG IDAT/zlib + целостность до IEND; JPEG SOS↔SOF и структура) байты кладутся в `staging/…`, в одной транзакции пишутся `attachments(status=queued)`, `attachment_jobs(pending)` и строка **transactional outbox**. HTTP 202 только после COMMIT. Байты и credentials в брокер не кладутся — только JSON job envelope.
- **Брокер — RabbitMQ 3.13** (durable queue, persistent messages, publisher confirm, consumer manual ack). Недоступность worker не заставляет API делать resize: клиент получает 202, content до ready → 409, текстовый чат работает.
- **Worker** (отдельный сервис, `WORKER_CONCURRENCY`, prefetch=concurrency): lease (`JOB_TIMEOUT_SECONDS`), идемпотентный `complete_job_ready`, варианты `thumbnail` (JPEG ≤256, floor, без upscale), `crop` 128×128 JPEG, `optimized` WebP (`WEBP_QUALITY=80`, method=4, без EXIF). Ключ оригинала: `original/{attachment_id_hex}{sha256[:16]}` — API читает тот же `object_key` из БД (без суффикса `.jpg`/без него расхождения нет).
- **Retry / DLQ / replay.** Временные ошибки — backoff+jitter (`JOB_BACKOFF_*`), постоянные и исчерпание `JOB_MAX_ATTEMPTS=5` → `failed` + `dead_letter_jobs`. Оператор: `python -m messenger.replay` / `scripts/replay_failed.py`. Ready того же/нового pipeline не переисполняется (`skipped_ready`).
- **Crash после side effect до ack.** `WORKER_CRASH_AFTER_SIDE_EFFECT=1`: после durable ready worker падает до ack; redelivery видит `already_done` и ack’ает. Outbox publisher на API закрывает разрыв COMMIT↔publish.
- **Deadline живого worker.** Lease не прерывает зависший процесс; `WORKER_PROCESS_TIMEOUT_SECONDS=45` (ThreadPool) даёт `processing_failed` → retry/failed, а не вечный `processing`.
- **Janitor / staging.** Staging для `queued`/`processing` не удаляется по retention. Для `failed` — только после `STAGING_RETENTION_SECONDS` и повторной проверки статуса под `FOR UPDATE` (replay failed→queued побеждает TOCTOU). Orphan sweep перед DELETE блокирует строку и отказывается, если ключ снова referenced / status ready|queued|processing / есть variant.
- **Сессии.** `session_revocations` в PostgreSQL: logout при упавшем Redis пишет revoke и отвечает 204; после подъёма кэша старый токен → 401. Повторный logout по уже отозванному → 401.
- **Миграции:** `005_async_attachments.sql` (variants/jobs/outbox/DLQ), `006_lab5_ops.sql` (revocations + индексы staging/status).
### Согласованность PostgreSQL, staging и брокера
| Отказ | Что делает код | Итог |
| --- | --- | --- |
| БД недоступна до staging PUT | intent не записан → 503 | в S3 ничего нет |
| Staging PUT упал | компенсация DELETE + intent | нет объекта |
| COMMIT queued+job+outbox оборвался (исход неизвестен) | staging не удаляется | outbox/reconcile дотащат или уберут orphan после grace |
| Outbox не успел publish | publisher на API догоняет unpublished | задача доходит до worker без ручного job |
| Worker упал после ready до ack | redelivery → already_done → ack | один набор variants |
| Worker завис на decode | process timeout → retry/failed | не вечный processing |
| Janitor vs replay | FOR UPDATE + re-check status | нет source_missing / ready без байтов |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Offline check | `scripts/check_contract.py` | PASS | `evidence/check.txt` |
| Публичный smoke lab5 | `tests/smoke.py` | PASS lab5 public API scenario (202→ready, variants) | `evidence/smoke.txt` |
| Битый/усечённый JPEG/PNG → 415 | `test_lab5_remarks`, `test_images_unit` | corrupted + truncated PNG/JPEG → 415, не 202 | `evidence/lab5-remarks.txt` |
| Thumbnail 319×200 → 256×160 | remarks | meta и JPEG-байты 256×160 | `evidence/lab5-remarks.txt` |
| JPEG download ключ API↔worker | remarks | ready + content 200, байты = исходник | `evidence/lab5-remarks.txt` |
| Variants crop/WebP + worker stop | `lab5_defense_evidence.py` | thumbnail/crop/optimized; worker down → queued+409+чат 201; после start → ready | `evidence/lab5-defense.txt` |
| Crash after side effect → redelivery | `lab5_outbox_replay_evidence.py` | ready, 3 variants без ручного job | `evidence/lab5-outbox-replay.txt` |
| Replay failed→queued→ready | тот же скрипт + `messenger.replay` | `replay_result=<new-job-id>`; attachment ready | `evidence/lab5-outbox-replay.txt` |
| Janitor не съедает queued staging | reconcile `--dry-run --grace-seconds 0` при queued | status остался queued; после start worker → ready | `evidence/lab5-outbox-replay.txt` |
| Logout при недоступном Redis | `test_redis_stop_returns_503_or_unready` | stop redis → logout 204; start → `/users/me` 401 | `evidence/logout-cache.txt` |
| Повторный logout | defense | второй DELETE → 401 | `evidence/lab5-defense.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Fault-injection для защиты: `WORKER_CRASH_AFTER_SIDE_EFFECT`, остановка worker скриптами `stop-worker`/`start-worker`, операторский replay через `python -m messenger.replay`.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Очередь и worker (уровень 4, демонстрация на стенде):**
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Что ожидали | Что наблюдали | Восстановление / данные |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `stop-worker` → upload | 202 queued, content 409, чат жив | так и есть (`ffbc209a-…`) | `start-worker` → ready |
| Crash worker после ready до ack | redelivery завершает тот же результат | ready, 3 variants (`378d8da4-…`) | нормальный worker подхватывает очередь |
| Inject failed + DLQ → replay | failed→queued→ready, staging цел | ready после replay id=3 | без `source_missing` |
| Reconcile dry-run при queued | staging не кандидат на удаление | `stale_staging_candidates=0`, status queued | затем ready |
| Redis stop → logout | 204 + durable revoke | `/me` 401 после start redis | сессия не «воскресла» |
## Ограничения и следующие шаги
Нагрузочный эксперимент памяти API/S3 из lab4 на этом стенде не повторялся как трек 5; лимиты слотов/S3/`GOMEMLIMIT` сохранены из lab4.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Ответы на вопросы защиты
- **Publisher confirm vs consumer ack.** Confirm — брокер принял сообщение на диск/очередь. Ack — consumer зафиксировал устойчивый эффект (ready/failed/retry) и можно снимать delivery. Outbox даёт confirm после COMMIT намерения; ack — после эффекта в worker.
- **Что сохранено в момент 202.** Staging-объект в S3 + строки attachment(`queued`)+job(`pending`)+outbox в PostgreSQL. Публикация в RabbitMQ может догнать outbox-поллером.
- **Повторная доставка.** Нормальна (at-least-once). Ключ идемпотентности — `job_id` + `pipeline_version` + уникальность variants `(attachment_id, pipeline_version, name)`; повтор после ready → `already_done`.
- **Зависший vs долгий worker.** Lease поднимает мёртвый lease; живой зависший обрывает `WORKER_PROCESS_TIMEOUT_SECONDS`. Prefetch/concurrency ограничивают параллелизм.
## Ограничения
- Один Docker-хост; отказ PostgreSQL/RabbitMQ/S3 целиком не маскируется. Worker один (масштабирование возможно, но не измерялось).
- Тело upload по-прежнему spool’ится на диск реплики до staging PUT (как lab4).
- Операторский reprocess «новой версии pipeline» отдельной утилитой не вынесен: смена `pipeline_version` не автоматизирована; замечание про eternal queued после `jobctl reprocess` к этой реализации не применяется.
- Трек 5 не выполнялся.
- Windows: `PYTHONUTF8=1`; `make` заменён эквивалентами из Makefile.
+384
View File
@@ -0,0 +1,384 @@
# Lab5: Client → HAProxy → API×N → PostgreSQL + Redis + RabbitMQ + worker
# ↘ private S3 (SeaweedFS): staging → original/variants.
# Public entrypoint only via lb:8080. API replicas and S3 publish no host ports.
services:
postgres:
image: postgres:16.6-bookworm
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
secrets:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
restart: unless-stopped
mem_limit: 512m
cpus: 0.50
networks:
- labnet
redis:
image: redis:7.4.2-bookworm
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
- --dir
- /data
volumes:
- redisdata:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
start_period: 5s
restart: unless-stopped
mem_limit: 128m
cpus: 0.25
networks:
- labnet
broker:
image: rabbitmq:3.13.7-management-alpine
# 3.13 rejects RABBITMQ_DEFAULT_PASS_FILE; load the Docker secret into env at start.
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
export RABBITMQ_DEFAULT_USER=messenger
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
exec docker-entrypoint.sh rabbitmq-server
secrets:
- broker_password
volumes:
- brokerdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 25s
restart: unless-stopped
mem_limit: 384m
cpus: 0.50
networks:
- labnet
# SeaweedFS all-in-one (master + volume + filer + S3 gateway). Every listener,
# including the master/filer web UIs and the unauthenticated gRPC ports, is bound
# to 127.0.0.1 inside this container; only s3-gw exposes the S3 HTTP API.
s3:
image: chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e
command:
- server
- -dir=/data
- -ip=127.0.0.1
- -ip.bind=127.0.0.1
- -s3
- -s3.port=8333
- -s3.ip.bind=127.0.0.1
- -s3.config=/run/secrets/s3_config
- -filer.exposeDirectoryData=false
- -master.volumeSizeLimitMB=512
- -master.telemetry=false
# 0 = free disk / volume size. A fixed count caps the bucket: with 16 slots
# (7 taken by the default collection) uploads failed after ≈4.7 GiB.
- -volume.max=0
environment:
# One weed process holds ≈30–40 MiB per in-flight 10 MiB PUT (s3 → filer → volume);
# API slots cap S3 at replicas × MAX_CONCURRENT_UPLOADS PUTs (16 → ≈500 MiB live).
# With 512m and no soft limit 10–40 parallel uploads OOM-killed weed (exit 137).
GOMEMLIMIT: 768MiB
secrets:
- s3_config
volumes:
- s3data:/data
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
interval: 5s
timeout: 3s
retries: 20
start_period: 15s
stop_grace_period: 20s
restart: unless-stopped
mem_limit: 1g
cpus: 0.50
networks:
storage:
aliases:
- s3
# TCP sidecar in the s3 network namespace: publishes ONLY the S3 HTTP API as s3:9000.
# A restart of s3 gives it a new network namespace; the entrypoint then exits and
# `restart: unless-stopped` starts s3-gw again inside the new one (infra/s3-gw-entrypoint.sh).
# Compose-driven restarts/recreation of s3 also restart s3-gw (depends_on restart: true).
s3-gw:
image: haproxy:2.9.14-bookworm
network_mode: "service:s3"
entrypoint: ["/bin/sh", "/usr/local/bin/s3-gw-entrypoint.sh"]
stop_signal: SIGTERM
volumes:
- ./infra/s3-gateway.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./infra/s3-gw-entrypoint.sh:/usr/local/bin/s3-gw-entrypoint.sh:ro
depends_on:
s3:
condition: service_healthy
restart: true
healthcheck:
# Config valid AND still attached to the live namespace of s3 (not just `lo`).
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && grep -qv -e '^ *lo:' -e '|' /proc/net/dev"]
interval: 10s
timeout: 3s
retries: 5
start_period: 5s
restart: unless-stopped
mem_limit: 64m
cpus: 0.25
# One-shot, idempotent bucket bootstrap with the S3 admin identity (API never gets it).
s3-init:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab5
command: ["python", "-m", "messenger.s3bootstrap"]
environment:
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_REGION: "${S3_REGION:-us-east-1}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
depends_on:
s3-gw:
condition: service_started
restart: "no"
dns_opt:
- ndots:0
- timeout:1
- attempts:1
mem_limit: 128m
cpus: 0.25
networks:
- storage
migrate:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab5
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
MIGRATIONS_DIR: /app/migrations
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
restart: "no"
mem_limit: 256m
cpus: 0.25
networks:
- labnet
api:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab5
environment:
APP_HOST: "0.0.0.0"
APP_PORT: "8080"
APP_ENV: lab
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
BROKER_URL_FILE: /run/secrets/broker_url
MIGRATIONS_DIR: /app/migrations
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_REGION: "${S3_REGION:-us-east-1}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}"
MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}"
MAX_CONCURRENT_UPLOADS: "${MAX_CONCURRENT_UPLOADS:-8}"
UPLOAD_IDLE_TIMEOUT_SECONDS: "${UPLOAD_IDLE_TIMEOUT_SECONDS:-10}"
UPLOAD_DEADLINE_SECONDS: "${UPLOAD_DEADLINE_SECONDS:-120}"
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
RECONCILE_INTERVAL_SECONDS: "${RECONCILE_INTERVAL_SECONDS:-300}"
SHUTDOWN_DRAIN_SECONDS: "${SHUTDOWN_DRAIN_SECONDS:-3}"
SHUTDOWN_TIMEOUT_SECONDS: "${SHUTDOWN_TIMEOUT_SECONDS:-8}"
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-60}"
OUTBOX_POLL_SECONDS: "${OUTBOX_POLL_SECONDS:-0.5}"
STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-86400}"
secrets:
- database_url
- cache_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
redis:
condition: service_healthy
broker:
condition: service_healthy
expose:
- "8080"
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
stop_grace_period: 15s
restart: unless-stopped
dns_opt:
- ndots:0
- timeout:1
- attempts:1
mem_limit: 512m
cpus: 0.50
sysctls:
net.ipv4.tcp_rmem: "4096 131072 262144"
net.ipv4.tcp_wmem: "4096 16384 262144"
networks:
- labnet
- storage
worker:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab5
command: ["python", "-m", "messenger.worker"]
environment:
APP_ENV: lab
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_REGION: "${S3_REGION:-us-east-1}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
WORKER_CONCURRENCY: "${WORKER_CONCURRENCY:-2}"
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-60}"
JOB_BACKOFF_BASE_SECONDS: "${JOB_BACKOFF_BASE_SECONDS:-1}"
JOB_BACKOFF_MAX_SECONDS: "${JOB_BACKOFF_MAX_SECONDS:-60}"
WORKER_PROCESS_TIMEOUT_SECONDS: "${WORKER_PROCESS_TIMEOUT_SECONDS:-45}"
OUTBOX_POLL_SECONDS: "${OUTBOX_POLL_SECONDS:-0.5}"
WEBP_QUALITY: "${WEBP_QUALITY:-80}"
WORKER_CRASH_AFTER_SIDE_EFFECT: "${WORKER_CRASH_AFTER_SIDE_EFFECT:-0}"
secrets:
- database_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
disable: true
stop_grace_period: 20s
restart: unless-stopped
dns_opt:
- ndots:0
- timeout:1
- attempts:1
mem_limit: 768m
cpus: 0.75
networks:
- labnet
- storage
lb:
image: haproxy:2.9.14-bookworm
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:8080:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
# Image has no curl/wget; validate config + process. Public readiness is /health/ready via LB.
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 5
start_period: 5s
restart: unless-stopped
mem_limit: 128m
cpus: 0.25
sysctls:
net.ipv4.tcp_rmem: "4096 131072 262144"
net.ipv4.tcp_wmem: "4096 16384 262144"
networks:
- labnet
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_config:
file: ./secrets/s3_config.json
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
broker_password:
file: ./secrets/broker_password
broker_url:
file: ./secrets/broker_url
volumes:
pgdata:
redisdata:
s3data:
brokerdata:
networks:
labnet:
driver: bridge
# Storage segment: no route outside Docker, only api and s3-init can reach s3:9000.
storage:
driver: bridge
internal: true
Binary file not shown.
+4
View File
@@ -0,0 +1,4 @@
PASS variants ready attachment=e24912da-5b11-4653-94ae-4f66550b7711 thumb_ct=image/jpeg names=['crop', 'optimized', 'thumbnail']
PASS worker-down queued status=queued content=409 chat=201 attachment=ffbc209a-6ae7-4e21-b460-d60a54446274
PASS worker-restart ready status=ready attachment=ffbc209a-6ae7-4e21-b460-d60a54446274
PASS logout second DELETE=401 and /me=401
+5
View File
@@ -0,0 +1,5 @@
PASS crash-after-side-effect→redelivery ready attachment=378d8da4-f525-4ef5-a5af-66f460ce1783 status=ready variants=3
PASS operator-replay dead_letter_id=3 stdout=replay_result=0ad9343a-0737-4a72-98a4-b72b41ed66de dead_letter_id=3
PASS failed→queued→ready after replay attachment=f313d877-3d7f-4899-a03c-9047f18e02fb status=ready variants=3
PASS janitor-dry-run while queued status=queued report={"dry_run": true, "grace_seconds": 0, "stale_intents": 0, "intent_objects_deleted": 0, "objects_scanned": 207, "referenced_objects": 205, "fresh_unreferenced_skipped": 0, "orphans_found": 2, "orphans_deleted": 0, "stale_staging_candidates": 0, "stale_staging_deleted": 0, "duration_ms": 307.2}
PASS queued attachment survived janitor dry-run attachment=97bac136-9337-49d2-a0a7-5641ec1facfc
+13
View File
@@ -0,0 +1,13 @@
PASS remarks 415
PASS remarks thumbnail 256x160
PASS remarks jpeg key match
PASS test_all_filter_types_accepted
PASS test_corrupted_jpeg_rejected
PASS test_corrupted_png_rejected
PASS test_exactly_at_pixel_limit_accepted
PASS test_foreign_formats_rejected
PASS test_pixel_limit_detected_from_header_only
PASS test_reviewer_corrupted_jpegs_structural_part
PASS test_reviewer_corrupted_pngs_rejected
PASS test_valid_jpegs_any_chunking
PASS test_valid_pngs_any_chunking
Binary file not shown.
Binary file not shown.
+12 -5
View File
@@ -1,7 +1,14 @@
# Инфраструктура студента
# Инфраструктура lab4
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
- `haproxy.cfg` — публичный балансировщик (`127.0.0.1:8080`, `balance leastconn`).
- Новый запрос уходит на реплику с наименьшим числом открытых соединений. Загрузка вложения держит соединение секунды, поэтому длинные upload не скапливаются на одной реплике, пока у другой свободны слоты. При равной нагрузке (короткие последовательные запросы) реплики чередуются, как при round-robin.
- Основной backend `messenger_api` проверяет `/health/ready` (БД + Redis + S3): неготовая реплика исключается из трафика.
- Если не готова ни одна реплика (общая зависимость недоступна), трафик идёт в `messenger_api_degraded`, который проверяет только `/health/live`. Клиент получает JSON 503 самого API на затронутых endpoint, а текстовый чат и метаданные продолжают работать.
- Ошибки, сгенерированные самим HAProxy (502/503/504), оформлены в контрактном JSON-конверте с `X-Request-Id` и `X-Instance-Id: lb`.
- DNS Docker `127.0.0.11` + `server-template` подхватывают `--scale api=N` и замену контейнеров.
- `inter 1s fall 2` + `observe layer4 … on-error mark-down`: остановленная реплика исключается после первой неудачной попытки подключения или примерно через 2–3 с по health check. При `docker stop` реплика API по SIGTERM сначала отвечает 503 только на `/health/ready` и ещё 3 с (`SHUTDOWN_DRAIN_SECONDS`) обслуживает запросы, поэтому HAProxy исключает её раньше, чем она закроет listener. `timeout connect 1s` + `option redispatch 1`: запрос, не дошедший до мёртвой реплики, уходит на другую. POST повторно не отправляется: повтор возможен только на этапе connect, когда запрос ещё не передан.
- Размер тела в HAProxy не ограничен: 10 MiB + 1 байт отклоняет API (JSON 413). `tune.*buf` фиксирует сокетные буферы, чтобы пачка загрузок не довела LB (128m) до OOM.
- `s3-gateway.cfg` — TCP-сайдкар `s3-gw` в сетевом namespace `s3`: наружу (только в internal-сеть `storage`) отдаёт один S3 HTTP API как `s3:9000`. Master/filer/volume/gRPC SeaweedFS слушают только `127.0.0.1` внутри контейнера.
- `s3-gw-entrypoint.sh` — entrypoint `s3-gw`: запускает HAProxy и раз в 2 с проверяет, что namespace `s3` жив. После падения или `docker restart` контейнера `s3` в старом namespace остаётся только `lo`. Тогда скрипт через 5 с завершает контейнер, и `restart: unless-stopped` запускает `s3-gw` уже в namespace нового `s3`. У `s3` и `s3-gw`, как у `api`, `lb`, `postgres` и `redis`, стоит `restart: unless-stopped`; у one-shot `migrate` и `s3-init` политики нет. Скрипт монтируется в Linux-контейнер, поэтому `.gitattributes` держит `*.sh` в LF и на Windows.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
Конфиги примонтированы как файлы, поэтому после правки `haproxy.cfg` нужен `docker compose --env-file .env restart lb`. `up -d` не пересоздаёт LB, если не изменился сам `compose.yaml`.
+70
View File
@@ -0,0 +1,70 @@
# Lab4 load balancer: leastconn across Compose DNS name "api" (all replicas).
# Uploads hold a connection for seconds, so new requests go to the replica with the
# fewest open connections (equal load still rotates, like round-robin).
global
maxconn 2048
log stdout format raw local0
# Fixed socket buffers: a burst of 10 MiB uploads must not grow kernel buffers
# (charged to this container's 128m cgroup) until the OOM killer stops the LB.
tune.rcvbuf.client 262144
tune.sndbuf.client 262144
tune.rcvbuf.server 262144
tune.sndbuf.server 262144
defaults
mode http
log global
option httplog
option dontlognull
# A stopped replica's IP vanishes and connects to it hang instead of being refused;
# on the local bridge a real connect takes milliseconds, so fail over after 1s.
timeout connect 1s
timeout client 30s
# After the body is forwarded the API may still verify the image and write to S3
# (bounded at ≈35 s in the worst case: decode 15 s + S3 ≈15 s + DB 6 s).
timeout server 45s
timeout check 1500ms
timeout http-request 10s
retries 2
# Connect-level retries go to another replica (request not sent yet, safe for POST).
option redispatch 1
# LB-generated errors use the API error envelope and carry the contract headers.
unique-id-format %[uuid()]
http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Bad gateway","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb
http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No API replica available","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb
http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Gateway timeout","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb
resolvers docker
nameserver dns1 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 5s
hold obsolete 10s
frontend messenger_fe
bind *:8080
acl is_lb_live path /__lb/live
http-request return status 200 content-type text/plain string ok if is_lb_live
# Normal routing uses readiness. If NO replica is ready (shared dependency down:
# PostgreSQL, Redis or S3), live replicas still answer so clients get the API's
# own JSON 503 for affected endpoints and unaffected ones keep working.
use_backend messenger_api_degraded if { nbsrv(messenger_api) lt 1 }
default_backend messenger_api
backend messenger_api
balance leastconn
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
# server-template expands A records for Compose service DNS name "api".
# inter=1s fall=2 → ≈2–3s to mark a backend down after failed checks; a failed
# live connect marks it down at once (observe layer4).
server-template api 8 api:8080 check inter 1s fall 2 rise 1 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
backend messenger_api_degraded
balance leastconn
option httpchk
http-check send meth GET uri /health/live ver HTTP/1.1 hdr Host api
http-check expect status 200
server-template live 8 api:8080 check inter 1s fall 2 rise 1 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
+22
View File
@@ -0,0 +1,22 @@
# S3 gateway sidecar (lab4). Runs in the network namespace of the `s3` service:
# SeaweedFS listens on 127.0.0.1 only, this proxy publishes just the S3 HTTP API
# as s3:9000 on the internal `storage` network. Master/volume/filer UIs and the
# unauthenticated gRPC ports (1xxxx) stay unreachable from other containers.
global
maxconn 512
log stdout format raw local0
defaults
mode tcp
log global
option tcplog
timeout connect 3s
timeout client 60s
timeout server 60s
frontend s3_api
bind *:9000
default_backend seaweed_s3
backend seaweed_s3
server s3 127.0.0.1:8333
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# s3-gw shares the network namespace of the `s3` container. When s3 restarts (crash,
# `docker restart`, Docker daemon restart) it gets a NEW namespace and this one is left
# with only `lo`: HAProxy keeps running but nothing can reach it. Exit in that case so
# the restart policy starts s3-gw again inside the current namespace of s3.
haproxy -W -db -f /usr/local/etc/haproxy/haproxy.cfg &
pid=$!
trap 'kill -TERM "$pid" 2>/dev/null' TERM INT USR1
orphaned() {
# /proc/net/dev: two header lines (with '|') and one line per interface.
! grep -qv -e '^ *lo:' -e '|' /proc/net/dev
}
while kill -0 "$pid" 2>/dev/null; do
if orphaned; then
echo "s3-gw: network namespace of s3 is gone; exiting so the restart policy re-joins it" >&2
kill -TERM "$pid" 2>/dev/null
wait "$pid"
# Docker does not retry a start that fails because s3 is not running yet:
# give the restart policy of s3 time to bring it back first.
sleep 5
exit 1
fi
sleep 2 &
wait $!
done
wait "$pid"
+48
View File
@@ -0,0 +1,48 @@
-- Lab2 schema: users, chats, membership, messages.
-- Applied once via schema_migrations; safe to re-run application bootstrap.
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_username_unique UNIQUE (username)
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_chat_members_user
ON chat_members (user_id, chat_id);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
-- Keyset pagination by (created_at, id) within a chat; avoids full history scan.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at ASC, id ASC);
-- List chats for a user ordered by chat created_at/id.
CREATE INDEX IF NOT EXISTS idx_chats_created_id
ON chats (created_at ASC, id ASC);
+16
View File
@@ -0,0 +1,16 @@
-- Lab3: shared idempotency records for POST /messages (across API replicas).
CREATE TABLE IF NOT EXISTS idempotency_records (
user_id UUID NOT NULL,
method TEXT NOT NULL,
path TEXT NOT NULL,
idem_key TEXT NOT NULL,
request_hash TEXT NOT NULL,
response_status INT NOT NULL,
response_body JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
PRIMARY KEY (user_id, method, path, idem_key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_created
ON idempotency_records (created_at);
+55
View File
@@ -0,0 +1,55 @@
-- Lab4: attachment metadata (bytes live only in the private S3 bucket).
-- Additive migration: lab3 replicas keep working while it is applied.
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
width INT NOT NULL,
height INT NOT NULL,
-- Server-generated S3 key; never derived from filename, never exposed by the API.
object_key TEXT NOT NULL,
status TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_check CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_check CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_pixels_check CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000),
CONSTRAINT attachments_status_check CHECK (status IN ('ready'))
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat
ON attachments (chat_id, created_at, id);
-- Ordered links message -> attachment (an attachment may be referenced by several messages).
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments(id),
position SMALLINT NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position)
);
CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment
ON message_attachments (attachment_id);
-- Write-ahead journal of S3 objects whose metadata is not committed yet.
-- A row exists from just before the S3 PUT until the metadata commit (same
-- transaction deletes it) or until compensation/reconciliation removes the object.
CREATE TABLE IF NOT EXISTS upload_intents (
object_key TEXT PRIMARY KEY,
attachment_id UUID NOT NULL,
chat_id UUID NOT NULL,
instance_id TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_upload_intents_created
ON upload_intents (created_at);
+8
View File
@@ -0,0 +1,8 @@
-- Lab4: message text may contain U+0000, which PostgreSQL TEXT/JSONB cannot store.
-- Text is kept as exact UTF-8 bytes; the idempotent response snapshot as JSON
-- (textual JSON keeps the "\u0000" escape, JSONB rejects it).
ALTER TABLE messages
ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8');
ALTER TABLE idempotency_records
ALTER COLUMN response_body TYPE JSON USING response_body::json;
+95
View File
@@ -0,0 +1,95 @@
-- Lab5: asynchronous attachment processing (queued → worker → ready/failed).
-- Additive: extends lab4 attachments with staging, jobs, outbox, variants.
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_check;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1;
-- Final original may be filled only after the worker publishes it.
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INT NOT NULL,
name TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
width INT NOT NULL,
height INT NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_name_check
CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_sha256_check CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachment_variants_size_check CHECK (size_bytes > 0),
CONSTRAINT attachment_variants_pixels_check CHECK (width > 0 AND height > 0)
);
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id UUID PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version INT NOT NULL DEFAULT 1,
attempt INT NOT NULL DEFAULT 1,
status TEXT NOT NULL,
lease_owner TEXT,
lease_until TIMESTAMPTZ,
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_error TEXT,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT attachment_jobs_status_check
CHECK (status IN ('pending', 'leased', 'done', 'failed')),
CONSTRAINT attachment_jobs_attempt_check CHECK (attempt >= 1)
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachment_jobs_one_active
ON attachment_jobs (attachment_id, pipeline_version)
WHERE status IN ('pending', 'leased');
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
ON attachment_jobs (status, lease_until)
WHERE status = 'leased';
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_pending
ON attachment_jobs (next_attempt_at)
WHERE status = 'pending';
-- Transactional outbox: COMMIT of job intent and broker publish are decoupled.
CREATE TABLE IF NOT EXISTS outbox (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
publish_attempts INT NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_outbox_unpublished
ON outbox (id)
WHERE published_at IS NULL;
-- Operator-visible failed registry (DLQ mirror) with safe explicit replay.
CREATE TABLE IF NOT EXISTS dead_letter_jobs (
id BIGSERIAL PRIMARY KEY,
job_id UUID NOT NULL,
attachment_id UUID NOT NULL,
pipeline_version INT NOT NULL DEFAULT 1,
payload JSONB NOT NULL,
error_code TEXT NOT NULL,
attempts INT NOT NULL,
failed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
replayed_at TIMESTAMPTZ,
replay_note TEXT
);
CREATE INDEX IF NOT EXISTS idx_dead_letter_open
ON dead_letter_jobs (failed_at)
WHERE replayed_at IS NULL;
+18
View File
@@ -0,0 +1,18 @@
-- Lab5 ops hardening: durable session revocation + indexes for janitor/replay.
CREATE TABLE IF NOT EXISTS session_revocations (
token_hash TEXT PRIMARY KEY,
revoked_at TIMESTAMPTZ NOT NULL DEFAULT now(),
expires_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_session_revocations_expires
ON session_revocations (expires_at);
-- Attachments may keep staging_key while queued/processing; janitor must see it.
CREATE INDEX IF NOT EXISTS idx_attachments_staging_key
ON attachments (staging_key)
WHERE staging_key IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_attachments_status
ON attachments (status);
+9
View File
@@ -0,0 +1,9 @@
fastapi==0.115.6
uvicorn[standard]==0.32.1
pydantic==2.10.3
psycopg[binary,pool]==3.2.4
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.35.99
pika==1.3.2
Pillow==11.0.0
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""Upload → stop the API replica that served the upload → download via another replica.
Runs one round per replica (the stopped one is restored before the next round).
Usage: python scripts/attachment_failover_evidence.py
Writes evidence/attachment-failover.json; tokens are never printed.
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
def request(method, path, body=None, token=None, raw=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=10)
except urllib.error.HTTPError as err:
resp = err
return resp.code, resp.headers, resp.read()
def actor(label):
username = f"fo_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201
return json.loads(data)["id"], json.loads(data2)["token"]
def api_containers():
rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.project=mtusi2026lab4",
"--format", "{{.ID}}"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip():
sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows
def observe(token, n=30):
return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)}
def wait_two(token, timeout=90):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if len(observe(token, 20)) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("two replicas not observed")
def one_round(index, alice, bob, chat_id, seen_targets):
data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible
digest = hashlib.sha256(data).hexdigest()
for _ in range(40):
status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png",
token=alice, raw=data, headers={"Content-Type": "image/png"})
assert status == 201, status
target = h.get("X-Instance-Id")
if target not in seen_targets:
break
att = json.loads(body)
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]},
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
assert status == 201
before, ids = api_containers()
victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target)
t0 = time.monotonic()
sh(["docker", "stop", "-t", "15", victim])
stop_seconds = time.monotonic() - t0
downloads, first_ok, errors = [], None, 0
deadline = time.monotonic() + 30
while time.monotonic() < deadline and len(downloads) < 20:
try:
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
except OSError:
errors += 1
continue
if status == 200:
if first_ok is None:
first_ok = time.monotonic() - t0
downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest})
else:
errors += 1
after, _ = api_containers()
meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0]
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"])
result = {
"round": index,
"upload_served_by": target,
"stopped_container_state_after": after.get(target),
"docker_stop_seconds": round(stop_seconds, 2),
"first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None,
"downloads": len(downloads),
"download_instances": sorted({d["instance"] for d in downloads}),
"target_served_any_download": any(d["instance"] == target for d in downloads),
"all_sha256_match": all(d["sha256_ok"] for d in downloads),
"errors_during_window": errors,
"metadata_status": meta_status,
"message_link_preserved": linked,
"containers_before": before,
"containers_after": after,
}
sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"])
wait_two(alice)
seen_targets.add(target)
return result
def main():
alice_id, alice = actor("a")
bob_id, bob = actor("b")
status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice)
assert status == 201
chat_id = json.loads(data)["id"]
wait_two(alice)
seen = set()
rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)]
ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"]
and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds)
report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds}
out = ROOT / "evidence" / "attachment-failover.json"
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
for r in rounds:
print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after",
"first_download_ok_after_stop_s", "downloads", "download_instances",
"target_served_any_download", "all_sha256_match", "errors_during_window")}))
print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out)
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Cross-check the student test corpus against independent decoders.
Corrupted inputs (student-tests/imagegen.py) must NOT decode, valid ones must.
PNG → Pillow full load(); JPEG → libjpeg-turbo `djpeg -strict` (what the API runs).
Run in a throwaway container (needs Pillow + libjpeg-turbo-progs):
docker run --rm -v "$PWD:/w" -w /w python:3.12.8-slim-bookworm sh -c \
"apt-get update -qq && apt-get install -y -qq libjpeg-turbo-progs >/dev/null; \
pip install -q Pillow==11.1.0; python scripts/decoder_crosscheck.py"
"""
import io
import os
import subprocess
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "student-tests"))
import imagegen # noqa: E402
from PIL import Image # noqa: E402
# PNG spec errors (PLTE length/placement, unknown critical chunk) that Pillow tolerates;
# libpng and the API reject them.
SPEC_ONLY = {"plte_not_multiple_of_3", "plte_too_long_for_1bit", "plte_in_grayscale", "unknown_critical_chunk"}
def pillow_ok(data: bytes) -> bool:
try:
img = Image.open(io.BytesIO(data))
img.load()
return True
except Exception:
return False
def djpeg_ok(data: bytes) -> bool:
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(data)
try:
cmd = ["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, f.name]
return subprocess.run(cmd, capture_output=True).returncode == 0
finally:
os.unlink(f.name)
def main() -> int:
mismatches = 0
for name, data in imagegen.corrupted_pngs().items():
ok = pillow_ok(data)
note = " (PNG spec violation tolerated by Pillow)" if ok and name in SPEC_ONLY else ""
mismatches += ok and name not in SPEC_ONLY
print(f"corrupt png {name:32} pillow_decodes={ok}{note}")
for name, data in imagegen.corrupted_jpegs().items():
ok = djpeg_ok(data)
mismatches += ok
print(f"corrupt jpeg {name:32} djpeg_strict_decodes={ok} pillow_decodes={pillow_ok(data)}")
for interlace in (0, 1):
for size in ((1, 1), (9, 7), (33, 20)):
ok = pillow_ok(imagegen.all_filters_png(*size, interlace=interlace))
mismatches += not ok
print(f"valid png all_filters interlace={interlace} {size} pillow_decodes={ok}")
for name in ("sample.jpg", "progressive.jpg", "gray.jpg"):
ok = djpeg_ok(imagegen.fixture(name))
mismatches += not ok
print(f"valid jpeg {name:32} djpeg_strict_decodes={ok}")
print("PASS" if mismatches == 0 else f"FAIL mismatches={mismatches}")
return 1 if mismatches else 0
if __name__ == "__main__":
sys.exit(main())
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Whole-stack restart: `docker compose down` (volumes kept) → `up` → same token reads the same file.
Stronger than scripts/restart.*: every container (API, LB, PostgreSQL, Redis, SeaweedFS,
s3-gw, one-shot bootstrap jobs) is removed and recreated; only named volumes survive.
Usage: python scripts/full_restart_evidence.py (writes evidence/full-restart.json)
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
def request(method, path, body=None, token=None, raw=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=10)
except urllib.error.HTTPError as err:
resp = err
return resp.code, resp.headers, resp.read()
def actor(label):
username = f"rs_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201
return json.loads(data)["id"], json.loads(data2)["token"]
def snapshot():
rows = sh(COMPOSE + ["ps", "-a", "--format", "{{.Service}} {{.ID}} {{.State}}"]).splitlines()
return sorted(rows)
def main():
alice_id, alice = actor("a")
bob_id, bob = actor("b")
status, _, data = request("POST", "/api/v1/chats", {"title": "restart", "member_ids": [alice_id, bob_id]}, token=alice)
chat_id = json.loads(data)["id"]
payload = imagegen.rgb_png(600, 400, noise=True)
status, _, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=keep.png", token=alice,
raw=payload, headers={"Content-Type": "image/png"})
assert status == 201, status
att = json.loads(body)
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "переживёт restart\u0000ok", "attachment_ids": [att["id"]]},
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
assert status == 201
message = json.loads(msg)
before = snapshot()
volumes = sh(["docker", "volume", "ls", "--filter", "label=com.docker.compose.project=mtusi2026lab4", "--format", "{{.Name}}"]).split()
t0 = time.monotonic()
sh(COMPOSE + ["down"])
down_s = time.monotonic() - t0
after_down = snapshot()
sh(COMPOSE + ["up", "-d", "--scale", "api=2"])
deadline = time.monotonic() + 180
while True:
try:
if request("GET", "/health/ready")[0] == 200 and request("GET", "/api/v1/users/me", token=bob)[0] == 200:
break
except OSError:
pass
if time.monotonic() > deadline:
raise SystemExit("FAIL: stack not ready after restart")
time.sleep(1)
recovery_s = time.monotonic() - t0
after = snapshot()
meta = json.loads(request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[2])
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
s3_init_log = sh(COMPOSE + ["logs", "--no-log-prefix", "s3-init"]).strip().splitlines()[-3:]
result = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"volumes_kept": volumes,
"containers_before": before,
"containers_after_down": after_down,
"containers_after_up": after,
"down_seconds": round(down_s, 1),
"down_to_ready_seconds": round(recovery_s, 1),
"same_token_works": True,
"metadata_unchanged": meta == att,
"download_status": status,
"sha256_match": hashlib.sha256(content).hexdigest() == att["sha256"] == hashlib.sha256(payload).hexdigest(),
"message_preserved": any(m == message for m in history["items"]),
"s3_init_rerun_log_tail": s3_init_log,
}
ok = result["metadata_unchanged"] and status == 200 and result["sha256_match"] and result["message_preserved"]
result["passed"] = ok
out = ROOT / "evidence" / "full-restart.json"
out.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps({k: result[k] for k in ("down_to_ready_seconds", "metadata_unchanged", "download_status",
"sha256_match", "message_preserved", "passed")}))
print("WROTE", out)
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Create local synthetic secrets in ./secrets (idempotent, prints only file names).
Existing files are kept, so re-running never rotates credentials of a running
stack. secrets/s3_config.json (SeaweedFS identities) is always re-derived from the
key files and S3_BUCKET (.env or environment):
* admin — Admin/Read/Write/List, used only by the one-shot s3-init;
* messenger-api — Read/Write/List on the attachments bucket only, used by API;
* no "anonymous" identity → unauthenticated S3 requests are denied.
"""
from __future__ import annotations
import json
import os
import secrets
import string
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
ALNUM = string.ascii_letters + string.digits
def token(length: int, alphabet: str = ALNUM) -> str:
return "".join(secrets.choice(alphabet) for _ in range(length))
def bucket_name() -> str:
value = os.environ.get("S3_BUCKET", "").strip()
env_file = ROOT / ".env"
if not value and env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
if line.startswith("S3_BUCKET="):
value = line.split("=", 1)[1].strip()
return value or "messenger-attachments"
def ensure(name: str, factory) -> str:
path = SECRETS / name
if path.is_file() and path.read_text(encoding="utf-8").strip():
print("keep secrets/" + name)
return path.read_text(encoding="utf-8").strip()
value = factory()
path.write_bytes(value.encode("utf-8"))
print("create secrets/" + name)
return value
def main() -> int:
SECRETS.mkdir(exist_ok=True)
pg_password = ensure("postgres_password", lambda: token(32))
ensure("database_url", lambda: f"postgresql://messenger:{pg_password}@postgres:5432/messenger")
ensure("cache_url", lambda: "redis://redis:6379/0")
broker_password = ensure("broker_password", lambda: token(32))
ensure(
"broker_url",
lambda: f"amqp://messenger:{broker_password}@broker:5672/%2F",
)
admin_key = ensure("s3_admin_access_key_id", lambda: "ADM" + token(17, string.ascii_uppercase + string.digits))
admin_secret = ensure("s3_admin_secret_access_key", lambda: token(40))
api_key = ensure("s3_access_key_id", lambda: "API" + token(17, string.ascii_uppercase + string.digits))
api_secret = ensure("s3_secret_access_key", lambda: token(40))
bucket = bucket_name()
config = {
"identities": [
{
"name": "admin",
"credentials": [{"accessKey": admin_key, "secretKey": admin_secret}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"],
},
{
"name": "messenger-api",
"credentials": [{"accessKey": api_key, "secretKey": api_secret}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"],
},
]
}
(SECRETS / "s3_config.json").write_bytes(json.dumps(config, indent=1).encode("utf-8"))
print(f"write secrets/s3_config.json (bucket {bucket}; identities admin, messenger-api; no anonymous)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Peak RSS of `djpeg -strict -scale 1/8` on worst-case 40 Mpx JPEGs (sizing the API limit).
Run in a throwaway container with Pillow + libjpeg-turbo-progs:
python scripts/jpeg_decode_memory.py gen # writes /tmp/j/*.jpg (8000×5000)
python scripts/jpeg_decode_memory.py measure # one fresh child per file
The measuring parent stays tiny so RUSAGE_CHILDREN reflects djpeg only.
"""
import os
import resource
import subprocess
import sys
import time
OUT = "/tmp/j"
def gen() -> None:
from PIL import Image
os.makedirs(OUT, exist_ok=True)
noise = Image.effect_noise((8000, 5000), 64).convert("L")
rgb = Image.merge("RGB", (noise, noise.rotate(180), noise.transpose(Image.FLIP_LEFT_RIGHT)))
for name, img, kw in (
("baseline_420", rgb, dict(quality=30)),
("progressive_420", rgb, dict(quality=30, progressive=True)),
("progressive_444", rgb, dict(quality=20, progressive=True, subsampling=0)),
("progressive_cmyk", rgb.convert("CMYK"), dict(quality=15, progressive=True, subsampling=0)),
):
img.save(os.path.join(OUT, name + ".jpg"), "JPEG", **kw)
def measure_one(path: str) -> None:
t0 = time.monotonic()
proc = subprocess.run(["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path], capture_output=True)
took = time.monotonic() - t0
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1024
print(f"{os.path.basename(path):22} {os.path.getsize(path) / 2**20:6.2f} MiB rc={proc.returncode} "
f"{took:5.2f}s djpeg_peak_rss={peak:.1f} MiB", flush=True)
if __name__ == "__main__":
if sys.argv[1] == "gen":
gen()
elif sys.argv[1] == "one":
measure_one(sys.argv[2])
else:
for name in sorted(os.listdir(OUT)):
subprocess.run([sys.executable, __file__, "one", os.path.join(OUT, name)], check=True)
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Capture lab5 grade-3/4 defense evidence (no tokens written)."""
from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from helpers import call, chat_for, register, upload, wait_ready # noqa: E402
OUT = ROOT / "evidence" / "lab5-defense.txt"
def main() -> int:
lines: list[str] = []
def log(msg: str) -> None:
print(msg)
lines.append(msg)
alice, bob = register("ev_a"), register("ev_b")
chat = chat_for(alice, bob, title="ev")
data = imagegen.PUBLIC_PNG.read_bytes()
att = upload(alice["token"], chat["id"], data, "sample.png", "image/png", 202)["json"]
assert att["status"] == "queued"
call("GET", f"/api/v1/attachments/{att['id']}/content", 409, token=bob["token"])
ready = wait_ready(bob["token"], att)
names = sorted(v["name"] for v in ready["variants"])
thumb = call(
"GET",
f"/api/v1/attachments/{att['id']}/content?variant=thumbnail",
200,
token=bob["token"],
)
log(
f"PASS variants ready attachment={att['id']} "
f"thumb_ct={thumb['headers'].get_content_type()} names={names}"
)
stop = ROOT / "scripts" / ("stop-worker.cmd" if os.name == "nt" else "stop-worker.sh")
start = ROOT / "scripts" / ("start-worker.cmd" if os.name == "nt" else "start-worker.sh")
subprocess.run([str(stop)], check=True, cwd=str(ROOT))
try:
alice2 = register("ev_w")
chat2 = chat_for(alice2, title="wstop")
data2 = imagegen.rgb_png(64, 48)
att2 = upload(alice2["token"], chat2["id"], data2, "park.png", "image/png", 202)["json"]
call(
"POST",
f"/api/v1/chats/{chat2['id']}/messages",
201,
token=alice2["token"],
body={"text": "alive"},
)
call("GET", f"/api/v1/attachments/{att2['id']}/content", 409, token=alice2["token"])
meta = call("GET", f"/api/v1/attachments/{att2['id']}", 200, token=alice2["token"])["json"]
log(
f"PASS worker-down queued status={meta['status']} content=409 chat=201 "
f"attachment={att2['id']}"
)
finally:
subprocess.run([str(start)], check=True, cwd=str(ROOT))
ready2 = wait_ready(alice2["token"], att2, wait_seconds=90)
log(f"PASS worker-restart ready status={ready2['status']} attachment={att2['id']}")
u = register("ev_lo")
call("DELETE", "/api/v1/auth/sessions/current", 204, token=u["token"])
call("DELETE", "/api/v1/auth/sessions/current", 401, token=u["token"])
call("GET", "/api/v1/users/me", 401, token=u["token"])
log("PASS logout second DELETE=401 and /me=401")
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"WROTE {OUT}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+202
View File
@@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""Demonstrate outbox recovery and operator replay (grade 4). Writes no tokens."""
from __future__ import annotations
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
sys.path.insert(0, str(ROOT / "src"))
import imagegen # noqa: E402
from helpers import call, chat_for, register, upload, wait_ready # noqa: E402
OUT = ROOT / "evidence" / "lab5-outbox-replay.txt"
def _compose(*args: str) -> None:
subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=str(ROOT))
def main() -> int:
lines: list[str] = []
def log(msg: str) -> None:
print(msg)
lines.append(msg)
# --- Crash after side effect before ack ---
_compose("stop", "worker")
_compose(
"run",
"-d",
"--name",
"lab5-worker-crash",
"-e",
"WORKER_CRASH_AFTER_SIDE_EFFECT=1",
"worker",
)
try:
time.sleep(3)
alice = register("ox_crash")
chat = chat_for(alice, title="crash-ack")
data = imagegen.rgb_png(48, 48)
att = upload(alice["token"], chat["id"], data, "c.png", "image/png", 202)["json"]
# Wait until ready despite worker exit after side effect (redelivery).
ready = wait_ready(alice["token"], att, wait_seconds=120)
log(
f"PASS crash-after-side-effect→redelivery ready "
f"attachment={att['id']} status={ready['status']} variants={len(ready['variants'])}"
)
finally:
subprocess.call(["docker", "rm", "-f", "lab5-worker-crash"], cwd=str(ROOT))
_compose("up", "-d", "worker")
time.sleep(3)
# --- Operator replay of a forced failed job via DLQ insert path ---
# Force a permanent fail by deleting staging after accept while worker is down,
# then restore staging is impossible → failed/source_missing; then we cannot
# usefully replay without bytes. Instead: mark a job failed via SQL and replay
# while staging still exists (worker stopped briefly).
_compose("stop", "worker")
time.sleep(1)
alice2 = register("ox_rep")
chat2 = chat_for(alice2, title="replay")
data2 = imagegen.rgb_png(32, 32)
att2 = upload(alice2["token"], chat2["id"], data2, "r.png", "image/png", 202)["json"]
# Inject a dead letter for this attachment (simulating exhausted attempts) while
# keeping staging; then operator replay → queued → ready after worker start.
def _psql(sql: str) -> str:
return subprocess.check_output(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
"-t",
"-A",
"-c",
sql,
],
cwd=str(ROOT),
text=True,
).strip()
_psql(
"UPDATE attachments SET status = 'failed', error_code = 'processing_failed', "
f"updated_at = now() WHERE id = '{att2['id']}';"
)
_psql(
"UPDATE attachment_jobs SET status = 'failed', updated_at = now() "
f"WHERE attachment_id = '{att2['id']}' AND status <> 'failed';"
)
dead_id = int(
_psql(
"INSERT INTO dead_letter_jobs "
"(job_id, attachment_id, pipeline_version, payload, error_code, attempts, failed_at) "
"SELECT gen_random_uuid(), a.id, 1, "
"jsonb_build_object("
"'job_id', gen_random_uuid()::text, "
"'attachment_id', a.id::text, "
"'pipeline_version', 1, "
"'attempt', 5, "
"'created_at', '1970-01-01T00:00:00Z'"
"), 'processing_failed', 5, now() "
f"FROM attachments a WHERE a.id = '{att2['id']}' "
"RETURNING id;"
).splitlines()[0]
)
meta = call("GET", f"/api/v1/attachments/{att2['id']}", 200, token=alice2["token"])["json"]
assert meta["status"] == "failed", meta
# Replay via API container module
replay_out = subprocess.check_output(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"api",
"python",
"-m",
"messenger.replay",
"--dead-letter-id",
str(dead_id),
"--note",
"evidence replay",
],
cwd=str(ROOT),
text=True,
)
log(f"PASS operator-replay dead_letter_id={dead_id} stdout={replay_out.strip()}")
_compose("up", "-d", "worker")
ready2 = wait_ready(alice2["token"], att2, wait_seconds=90)
log(
f"PASS failed→queued→ready after replay attachment={att2['id']} "
f"status={ready2['status']} variants={len(ready2['variants'])}"
)
# --- Staging retained while queued (janitor dry-run does not clear) ---
_compose("stop", "worker")
alice3 = register("ox_st")
chat3 = chat_for(alice3)
att3 = upload(
alice3["token"],
chat3["id"],
imagegen.rgb_png(16, 16),
"s.png",
"image/png",
202,
)["json"]
dry = subprocess.check_output(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"api",
"python",
"-m",
"messenger.reconcile",
"--dry-run",
"--grace-seconds",
"0",
],
cwd=str(ROOT),
text=True,
)
meta3 = call("GET", f"/api/v1/attachments/{att3['id']}", 200, token=alice3["token"])["json"]
assert meta3["status"] in ("queued", "processing"), meta3
log(f"PASS janitor-dry-run while queued status={meta3['status']} report={dry.strip()[:300]}")
_compose("up", "-d", "worker")
wait_ready(alice3["token"], att3, wait_seconds=60)
log(f"PASS queued attachment survived janitor dry-run attachment={att3['id']}")
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"WROTE {OUT}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""LB balancing under mixed traffic: slow uploads interleaved with short GETs.
Each of 8 uploads (~1 MiB PNG) streams its body over ~4 s; between upload starts the
client sends one short GET. Records which replica served every upload and GET.
The balance algorithm is read from the running LB config. Results are merged into
evidence/lb-interleave.json keyed by algorithm. Tokens are kept in memory only.
"""
from __future__ import annotations
import base64
import collections
import http.client
import json
import os
import subprocess
import sys
import threading
import time
import uuid
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
HOST = os.environ.get("LB_HOST", "127.0.0.1")
PORT = int(os.environ.get("LB_PORT", "8080"))
UPLOADS = 8
def call(method, path, body=None, token=None, headers=None):
h = {"Accept": "application/json", **(headers or {})}
data = None
if body is not None:
data = json.dumps(body).encode()
h["Content-Type"] = "application/json"
if token:
h["Authorization"] = "Bearer " + token
req = urllib.request.Request(f"http://{HOST}:{PORT}{path}", data=data, headers=h, method=method)
resp = urllib.request.urlopen(req, timeout=30)
return resp.headers, json.loads(resp.read() or b"null")
def actor(label):
username, password = f"ilv_{label}_{uuid.uuid4().hex[:10]}", "T3st-" + uuid.uuid4().hex
_, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
_, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
return user["id"], session["token"]
def balance_algorithm():
cfg = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "exec", "-T", "lb",
"cat", "/usr/local/etc/haproxy/haproxy.cfg"], cwd=ROOT, text=True)
algos = {line.split()[1] for line in cfg.splitlines() if line.strip().startswith("balance ")}
return "/".join(sorted(algos))
def main():
algo = balance_algorithm()
alice, token = actor("a")
bob, _ = actor("b")
_, chat = call("POST", "/api/v1/chats", {"title": "interleave", "member_ids": [alice, bob]}, token=token)
payload = imagegen.rgb_png(700, 500, noise=True)
uploads, gets = [], []
def slow_upload(i):
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=ilv{i}.png")
for k, v in {"Authorization": "Bearer " + token, "Content-Type": "image/png",
"Content-Length": str(len(payload))}.items():
conn.putheader(k, v)
conn.endheaders()
step = len(payload) // 20
for off in range(0, len(payload), step):
conn.send(payload[off:off + step])
time.sleep(0.2)
resp = conn.getresponse()
resp.read()
uploads.append({"upload": i, "status": resp.status, "instance": resp.getheader("X-Instance-Id")})
threads = []
for i in range(UPLOADS):
th = threading.Thread(target=slow_upload, args=(i,))
th.start()
threads.append(th)
time.sleep(0.15)
gets.append(call("GET", "/api/v1/users/me", token=token)[0]["X-Instance-Id"])
time.sleep(0.15)
for th in threads:
th.join()
result = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"balance": algo,
"uploads": UPLOADS,
"upload_bytes": len(payload),
"uploads_per_replica": dict(collections.Counter(u["instance"] for u in uploads)),
"upload_statuses": dict(collections.Counter(str(u["status"]) for u in uploads)),
"short_gets_per_replica": dict(collections.Counter(gets)),
}
out = ROOT / "evidence" / "lb-interleave.json"
report = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {}
report[algo] = result
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps(result, ensure_ascii=False))
print("WROTE", out)
if __name__ == "__main__":
main()
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Operator replay of failed attachment jobs (lab5 grade 4).
Usage (inside API/worker network or with secrets mounted):
python scripts/replay_failed.py --dead-letter-id N
python scripts/replay_failed.py --attachment-id UUID
python -m messenger.replay --list
Safe against ready results: already-ready attachments are skipped, not re-executed.
Staging for the attachment is preserved; janitor will not delete it while queued.
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from messenger import db # noqa: E402
from messenger.outbox import OutboxPublisher # noqa: E402
from messenger.store import Store # noqa: E402
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dead-letter-id", type=int, default=None)
parser.add_argument("--attachment-id", default=None)
parser.add_argument("--list", action="store_true", help="list open dead letters")
parser.add_argument("--note", default="operator replay")
args = parser.parse_args()
db.init_pool()
store = Store()
try:
if args.list:
with db.get_pool().connection() as conn:
rows = conn.execute(
"""
SELECT id, job_id, attachment_id, error_code, attempts, failed_at
FROM dead_letter_jobs
WHERE replayed_at IS NULL
ORDER BY failed_at
LIMIT 50
"""
).fetchall()
for r in rows:
print(
f"id={r['id']} attachment={r['attachment_id']} "
f"error={r['error_code']} attempts={r['attempts']}"
)
print(f"open_dead_letters={len(rows)}")
return 0
dead_id = args.dead_letter_id
if dead_id is None and args.attachment_id:
with db.get_pool().connection() as conn:
row = conn.execute(
"""
SELECT id FROM dead_letter_jobs
WHERE attachment_id = %s AND replayed_at IS NULL
ORDER BY failed_at DESC
LIMIT 1
""",
(args.attachment_id,),
).fetchone()
if row is None:
print("no open dead letter for attachment", file=sys.stderr)
return 2
dead_id = int(row["id"])
if dead_id is None:
parser.error("need --dead-letter-id or --attachment-id or --list")
result = store.replay_dead_letter(dead_id, note=args.note)
if result is None:
print("dead letter not found", file=sys.stderr)
return 2
OutboxPublisher(store).flush_once()
print(f"replay_result={result} dead_letter_id={dead_id}")
return 0
finally:
db.close_pool()
if __name__ == "__main__":
raise SystemExit(main())
+19
View File
@@ -0,0 +1,19 @@
@echo off
setlocal
rem Restart API replicas, worker and ALL state stores; volumes are preserved.
cd /d "%~dp0\.."
if "%API_REPLICAS%"=="" set API_REPLICAS=2
docker compose --env-file .env restart postgres redis broker s3
if errorlevel 1 exit /b 1
rem s3-gw shares the s3 network namespace and must be restarted after it.
docker compose --env-file .env restart s3-gw
if errorlevel 1 exit /b 1
docker compose --env-file .env restart api worker
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps --scale api=%API_REPLICAS% api
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps worker
if errorlevel 1 exit /b 1
docker compose --env-file .env restart lb
if errorlevel 1 exit /b 1
exit /b 0
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
# Restart API replicas, worker and ALL state stores; preserve volumes.
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
docker compose --env-file .env restart postgres redis broker s3
# s3-gw shares the s3 network namespace and must be restarted after it.
docker compose --env-file .env restart s3-gw
# "up -d" alone leaves running replicas untouched; restart them, then restore the scale
# (migrate/s3-init already applied; skip one-shots).
docker compose --env-file .env restart api worker
docker compose --env-file .env up -d --no-deps --scale "api=${REPLICAS}" api
docker compose --env-file .env up -d --no-deps worker
docker compose --env-file .env restart lb
exit 0
+51
View File
@@ -0,0 +1,51 @@
@echo off
setlocal EnableExtensions EnableDelayedExpansion
rem Grade-4 rolling replacement: replace API containers one-by-one under LB.
rem Usage: scripts\rolling-update.cmd
cd /d "%~dp0\.."
set "REPLICAS=%API_REPLICAS%"
if "%REPLICAS%"=="" set "REPLICAS=2"
set "URL=http://127.0.0.1:8080/health/ready"
set "IDS="
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do set "IDS=!IDS! %%I"
set "KNOWN=%IDS%"
set /a COUNT=0
for %%I in (%IDS%) do (
set /a COUNT+=1
echo rolling: replacing %%I
rem -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
docker stop -t 15 %%I >nul || exit /b 1
docker rm %%I >nul || exit /b 1
rem --no-recreate: the surviving replica keeps serving even if the image/config changed.
docker compose --env-file .env up -d --no-deps --no-recreate --scale api=%REPLICAS% api || exit /b 1
call :wait_new || exit /b 1
)
echo rolling update finished (%COUNT% steps, scale=%REPLICAS%)
exit /b 0
:wait_new
set "NEW="
for /f "usebackq delims=" %%J in (`docker compose --env-file .env ps -q api`) do (
if "!KNOWN:%%J=!"=="!KNOWN!" set "NEW=%%J"
)
if "%NEW%"=="" (
echo rolling: expected one new replica >&2
exit /b 1
)
set "KNOWN=%KNOWN% %NEW%"
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %NEW%`) do set "HOST=%%H"
rem Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
set /a TRIES=0
:poll
curl.exe -s -D - -o NUL "%URL%" | findstr /I /B /C:"x-instance-id: %HOST%" >nul && goto ready
set /a TRIES+=1
if %TRIES% geq 90 (
echo rolling: %HOST% not in LB rotation >&2
exit /b 1
)
ping -n 2 127.0.0.1 >nul
goto poll
:ready
echo rolling: step %COUNT% ready (%HOST%)
exit /b 0
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
set -eu
# Grade-4 rolling replacement: replace API containers one-by-one under LB.
# Usage: scripts/rolling-update.sh
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
URL="http://127.0.0.1:8080/health/ready"
ids=$(docker compose --env-file .env ps -q api | tr -d '\r')
known="$ids"
count=0
for id in $ids; do
[ -n "$id" ] || continue
count=$((count + 1))
printf '%s\n' "rolling: replacing $id"
# -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
docker stop -t 15 "$id" >/dev/null
docker rm "$id" >/dev/null
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=${REPLICAS}" api
new=$(docker compose --env-file .env ps -q api | tr -d '\r' | grep -v -F "$known" || true)
[ "$(printf '%s\n' "$new" | grep -c .)" -eq 1 ] || { printf '%s\n' "rolling: expected one new replica" >&2; exit 1; }
known="$known
$new"
host=$(docker inspect -f '{{.Config.Hostname}}' "$new" | tr -d '\r')
# Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
i=0
until curl -s -D - -o /dev/null "$URL" | tr -d '\r' | grep -qi "^x-instance-id: ${host}\$"; do
i=$((i + 1))
[ "$i" -lt 180 ] || { printf '%s\n' "rolling: $host not in LB rotation" >&2; exit 1; }
sleep 0.5
done
printf '%s\n' "rolling: step $count ready ($host)"
done
printf '%s\n' "rolling update finished ($count steps, scale=${REPLICAS})"
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""Grade-4: closed-loop POSTs during rolling API replacement."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import threading
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
REPLICAS = os.environ.get("API_REPLICAS", "2")
def call(method, path, body=None, token=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
except Exception as exc:
return {"status": 0, "error": type(exc).__name__}
raw = resp.read()
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")}
def api_ids():
out = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True
)
return out.split()
def hostname(cid):
return subprocess.check_output(
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
).strip()
def replace_replica(old_id, known):
"""Stop+remove one replica, create a fresh one, wait until the LB routes to it."""
subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL)
subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL)
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
subprocess.check_call(
["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate",
"--scale", f"api={REPLICAS}", "api"],
cwd=ROOT,
)
new = [cid for cid in api_ids() if cid not in known]
if len(new) != 1:
raise RuntimeError(f"expected one new replica, got {new}")
host = hostname(new[0])
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
r = call("GET", "/health/ready")
if r.get("status") == 200 and r.get("instance") == host:
return new[0], host
time.sleep(0.5)
raise RuntimeError(f"new replica {host} not in LB rotation after 90s")
def bootstrap():
uname = "roll_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw})
assert user["status"] == 201, user
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"]
peer_u = "rollb_" + uuid.uuid4().hex[:10]
peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]},
token=tok,
)["json"]
return tok, chat["id"]
def main():
tok, chat_id = bootstrap()
stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []}
stop = threading.Event()
def worker():
while not stop.is_set():
key = uuid.uuid4().hex
t0 = time.monotonic()
r = call(
"POST",
f"/api/v1/chats/{chat_id}/messages",
{"text": "roll-" + key},
token=tok,
headers={"Idempotency-Key": key},
)
dt = (time.monotonic() - t0) * 1000
if r.get("status") == 201:
stats["ok"] += 1
stats["latencies_ms"].append(dt)
else:
stats["err"] += 1
msg = ((r.get("json") or {}).get("error") or {}).get("message", "")
code = f"{r.get('status') or r.get('error')} {msg}".strip()
stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1
time.sleep(0.05)
ids = api_ids()
known = set(ids)
replaced = []
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
started = time.monotonic()
for th in threads:
th.start()
time.sleep(2) # warm-up
# Rolling: replace each original replica in turn; the next one is stopped
# only after the LB already routes requests to the fresh container.
for cid in ids:
old_host = hostname(cid)
new_id, new_host = replace_replica(cid, known)
known.add(new_id)
replaced.append({"old": old_host, "new": new_host})
time.sleep(2)
time.sleep(2)
stop.set()
for th in threads:
th.join(timeout=5)
duration = time.monotonic() - started
lats = sorted(stats["latencies_ms"])
def pct(p):
if not lats:
return None
return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2)
summary = {
"successful": stats["ok"],
"errors": stats["err"],
"errors_by_status": stats["err_by_status"],
"p50_ms": pct(0.50),
"p95_ms": pct(0.95),
"p99_ms": pct(0.99),
"duration_s": round(duration, 1),
"replicas_rolled": len(replaced),
"replaced": replaced,
}
out = ROOT / "evidence" / "rolling-load.json"
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(summary, ensure_ascii=False))
print("WROTE", out)
if __name__ == "__main__":
main()
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Grade-4 evidence: stop each observed API replica in turn; restore between steps."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
def call(method, path, body=None, token=None, headers=None, expected=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
raw = resp.read()
status = resp.code
if expected is not None and status != expected:
raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}")
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": status, "headers": resp.headers, "json": js}
def header(resp, name: str) -> str:
return resp["headers"].get(name) or resp["headers"].get(name.lower()) or ""
def restore_api(target: str):
replicas = os.environ.get("API_REPLICAS", "2")
subprocess.check_call(
[
"docker",
"compose",
"--env-file",
".env",
"up",
"-d",
"--no-deps",
"--scale",
f"api={replicas}",
"--remove-orphans",
"api",
],
cwd=ROOT,
)
# Public ready is 200 via the surviving replica; wait until the LB routes to the restored one.
for _ in range(180):
try:
if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError(f"replica {target} not back in LB rotation after restore")
def one_round(round_no: int) -> dict:
uname = "seq_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call(
"POST",
"/api/v1/users",
{"username": uname, "display_name": "Seq", "password": pw},
expected=201,
)["json"]
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + cred},
expected=201,
)["json"]["token"]
peer_u = "seqb_" + uuid.uuid4().hex[:10]
peer = call(
"POST",
"/api/v1/users",
{"username": peer_u, "display_name": "SeqB", "password": pw},
expected=201,
)["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "seq", "member_ids": [user["id"], peer["id"]]},
token=tok,
expected=201,
)["json"]
observed = set()
for _ in range(40):
observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id"))
if len(observed) < 2:
raise RuntimeError(f"need ≥2 instances, got {observed}")
target = sorted(observed)[round_no % len(observed)]
env = dict(os.environ)
env["TARGET_INSTANCE_ID"] = target
script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh")
started = time.monotonic()
proc = subprocess.Popen([str(script)], env=env, cwd=ROOT)
ok = fail = 0
first = None
after = []
while time.monotonic() - started < 20:
code = proc.poll()
try:
key = uuid.uuid4().hex
r = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": "seq-" + key},
token=tok,
headers={"Idempotency-Key": key},
expected=201,
)
ok += 1
if code == 0:
if first is None:
first = time.monotonic() - started
after.append(header(r, "X-Instance-Id"))
except Exception:
fail += 1
time.sleep(0.1)
if proc.poll() is None:
proc.wait(timeout=10)
result = {
"round": round_no,
"observed_before": sorted(observed),
"target": target,
"successful_writes": ok,
"failed_attempts": fail,
"first_recovery_seconds": round(first, 3) if first is not None else None,
"target_absent_in_last5": target not in after[-5:],
}
t0 = time.monotonic()
restore_api(target)
result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3)
return result
def main():
results = []
for i in range(2):
results.append(one_round(i))
print(json.dumps(results[-1], ensure_ascii=False))
out = ROOT / "evidence" / "seq-failover.json"
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print("WROTE", out)
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0.."
docker compose --env-file .env up -d --no-deps worker
exit /b 0
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env up -d --no-deps worker
exit 0
+17
View File
@@ -0,0 +1,17 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
if "%TARGET_INSTANCE_ID%"=="" (
echo stop-one: TARGET_INSTANCE_ID required >&2
exit /b 1
)
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do (
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %%I`) do (
if /I "%%H"=="%TARGET_INSTANCE_ID%" (
docker stop -t 15 %%I >nul
exit /b 0
)
)
)
echo stop-one: replica not found >&2
exit /b 1
+44
View File
@@ -0,0 +1,44 @@
#!/bin/sh
set -eu
# Stop ONE API replica matching TARGET_INSTANCE_ID (= X-Instance-Id / hostname).
# Leave it stopped; do not touch DB/Redis/LB/volumes.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
found=""
# Compose project containers for service api
ids=$(docker compose --env-file .env ps -q api 2>/dev/null || true)
for id in $ids; do
[ -n "$id" ] || continue
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
iid=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$id" | sed -n 's/^INSTANCE_ID=//p' | head -n1)
if [ -n "$iid" ] && [ "$iid" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
# Fallback: any running container with matching hostname (scaled names differ).
for id in $(docker ps -q --filter "ancestor=mtusi-messenger:lab4"); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
fi
if [ -z "$found" ]; then
printf '%s\n' "stop-one: replica not found for TARGET_INSTANCE_ID" >&2
exit 1
fi
# Graceful stop (= stop_grace_period): on SIGTERM the API reports not-ready but keeps
# serving for a 3 s drain window, then finishes in-flight requests and exits (~3.5 s).
docker stop -t 15 "$found" >/dev/null
exit 0
+5
View File
@@ -0,0 +1,5 @@
@echo off
REM Stop the attachment worker only; API/broker stay up so uploads return 202 queued.
cd /d "%~dp0.."
docker compose --env-file .env stop worker
exit /b 0
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
# Stop the attachment worker only; API/broker stay up so uploads return 202 queued.
cd "$(dirname "$0")/.."
docker compose --env-file .env stop worker
exit 0
+12
View File
@@ -0,0 +1,12 @@
@echo off
setlocal
cd /d "%~dp0\.."
if "%API_REPLICAS%"=="" set API_REPLICAS=2
python scripts\init_secrets.py
if errorlevel 1 exit /b 1
docker compose --env-file .env config --quiet
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --build --scale api=%API_REPLICAS%
if errorlevel 1 exit /b 1
echo stack up: api scale=%API_REPLICAS%, entry http://127.0.0.1:8080
exit /b 0
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
# Bootstrap lab4 stack with ≥2 API replicas behind LB, PostgreSQL, Redis and private S3.
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
python3 scripts/init_secrets.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale "api=${REPLICAS}"
printf '%s\n' "stack up: api scale=${REPLICAS}, entry http://127.0.0.1:8080"
+284
View File
@@ -0,0 +1,284 @@
#!/usr/bin/env python3
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
file) keeps the API process RSS growth far below N × file size; total in-flight
work per replica is capped by MAX_CONCURRENT_UPLOADS.
Method (per concurrency level, fresh replicas):
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
Writes evidence/upload-memory.json. Tokens are kept in memory only.
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args, check=True) -> str:
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
return proc.stdout
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=timeout)
except urllib.error.HTTPError as err:
resp = err
data = resp.read()
return resp.code, resp.headers, data
def actor(label):
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, status
user = json.loads(data)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, status
return user["id"], json.loads(data)["token"]
def wait_ready(timeout=120):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if request("GET", "/health/ready", timeout=3)[0] == 200:
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
if len(seen) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("stack not ready")
def containers():
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
def proc_status(cid):
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
fields = {}
for line in out.splitlines():
if line.startswith(("VmRSS", "VmHWM")):
k, v = line.split(":", 1)
fields[k] = int(v.split()[0]) // 1024 # MiB
return fields
def cgroup(cid):
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
vals = dict(line.split() for line in stat.splitlines() if line)
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
mib = 1024 * 1024
return {
"anon_mib": round(int(vals["anon"]) / mib, 1),
"file_mib": round(int(vals["file"]) / mib, 1),
"current_mib": round(int(current) / mib, 1),
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
}
def sampler(cid, stop, out):
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
while not stop.is_set():
try:
c = cgroup(cid)
for k in ("anon_mib", "file_mib", "current_mib"):
best[k] = max(best[k], c[k])
best["samples"] += 1
except Exception:
pass
time.sleep(0.25)
out[cid] = best
def pct(values, q):
values = sorted(values)
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
def run_level(n, payload, chat_id, token):
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
# s3-gw shares the s3 network namespace and must be restarted after it.
sh(COMPOSE + ["restart", "s3"])
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
wait_ready()
by_host = containers()
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
# Warm-up: at least one upload served by every replica.
served = set()
for _ in range(20):
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
served.add(h.get("X-Instance-Id"))
if served >= set(by_host):
break
baseline = {}
for host, cid in by_host.items():
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
baseline[host] = {**proc_status(cid), **cgroup(cid)}
lb_baseline = cgroup(lb)
s3_baseline = cgroup(s3c)
stop = threading.Event()
sampled = {}
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
for t in threads:
t.start()
time.sleep(0.5)
barrier = threading.Barrier(n)
results = [None] * n
def one(i):
barrier.wait()
t0 = time.monotonic()
try:
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
token=token, raw=payload, headers={"Content-Type": "image/png"})
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
except OSError as exc:
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
started = time.monotonic()
for w in workers:
w.start()
for w in workers:
w.join()
wall = time.monotonic() - started
time.sleep(1.0)
stop.set()
for t in threads:
t.join()
replicas = {}
for host, cid in by_host.items():
after = proc_status(cid)
cg = cgroup(cid)
replicas[host] = {
"uploads_served": sum(1 for r in results if r[2] == host),
"rss_baseline_mib": baseline[host]["VmRSS"],
"rss_peak_vmhwm_mib": after["VmHWM"],
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
"cgroup_peak_mib": cg["peak_mib"],
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
"samples": sampled.get(cid, {}).get("samples"),
}
latencies = [r[1] for r in results if r[0] == 201]
statuses = {}
for r in results:
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
per_replica_max = max(v["uploads_served"] for v in replicas.values())
return {
"concurrency": n,
"payload_bytes": len(payload),
"statuses": statuses,
"wall_seconds": round(wall, 3),
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
"replicas": replicas,
"lb": {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
"baseline_current_mib": lb_baseline["current_mib"],
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
},
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
}
def _s3_report(cid, baseline, sampled):
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
return {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
"baseline_current_mib": baseline["current_mib"],
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
"sampled_max_anon_mib": sampled.get("anon_mib"),
"sampled_max_current_mib": sampled.get("current_mib"),
"running": running,
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
}
def main():
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
p.add_argument("--size-mib", type=float, default=9.5)
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
args = p.parse_args()
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
_, token = actor("a")
bob_id, _ = actor("b")
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
assert status == 201, status
chat_id = json.loads(data)["id"]
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
list(containers().values())[0]]).split()
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
"api_cpus": int(limits[1]) / 1e9,
"max_concurrent_uploads_per_replica": 8,
"levels": [],
}
for n in args.levels:
level = run_level(n, payload, chat_id, token)
report["levels"].append(level)
print(json.dumps(level, ensure_ascii=False), flush=True)
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
print("WROTE", args.output)
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""Messenger API for lab4 (LB + shared sessions + idempotent messages + S3 attachments)."""
+36
View File
@@ -0,0 +1,36 @@
"""Run the messenger HTTP server."""
from __future__ import annotations
import sys
from pathlib import Path
_SRC = Path(__file__).resolve().parents[1]
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
import uvicorn
from messenger import config, lifecycle
def main() -> None:
server = lifecycle.DrainingServer(
uvicorn.Config(
"messenger.app:app",
host=config.app_host(),
port=config.app_port(),
log_level="info",
access_log=False,
timeout_keep_alive=5,
timeout_graceful_shutdown=config.shutdown_timeout_seconds(),
),
drain_seconds=config.shutdown_drain_seconds(),
)
server.run()
if not server.started:
sys.exit(3)
if __name__ == "__main__":
main()
+909
View File
@@ -0,0 +1,909 @@
"""HTTP API: multi-replica messenger behind LB with async attachments (lab5)."""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import re
import urllib.parse
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any, List, Optional, Tuple
from fastapi import FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.requests import ClientDisconnect
from messenger import attachments, config, db, lifecycle, reconcile, s3, sessions
from messenger.outbox import OutboxPublisher
from messenger.sessions import SessionStoreError
from messenger.store import (
Attachment,
AttachmentFailed,
AttachmentNotFound,
CommitUncertain,
IntentLost,
Store,
StoreUnavailable,
format_ts,
)
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z")
UUID_RE = re.compile(
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z"
)
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z")
BODY_READ_TIMEOUT_S = 5.0
PASSWORD_MIN = 12
PASSWORD_MAX = 128
store = Store()
ready = False
upload_slots: Optional[asyncio.Semaphore] = None
outbox_publisher: Optional[OutboxPublisher] = None
INSTANCE_ID = config.instance_id()
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
UPLOAD_SLOT_WAIT_S = 5.0
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
log = logging.getLogger("messenger")
@asynccontextmanager
async def lifespan(_app: FastAPI):
# SIGTERM/SIGINT are handled by lifecycle.DrainingServer: readiness goes 503 first,
# requests are served through the drain window and this shutdown runs last.
global ready, upload_slots, outbox_publisher
db.init_pool()
# Migrations run once via `python -m messenger.migrate` (compose migrate service).
sessions.init_client()
s3.init_client()
attachments.prepare_spool_dir()
upload_slots = asyncio.Semaphore(config.max_concurrent_uploads())
outbox_publisher = OutboxPublisher(store)
outbox_publisher.start()
reconcile.start_background()
ready = True
log.info("messenger started instance_id=%s", INSTANCE_ID)
yield
ready = False
reconcile.stop_background()
if outbox_publisher is not None:
outbox_publisher.stop()
outbox_publisher = None
sessions.close_client()
db.close_pool()
log.info("messenger stopped")
def request_id_from(request: Request) -> str:
incoming = request.headers.get("x-request-id")
if incoming and incoming.strip():
return incoming.strip()[:128]
return str(uuid.uuid4())
def error_body(code: str, message: str, request_id: str) -> dict:
return {"error": {"code": code, "message": message, "request_id": request_id}}
def common_headers(request_id: str, extra: Optional[dict] = None) -> dict:
headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}
if extra:
headers.update(extra)
return headers
def error_response(
status: int,
code: str,
message: str,
request_id: str,
www_authenticate: Optional[str] = None,
extra_headers: Optional[dict] = None,
) -> JSONResponse:
headers = common_headers(request_id, extra_headers)
if www_authenticate:
headers["WWW-Authenticate"] = www_authenticate
return JSONResponse(
status_code=status,
content=error_body(code, message, request_id),
headers=headers,
)
def is_blank(value: str) -> bool:
return value.strip() == ""
def has_nul(value: str) -> bool:
# Names/titles/passwords live in PostgreSQL TEXT, which cannot store U+0000.
return "\x00" in value
def has_lone_surrogate(value: str) -> bool:
return any(0xD800 <= ord(ch) <= 0xDFFF for ch in value)
def codepoint_len(value: str) -> int:
return len(value)
def parse_uuid(value: str) -> Optional[str]:
if not isinstance(value, str) or not UUID_RE.match(value):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def encode_cursor(resource: str, created_at: datetime, item_id: str) -> str:
raw = f"{resource}|{created_at.strftime('%Y-%m-%dT%H:%M:%SZ')}|{item_id}"
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii")
def decode_cursor(cursor: str, expected_resource: str) -> Tuple[datetime, uuid.UUID]:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii")).decode("utf-8")
resource, ts, item_id = raw.split("|", 2)
if resource != expected_resource:
raise ValueError("wrong resource")
created_at = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(
tzinfo=timezone.utc
)
return created_at, uuid.UUID(item_id)
except Exception as exc:
raise ValueError("invalid cursor") from exc
app = FastAPI(
title="MTUSI Messenger lab4",
docs_url=None,
redoc_url=None,
lifespan=lifespan,
)
@app.middleware("http")
async def add_request_id_and_gate(request: Request, call_next):
rid = request_id_from(request)
request.state.request_id = rid
path = request.url.path
if not ready and path not in ("/health/live",):
if path == "/health/ready":
return error_response(503, "unavailable", "Shutting down", rid)
return error_response(503, "unavailable", "Service unavailable", rid)
response = await call_next(request)
if "X-Request-Id" not in response.headers:
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = INSTANCE_ID
log.info(
"%s %s -> %s request_id=%s instance_id=%s",
request.method,
path,
response.status_code,
rid,
INSTANCE_ID,
)
return response
class ApiError(Exception):
def __init__(
self,
status: int,
code: str,
message: str,
www_authenticate: Optional[str] = None,
headers: Optional[dict] = None,
):
self.status = status
self.code = code
self.message = message
self.www_authenticate = www_authenticate
self.headers = headers
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
return error_response(
exc.status,
exc.code,
exc.message,
rid,
www_authenticate=exc.www_authenticate,
extra_headers=exc.headers,
)
@app.exception_handler(RequestValidationError)
async def validation_handler(request: Request, exc: RequestValidationError):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
return error_response(400, "invalid_request", "Invalid request", rid)
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
if exc.status_code == 404:
return error_response(404, "not_found", "Not found", rid)
if exc.status_code == 405:
return error_response(400, "invalid_request", "Method not allowed", rid)
return error_response(exc.status_code, "invalid_request", "Request failed", rid)
@app.exception_handler(StoreUnavailable)
async def store_unavailable_handler(request: Request, exc: StoreUnavailable):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
return error_response(503, "unavailable", "Database unavailable", rid)
@app.exception_handler(SessionStoreError)
async def session_store_unavailable_handler(request: Request, exc: SessionStoreError):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
return error_response(503, "unavailable", "Session store unavailable", rid)
@app.exception_handler(s3.S3Unavailable)
async def s3_unavailable_handler(request: Request, exc: s3.S3Unavailable):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
return error_response(503, "unavailable", "Attachment storage unavailable", rid)
@app.exception_handler(Exception)
async def unhandled_handler(request: Request, exc: Exception):
rid = getattr(request.state, "request_id", None) or request_id_from(request)
log.exception("unhandled error request_id=%s", rid)
return error_response(500, "internal_error", "Internal error", rid)
async def read_json_object(request: Request) -> Any:
content_type = request.headers.get("content-type", "")
media = content_type.split(";")[0].strip().lower()
if media and media != "application/json":
raise ApiError(415, "unsupported_media_type", "Unsupported media type")
try:
body = await asyncio.wait_for(request.body(), timeout=BODY_READ_TIMEOUT_S)
except asyncio.TimeoutError:
raise ApiError(400, "invalid_request", "Request body timeout")
if not body:
raise ApiError(400, "invalid_request", "Empty body")
try:
text = body.decode("utf-8")
# Reject trailing garbage after one JSON value (read to EOF).
data, end = json.JSONDecoder().raw_decode(text)
if text[end:].strip():
raise ApiError(400, "invalid_request", "Malformed JSON")
except ApiError:
raise
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
raise ApiError(400, "invalid_request", "Malformed JSON")
if not isinstance(data, dict):
raise ApiError(400, "invalid_request", "JSON object required")
return data
def require_bearer(authorization: Optional[str]) -> Tuple[str, str]:
"""Return (user_id, token). X-User-Id is ignored."""
bearer_challenge = 'Bearer realm="messenger"'
if not authorization or not authorization.strip():
raise ApiError(
401, "unauthorized", "Missing bearer token", www_authenticate=bearer_challenge
)
parts = authorization.strip().split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
# Basic on business endpoints must not work.
raise ApiError(
401, "unauthorized", "Invalid authorization", www_authenticate=bearer_challenge
)
token = parts[1].strip()
try:
session = sessions.get_session(token)
except SessionStoreError:
raise ApiError(503, "unavailable", "Session store unavailable")
if session is None:
raise ApiError(
401, "unauthorized", "Invalid or expired session", www_authenticate=bearer_challenge
)
user_id = session["user_id"]
user = store.get_user(user_id)
if user is None:
raise ApiError(
401, "unauthorized", "Invalid or expired session", www_authenticate=bearer_challenge
)
return user_id, token
def require_chat_member(chat_id_raw: str, user_id: str):
chat_id = parse_uuid(chat_id_raw)
if chat_id is None:
raise ApiError(400, "invalid_request", "Invalid chat_id")
chat = store.get_chat(chat_id)
if chat is None or user_id not in chat.member_ids:
raise ApiError(404, "not_found", "Chat not found")
return chat
def parse_limit(limit: Optional[int]) -> int:
if limit is None:
return 50
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 1 or limit > 100:
raise ApiError(400, "invalid_request", "Invalid limit")
return limit
def validate_display_text(value: Any, field: str, max_len: int) -> str:
if not isinstance(value, str) or is_blank(value) or has_nul(value):
raise ApiError(400, "invalid_request", f"Invalid {field}")
if has_lone_surrogate(value):
raise ApiError(400, "invalid_request", f"Invalid {field}")
length = codepoint_len(value)
if length < 1 or length > max_len:
raise ApiError(400, "invalid_request", f"Invalid {field}")
return value
def validate_password(value: Any) -> str:
# Required strings must not be whitespace-only (contracts/README).
if not isinstance(value, str) or is_blank(value) or has_nul(value):
raise ApiError(400, "invalid_request", "Invalid password")
if has_lone_surrogate(value):
raise ApiError(400, "invalid_request", "Invalid password")
length = codepoint_len(value)
if length < PASSWORD_MIN or length > PASSWORD_MAX:
raise ApiError(400, "invalid_request", "Invalid password")
return value
def parse_basic_credentials(authorization: Optional[str]) -> Tuple[str, str]:
challenge = 'Basic realm="messenger", charset="UTF-8"'
if not authorization or not authorization.strip():
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
parts = authorization.strip().split(None, 1)
if len(parts) != 2 or parts[0].lower() != "basic":
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
try:
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
except Exception:
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
if ":" not in decoded:
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
username, password = decoded.split(":", 1)
# Username ASCII-only per contract; password may contain ':'.
if not username or any(ord(ch) > 127 for ch in username):
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
return username, password
@app.get("/health/live")
async def liveness(request: Request):
rid = request.state.request_id
return JSONResponse(content={"status": "ok"}, headers=common_headers(rid))
@app.get("/health/ready")
async def readiness(request: Request):
rid = request.state.request_id
if not ready:
return error_response(503, "unavailable", "Not ready", rid)
if lifecycle.draining:
# Only readiness reflects the drain: other endpoints keep serving until the
# LB has stopped routing here and the listener is closed.
return error_response(503, "unavailable", "Shutting down", rid)
# Mandatory synchronous path in lab4 = sessions + metadata + attachment bytes,
# so PostgreSQL, Redis and S3 are all readiness dependencies.
if not await run_in_threadpool(_dependencies_ok):
return error_response(503, "unavailable", "Dependency unavailable", rid)
return JSONResponse(content={"status": "ok"}, headers=common_headers(rid))
def _dependencies_ok() -> bool:
return db.ping() and sessions.ping() and s3.ping()
@app.post("/api/v1/users")
async def register_user(request: Request):
rid = request.state.request_id
data = await read_json_object(request)
allowed = {"username", "display_name", "password"}
if set(data.keys()) - allowed:
raise ApiError(400, "invalid_request", "Unexpected fields")
if "username" not in data or "display_name" not in data or "password" not in data:
raise ApiError(400, "invalid_request", "Missing required fields")
username = data["username"]
display_name = data["display_name"]
if (
not isinstance(username, str)
or is_blank(username)
or has_lone_surrogate(username)
or not USERNAME_RE.match(username)
):
raise ApiError(400, "invalid_request", "Invalid username")
display_name = validate_display_text(display_name, "display_name", 100)
password = validate_password(data["password"])
try:
user = store.create_user(username, display_name, password)
except KeyError:
raise ApiError(409, "conflict", "Username already taken")
return JSONResponse(
status_code=201,
content=user.to_dict(),
headers=common_headers(rid),
)
@app.post("/api/v1/auth/sessions")
async def create_session(request: Request):
rid = request.state.request_id
challenge = 'Basic realm="messenger", charset="UTF-8"'
# Contract: Basic only, no JSON body (even with valid credentials).
try:
raw_body = await asyncio.wait_for(request.body(), timeout=BODY_READ_TIMEOUT_S)
except asyncio.TimeoutError:
raise ApiError(400, "invalid_request", "Request body timeout")
if raw_body and raw_body.strip():
raise ApiError(400, "invalid_request", "Login must not include a body")
username, password = parse_basic_credentials(
request.headers.get("authorization")
)
# Uniform 401 for wrong user / wrong password (no user enumeration).
user = store.get_user_by_username(username)
if user is None or not store.verify_password(user.password_hash, password):
raise ApiError(
401, "unauthorized", "Invalid credentials", www_authenticate=challenge
)
try:
session = sessions.create_session(user.id)
except SessionStoreError:
raise ApiError(503, "unavailable", "Session store unavailable")
body = {
"token": session["token"],
"token_type": "Bearer",
"expires_at": session["expires_at"],
"user": user.to_dict(),
}
return JSONResponse(
status_code=201, content=body, headers=common_headers(rid)
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
"""Revoke the presented Bearer token.
Does not require a live Redis lookup: knowing the token is enough to revoke it.
Revocation is durable in PostgreSQL so a later Redis restore cannot resurrect it.
"""
rid = request.state.request_id
bearer_challenge = 'Bearer realm="messenger"'
if not authorization or not authorization.strip():
raise ApiError(
401, "unauthorized", "Missing bearer token", www_authenticate=bearer_challenge
)
parts = authorization.strip().split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
raise ApiError(
401, "unauthorized", "Invalid authorization", www_authenticate=bearer_challenge
)
token = parts[1].strip()
try:
revoked = sessions.revoke_session(token)
except SessionStoreError:
raise ApiError(503, "unavailable", "Session store unavailable")
if not revoked:
raise ApiError(
401, "unauthorized", "Invalid or expired session", www_authenticate=bearer_challenge
)
return Response(status_code=204, headers=common_headers(rid))
@app.get("/api/v1/users/me")
async def current_user(
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
user = store.get_user(user_id)
assert user is not None
return JSONResponse(content=user.to_dict(), headers=common_headers(rid))
@app.post("/api/v1/chats")
async def create_chat(
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
data = await read_json_object(request)
allowed = {"title", "member_ids"}
if set(data.keys()) - allowed:
raise ApiError(400, "invalid_request", "Unexpected fields")
if "title" not in data or "member_ids" not in data:
raise ApiError(400, "invalid_request", "Missing required fields")
title = validate_display_text(data["title"], "title", 100)
member_ids = data["member_ids"]
if not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100:
raise ApiError(400, "invalid_request", "Invalid member_ids")
parsed: List[str] = []
seen = set()
for mid in member_ids:
if not isinstance(mid, str):
raise ApiError(400, "invalid_request", "Invalid member_ids")
uid = parse_uuid(mid)
if uid is None:
raise ApiError(400, "invalid_request", "Invalid member_ids")
if uid in seen:
raise ApiError(400, "invalid_request", "Duplicate member_ids")
seen.add(uid)
parsed.append(uid)
if user_id not in seen:
raise ApiError(400, "invalid_request", "Caller must be a member")
try:
chat = store.create_chat(title, parsed)
except KeyError:
raise ApiError(400, "invalid_request", "Unknown member")
return JSONResponse(
status_code=201,
content=chat.to_dict(),
headers=common_headers(rid),
)
@app.get("/api/v1/chats")
async def list_chats(
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
limit: Optional[int] = Query(default=None),
cursor: Optional[str] = Query(default=None),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
lim = parse_limit(limit)
resource = f"chats:{user_id}"
cursor_key = None
if cursor is not None:
if not cursor:
raise ApiError(400, "invalid_request", "Invalid cursor")
try:
cursor_ts, cursor_id = decode_cursor(cursor, resource)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if not store.chat_exists_for_user(user_id, cursor_ts, cursor_id):
raise ApiError(400, "invalid_request", "Invalid cursor")
cursor_key = (cursor_ts, cursor_id)
chats, more = store.list_chats_for_user(user_id, lim, cursor_key)
next_cursor = None
if more and chats:
last = chats[-1]
next_cursor = encode_cursor(resource, last.created_at, last.id)
return JSONResponse(
content={"items": [c.to_dict() for c in chats], "next_cursor": next_cursor},
headers=common_headers(rid),
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(
chat_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
chat = require_chat_member(chat_id, user_id)
return JSONResponse(content=chat.to_dict(), headers=common_headers(rid))
def parse_attachment_ids(raw: Any) -> List[str]:
if not isinstance(raw, list) or len(raw) > 10:
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
parsed: List[str] = []
for item in raw:
aid = parse_uuid(item) if isinstance(item, str) else None
if aid is None or aid in parsed:
raise ApiError(400, "invalid_request", "Invalid attachment_ids")
parsed.append(aid)
return parsed
def parse_idempotency_key(raw: Optional[str]) -> str:
if raw is None or not raw.strip() or not IDEMPOTENCY_KEY_RE.match(raw.strip()):
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
return raw.strip()
@app.post("/api/v1/chats/{chat_id}/messages")
async def create_message(
chat_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
chat = require_chat_member(chat_id, user_id)
key = parse_idempotency_key(idempotency_key)
data = await read_json_object(request)
allowed = {"text", "attachment_ids"}
if set(data.keys()) - allowed:
raise ApiError(400, "invalid_request", "Unexpected fields")
if "text" not in data:
raise ApiError(400, "invalid_request", "Missing text")
attachment_ids = parse_attachment_ids(data.get("attachment_ids", []))
text = data["text"]
# U+0000 is a valid code point of message text (stored as UTF-8 bytes, see 004 migration).
if (
not isinstance(text, str)
or has_lone_surrogate(text)
or codepoint_len(text) > 4000
):
raise ApiError(400, "invalid_request", "Invalid text")
# Empty/whitespace text is allowed only together with at least one attachment.
if not attachment_ids and is_blank(text):
raise ApiError(400, "invalid_request", "Invalid text")
path = f"/api/v1/chats/{chat.id}/messages"
try:
message, _replayed = store.create_message_idempotent(
chat.id, user_id, text, attachment_ids, "POST", path, key
)
except KeyError:
raise ApiError(409, "conflict", "Idempotency-Key reused with different body")
except AttachmentNotFound:
raise ApiError(404, "not_found", "Attachment not found")
except AttachmentFailed:
raise ApiError(409, "conflict", "Attachment is failed")
except ValueError:
raise ApiError(400, "invalid_request", "Invalid text")
return JSONResponse(
status_code=201,
content=message.to_dict(),
headers=common_headers(rid),
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(
chat_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
limit: Optional[int] = Query(default=None),
cursor: Optional[str] = Query(default=None),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
chat = require_chat_member(chat_id, user_id)
lim = parse_limit(limit)
resource = f"messages:{chat.id}"
cursor_key = None
if cursor is not None:
if not cursor:
raise ApiError(400, "invalid_request", "Invalid cursor")
try:
cursor_ts, cursor_id = decode_cursor(cursor, resource)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if not store.message_exists(chat.id, cursor_ts, cursor_id):
raise ApiError(400, "invalid_request", "Invalid cursor")
cursor_key = (cursor_ts, cursor_id)
messages, has_more = store.list_messages(chat.id, lim, cursor_key)
next_cursor = None
if has_more and messages:
last = messages[-1]
next_cursor = encode_cursor(resource, last.created_at, last.id)
return JSONResponse(
content={
"items": [m.to_dict() for m in messages],
"next_cursor": next_cursor,
},
headers=common_headers(rid),
)
def parse_content_length(raw: Optional[str]) -> Optional[int]:
if raw is None:
return None
if not raw.strip().isdigit():
raise ApiError(400, "invalid_request", "Invalid Content-Length")
return int(raw.strip())
def require_attachment(attachment_id_raw: str, user_id: str) -> Attachment:
attachment_id = parse_uuid(attachment_id_raw)
if attachment_id is None:
raise ApiError(400, "invalid_request", "Invalid attachment_id")
att = store.get_attachment_for_user(attachment_id, user_id)
if att is None:
# Unknown and foreign attachments are indistinguishable.
raise ApiError(404, "not_found", "Attachment not found")
return att
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload_attachment(
chat_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
# Input is validated before the chat lookup: a bad filename never reaches S3/DB queries.
try:
filename = attachments.validate_filename(attachments.query_filename(request.scope.get("query_string", b"")))
content_type = attachments.declared_type(request.headers.get("content-type"))
except attachments.UploadRejected as exc:
raise ApiError(exc.status, exc.code, exc.message)
chat = require_chat_member(chat_id, user_id)
length = parse_content_length(request.headers.get("content-length"))
if length is not None and length > config.max_upload_bytes():
# Rejected before taking an upload slot; the body is drained (bounded), not stored.
await attachments.drain_oversized(request.stream(), length)
raise ApiError(413, "payload_too_large", "File exceeds 10 MiB")
assert upload_slots is not None
try:
await asyncio.wait_for(upload_slots.acquire(), timeout=UPLOAD_SLOT_WAIT_S)
except asyncio.TimeoutError:
raise ApiError(503, "unavailable", "Too many concurrent uploads", headers={"Retry-After": "1"})
spooled = None
try:
try:
spooled = await attachments.spool_upload(request.stream(), content_type, length)
except attachments.UploadRejected as exc:
raise ApiError(exc.status, exc.code, exc.message)
except (attachments.ClientGone, ClientDisconnect):
raise ApiError(400, "invalid_request", "Upload interrupted or timed out")
try:
att = await run_in_threadpool(
attachments.persist,
store,
spooled,
chat_id=chat.id,
owner_id=user_id,
filename=filename,
instance=INSTANCE_ID,
)
except IntentLost:
raise ApiError(503, "unavailable", "Upload expired, retry")
except (StoreUnavailable, CommitUncertain):
raise ApiError(503, "unavailable", "Upload temporarily unavailable")
except Exception:
log.exception("upload persist failed request_id=%s", rid)
raise ApiError(503, "unavailable", "Upload temporarily unavailable")
finally:
attachments.discard(spooled)
upload_slots.release()
return JSONResponse(
status_code=202,
content=att.to_dict(),
headers=common_headers(rid, {"Location": f"/api/v1/attachments/{att.id}"}),
)
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(
attachment_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
att = require_attachment(attachment_id, user_id)
return JSONResponse(content=att.to_dict(), headers=common_headers(rid))
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(
attachment_id: str,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
variant: Optional[str] = Query(default=None),
):
rid = request.state.request_id
user_id, _token = require_bearer(authorization)
att = require_attachment(attachment_id, user_id)
name = "original" if variant in (None, "original") else variant
if name != "original" and name not in VARIANT_NAMES:
raise ApiError(400, "invalid_request", "Invalid variant")
if att.status in ("queued", "processing"):
raise ApiError(409, "conflict", "Attachment is not ready yet")
if att.status == "failed":
raise ApiError(409, "conflict", "Attachment processing failed")
if name == "original":
if not att.object_key:
raise ApiError(503, "unavailable", "Attachment content unavailable")
object_key = att.object_key
media_type = att.content_type
size_bytes = att.size_bytes
digest = att.sha256
filename = att.filename
else:
matches = [v for v in att.variants if v.name == name]
if not matches:
raise ApiError(404, "not_found", "Variant not found")
chosen = matches[0]
object_key = chosen.object_key
media_type = chosen.content_type
size_bytes = chosen.size_bytes
digest = chosen.sha256
filename = f"{name}-{att.filename}"
try:
length, chunks, body = await run_in_threadpool(s3.open_object, object_key)
except s3.S3ObjectMissing:
log.error("object missing in S3 attachment_id=%s variant=%s request_id=%s", att.id, name, rid)
raise ApiError(503, "unavailable", "Attachment content unavailable")
if length != size_bytes:
body.close()
log.error("size mismatch attachment_id=%s variant=%s request_id=%s", att.id, name, rid)
raise ApiError(503, "unavailable", "Attachment content unavailable")
def stream():
try:
yield from chunks
finally:
body.close()
headers = common_headers(
rid,
{
"Content-Length": str(size_bytes),
"ETag": f'"{digest}"',
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
"Content-Disposition": "inline; filename*=UTF-8''" + urllib.parse.quote(filename, safe=""),
},
)
return StreamingResponse(stream(), media_type=media_type, headers=headers)
@app.post("/internal/v1/dead-letters/{dead_letter_id}/replay")
async def replay_dead_letter(
dead_letter_id: int,
request: Request,
authorization: Optional[str] = Header(default=None, alias="Authorization"),
):
"""Operator replay of a failed job. Requires a valid bearer session (lab admin demo)."""
rid = request.state.request_id
require_bearer(authorization)
try:
result = await run_in_threadpool(store.replay_dead_letter, dead_letter_id)
except StoreUnavailable:
raise ApiError(503, "unavailable", "Database unavailable")
if result is None:
raise ApiError(404, "not_found", "Dead letter not found")
if outbox_publisher is not None:
await run_in_threadpool(outbox_publisher.flush_once)
return JSONResponse(
content={"result": result, "dead_letter_id": dead_letter_id},
headers=common_headers(rid),
)
def create_app() -> FastAPI:
return app
+342
View File
@@ -0,0 +1,342 @@
"""Attachment upload pipeline: bounded streaming intake, then intent → S3 PUT → metadata commit."""
from __future__ import annotations
import asyncio
import hashlib
import logging
import os
import shutil
import tempfile
import time
import unicodedata
import urllib.parse
import uuid
from dataclasses import dataclass
from typing import AsyncIterator, Callable, Optional
from messenger import config, images, s3
from messenger.store import Attachment, CommitUncertain, Store, StoreUnavailable, utc_now
log = logging.getLogger("messenger.attachments")
ALLOWED_TYPES = ("image/png", "image/jpeg")
# Rejected bodies are drained (never stored) up to this many bytes so that the
# client can read the error response instead of a connection reset.
DRAIN_FACTOR = 2
DECODE_TIMEOUT_S = 15
DECODE_MEMORY_BYTES = 384 * 1024 * 1024
_decode_slots = asyncio.Semaphore(1)
class UploadRejected(Exception):
def __init__(self, status: int, code: str, message: str):
super().__init__(message)
self.status = status
self.code = code
self.message = message
class ClientGone(Exception):
"""Client disconnected or stalled mid-upload; nothing was written to S3/DB."""
class SimulatedCrash(BaseException):
"""Fault-injection only: bypasses compensation like a killed process would."""
def query_filename(query_string: bytes) -> Optional[str]:
"""The single `filename` query value, percent-decoded strictly.
Starlette's query_params replace invalid UTF-8 with U+FFFD; such a name must be
rejected, not silently rewritten.
"""
try:
pairs = urllib.parse.parse_qsl(
query_string.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict"
)
except (UnicodeDecodeError, ValueError):
raise UploadRejected(400, "invalid_request", "Invalid filename") from None
values = [v for k, v in pairs if k == "filename"]
if len(values) > 1:
raise UploadRejected(400, "invalid_request", "Invalid filename")
return values[0] if values else None
def validate_filename(value: Optional[str]) -> str:
if value is None or not isinstance(value, str):
raise UploadRejected(400, "invalid_request", "filename is required")
if not 1 <= len(value) <= 255 or value.strip() == "" or value in (".", ".."):
raise UploadRejected(400, "invalid_request", "Invalid filename")
for ch in value:
if ch in "/\\" or 0xD800 <= ord(ch) <= 0xDFFF:
raise UploadRejected(400, "invalid_request", "Invalid filename")
# Cc: NUL/control characters; Cf: bidi overrides (U+202E) and other invisible format chars.
if unicodedata.category(ch) in ("Cc", "Cf"):
raise UploadRejected(400, "invalid_request", "Invalid filename")
return value
def declared_type(header: Optional[str]) -> str:
media = (header or "").split(";")[0].strip().lower()
if media not in ALLOWED_TYPES:
raise UploadRejected(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted")
return media
def new_object_key() -> str:
# Random, server-generated; carries no user input (filename/chat title).
return s3.ORIGINAL_PREFIX + uuid.uuid4().hex
def new_staging_key() -> str:
return s3.STAGING_PREFIX + uuid.uuid4().hex
@dataclass
class Spooled:
path: str
size: int
sha256: str
content_type: str
width: int
height: int
def prepare_spool_dir() -> None:
"""Remove leftovers of requests interrupted by a crash; buffers never outlive a request."""
root = config.spool_dir()
if root.exists():
shutil.rmtree(root, ignore_errors=True)
root.mkdir(parents=True, exist_ok=True)
async def _next_chunk(it: AsyncIterator[bytes], timeout: float) -> Optional[bytes]:
try:
return await asyncio.wait_for(it.__anext__(), timeout=timeout)
except StopAsyncIteration:
return None
async def _drain(it: AsyncIterator[bytes], already: int, cap: int, deadline: float) -> None:
"""Discard the rest of a rejected body (bounded) so the error reaches the client."""
received = already
try:
while received <= cap and time.monotonic() < deadline:
chunk = await _next_chunk(it, min(2.0, max(0.1, deadline - time.monotonic())))
if chunk is None:
return
received += len(chunk)
except Exception:
return
async def drain_oversized(stream: AsyncIterator[bytes], content_length: int) -> None:
cap = config.max_upload_bytes() * DRAIN_FACTOR
if content_length <= cap:
await _drain(stream.__aiter__(), 0, cap, time.monotonic() + 30)
async def spool_upload(
stream: AsyncIterator[bytes],
content_type: str,
content_length: Optional[int],
) -> Spooled:
"""Read the body chunk by chunk into a private temp file while validating it.
Memory per upload is bounded by the transport chunk size (≈64 KiB) plus small
parser state; size and pixel limits are enforced before the body is fully read.
"""
max_bytes = config.max_upload_bytes()
idle = float(config.upload_idle_timeout_seconds())
deadline = time.monotonic() + config.upload_deadline_seconds()
it = stream.__aiter__()
drain_cap = max_bytes * DRAIN_FACTOR
if content_length is not None and content_length > max_bytes:
await drain_oversized(it, content_length)
raise UploadRejected(413, "payload_too_large", "File exceeds 10 MiB")
validator = images.validator_for(content_type, config.max_image_pixels())
digest = hashlib.sha256()
size = 0
prefix = b""
fd, path = tempfile.mkstemp(prefix="up-", dir=str(config.spool_dir()))
ok = False
try:
with os.fdopen(fd, "wb") as out:
while True:
if time.monotonic() > deadline:
raise ClientGone("upload deadline exceeded")
try:
chunk = await _next_chunk(it, idle)
except asyncio.TimeoutError:
raise ClientGone("client stalled") from None
if chunk is None:
break
if not chunk:
continue
size += len(chunk)
if size > max_bytes:
await _drain(it, size, drain_cap, min(deadline, time.monotonic() + 30))
raise UploadRejected(413, "payload_too_large", "File exceeds 10 MiB")
try:
if len(prefix) < 8:
# MIME is decided by content: the declared type must match the magic bytes.
prefix = (prefix + chunk)[:8]
sniffed = images.sniff(prefix) if len(prefix) >= 3 else None
if (len(prefix) >= 8 or sniffed == "image/jpeg") and sniffed != content_type:
raise images.ImageRejected(415, "unsupported_media_type", "Content does not match declared image type")
validator.feed(chunk)
except images.ImageRejected:
await _drain(it, size, drain_cap, min(deadline, time.monotonic() + 30))
raise
digest.update(chunk)
out.write(chunk)
if size == 0:
raise UploadRejected(400, "invalid_request", "Empty body")
if len(prefix) < 8 and images.sniff(prefix) != content_type:
raise UploadRejected(415, "unsupported_media_type", "Content does not match declared image type")
try:
width, height = validator.finish()
except images.ImageRejected as exc:
raise UploadRejected(exc.status, exc.code, exc.message) from None
if content_type == "image/jpeg":
await verify_jpeg(path)
ok = True
return Spooled(path, size, digest.hexdigest(), content_type, width, height)
except images.ImageRejected as exc:
raise UploadRejected(exc.status, exc.code, exc.message) from None
finally:
if not ok:
_unlink(path)
async def verify_jpeg(path: str) -> None:
"""Fully entropy-decode the JPEG with libjpeg-turbo; any corruption warning is fatal.
`-scale 1/8` keeps the output tiny, but every Huffman-coded block is still decoded.
Progressive JPEGs need a whole-image coefficient buffer (≈2 bytes/pixel/component,
≈310 MiB for a 40 Mpx CMYK image), so decodes run one at a time per replica in a
separate process with an address-space cap and a wall-clock timeout.
"""
async with _decode_slots:
proc = await asyncio.create_subprocess_exec(
"prlimit", f"--as={DECODE_MEMORY_BYTES}", "--",
"djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
try:
_, err = await asyncio.wait_for(proc.communicate(), DECODE_TIMEOUT_S)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
log.warning("jpeg verification timed out")
raise UploadRejected(503, "unavailable", "Image verification timed out, retry later") from None
if proc.returncode == 0:
return
message = err.decode("utf-8", "replace")
if proc.returncode < 0 or "Insufficient memory" in message:
log.warning("jpeg verification aborted rc=%s", proc.returncode)
raise UploadRejected(503, "unavailable", "Image verification failed, retry later")
raise UploadRejected(415, "unsupported_media_type", "Corrupted JPEG image data")
def _unlink(path: str) -> None:
try:
os.unlink(path)
except FileNotFoundError:
pass
except OSError as exc:
log.warning("spool cleanup failed: %s", type(exc).__name__)
def persist(
store: Store,
spooled: Spooled,
*,
chat_id: str,
owner_id: str,
filename: str,
instance: str,
fault: Optional[Callable[[str, str], None]] = None,
) -> Attachment:
"""Accept upload into durable staging + queued job (lab5). Blocking.
Returns Attachment with status=queued. Broker publish is via transactional outbox.
"""
from messenger.store import format_ts # local import avoids cycles at module load
attachment_id = str(uuid.uuid4())
job_id = str(uuid.uuid4())
staging_key = new_staging_key()
now = utc_now()
att = Attachment(
id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
content_type=spooled.content_type,
size_bytes=spooled.size,
sha256=spooled.sha256,
width=spooled.width,
height=spooled.height,
object_key=None,
status="queued",
created_at=now,
updated_at=now,
staging_key=staging_key,
error_code=None,
pipeline_version=1,
variants=[],
)
job_payload = {
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": attachment_id,
"pipeline_version": 1,
"attempt": 1,
"created_at": format_ts(now),
}
# 1. Intent first: if PostgreSQL is down nothing reaches S3 (fast 503).
store.record_intent(staging_key, attachment_id, chat_id, instance)
try:
if fault:
fault("before_put", staging_key)
with open(spooled.path, "rb") as body:
s3.put_object(staging_key, body, spooled.size, spooled.content_type, spooled.sha256)
if fault:
fault("after_put", staging_key)
# 2. Queued metadata + job + outbox in one COMMIT (durable handoff before 202).
committed = store.commit_queued_attachment(
att, job_id=job_id, staging_key=staging_key, job_payload=job_payload
)
if fault:
fault("after_commit", staging_key)
return committed
except CommitUncertain:
log.warning("commit outcome unknown, staging kept key_suffix=%s", staging_key[-8:])
raise
except Exception as exc:
log.warning("upload failed, compensating key_suffix=%s err=%s", staging_key[-8:], type(exc).__name__)
_compensate(store, staging_key)
raise
def _compensate(store: Store, key: str) -> None:
try:
s3.delete_object(key)
except s3.S3Unavailable:
return
try:
store.drop_intent(key)
except StoreUnavailable:
pass
def discard(spooled: Optional[Spooled]) -> None:
if spooled is not None:
_unlink(spooled.path)
+141
View File
@@ -0,0 +1,141 @@
"""RabbitMQ helpers: durable queue, publisher confirms, consumer ack."""
from __future__ import annotations
import json
import logging
import time
from typing import Any, Callable, Optional
import pika
from pika.exceptions import AMQPConnectionError, AMQPError, ChannelClosedByBroker, StreamLostError
from messenger import config
log = logging.getLogger("messenger.broker")
EXCHANGE = ""
QUEUE = "attachment.process"
DLQ = "attachment.process.dlq"
def _params(url: Optional[str] = None) -> pika.URLParameters:
params = pika.URLParameters(url or config.broker_url())
params.heartbeat = 30
params.blocked_connection_timeout = 30
params.connection_attempts = 3
params.retry_delay = 1
return params
def connect(url: Optional[str] = None) -> pika.BlockingConnection:
return pika.BlockingConnection(_params(url))
def declare_topology(channel: pika.channel.Channel) -> None:
channel.queue_declare(queue=DLQ, durable=True)
channel.queue_declare(
queue=QUEUE,
durable=True,
arguments={
"x-dead-letter-exchange": EXCHANGE,
"x-dead-letter-routing-key": DLQ,
},
)
def publish_job(payload: dict[str, Any], *, connection: Optional[pika.BlockingConnection] = None) -> None:
"""Publish one job with delivery_mode=2 and wait for publisher confirm."""
own = connection is None
conn = connection or connect()
try:
channel = conn.channel()
channel.confirm_delivery()
declare_topology(channel)
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
channel.basic_publish(
exchange=EXCHANGE,
routing_key=QUEUE,
body=body,
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
message_id=str(payload.get("job_id", "")),
type=str(payload.get("type", "attachment.process.v1")),
),
mandatory=True,
)
finally:
if own:
try:
conn.close()
except Exception:
pass
def ping(url: Optional[str] = None) -> bool:
try:
conn = connect(url)
try:
ch = conn.channel()
declare_topology(ch)
return True
finally:
conn.close()
except (AMQPConnectionError, AMQPError, OSError) as exc:
log.warning("broker ping failed: %s", type(exc).__name__)
return False
def consume_forever(
handler: Callable[[dict[str, Any], Callable[[], None], Callable[[bool], None]], None],
*,
prefetch: int,
stop_flag: Callable[[], bool],
) -> None:
"""Block consuming QUEUE. handler(payload, ack, nack(requeue))."""
while not stop_flag():
try:
conn = connect()
channel = conn.channel()
declare_topology(channel)
channel.basic_qos(prefetch_count=max(1, prefetch))
for method, _props, body in channel.consume(QUEUE, inactivity_timeout=1.0):
if stop_flag():
break
if method is None:
continue
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
log.error("dropping malformed broker message")
channel.basic_nack(method.delivery_tag, requeue=False)
continue
acked = False
def ack() -> None:
nonlocal acked
channel.basic_ack(method.delivery_tag)
acked = True
def nack(requeue: bool) -> None:
nonlocal acked
channel.basic_nack(method.delivery_tag, requeue=requeue)
acked = True
try:
handler(payload, ack, nack)
except Exception:
log.exception("consumer handler crashed")
if not acked:
channel.basic_nack(method.delivery_tag, requeue=True)
try:
channel.cancel()
except Exception:
pass
conn.close()
except (AMQPConnectionError, StreamLostError, ChannelClosedByBroker, OSError) as exc:
log.warning("broker consumer reconnecting after %s", type(exc).__name__)
time.sleep(1.0)
+235
View File
@@ -0,0 +1,235 @@
"""Runtime configuration from environment / secret files."""
from __future__ import annotations
import os
import socket
from pathlib import Path
def _read_secret(env_key: str, file_key: str, default: str | None = None) -> str:
file_path = os.environ.get(file_key, "").strip()
if file_path:
path = Path(file_path)
if path.is_file():
return path.read_text(encoding="utf-8").strip()
value = os.environ.get(env_key, "").strip()
if value:
return value
if default is not None:
return default
raise RuntimeError(f"Missing secret: set {env_key} or {file_key}")
def _int_env(key: str, default: int, minimum: int = 1) -> int:
raw = os.environ.get(key, str(default)).strip()
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError(f"{key} must be an integer") from exc
if value < minimum:
raise RuntimeError(f"{key} must be >= {minimum}")
return value
def database_url() -> str:
return _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
def cache_url() -> str:
return _read_secret("CACHE_URL", "CACHE_URL_FILE")
def session_ttl_seconds() -> int:
return _int_env("SESSION_TTL_SECONDS", 3600)
def idempotency_ttl_seconds() -> int:
return _int_env("IDEMPOTENCY_TTL_SECONDS", 86400)
def instance_id() -> str:
"""Stable for process lifetime; used as X-Instance-Id (maps to container hostname)."""
override = os.environ.get("INSTANCE_ID", "").strip()
if override:
return override[:128]
return socket.gethostname()[:128]
def app_host() -> str:
return os.environ.get("APP_HOST", "0.0.0.0")
def app_port() -> int:
return int(os.environ.get("APP_PORT", "8080"))
def migrations_dir() -> Path:
override = os.environ.get("MIGRATIONS_DIR", "").strip()
if override:
return Path(override)
here = Path(__file__).resolve()
candidates = [
Path("/app/migrations"),
here.parents[2] / "migrations",
Path.cwd() / "migrations",
]
for path in candidates:
if path.is_dir():
return path
return candidates[0]
# --- S3 / attachments (lab4) -------------------------------------------------
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
def s3_endpoint() -> str:
return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
def s3_region() -> str:
return os.environ.get("S3_REGION", "us-east-1").strip()
def s3_bucket() -> str:
return os.environ.get("S3_BUCKET", "messenger-attachments").strip()
def s3_access_key_id() -> str:
return _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
def s3_secret_access_key() -> str:
return _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
def s3_admin_access_key_id() -> str:
return _read_secret("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE")
def s3_admin_secret_access_key() -> str:
return _read_secret("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE")
def max_upload_bytes() -> int:
# The contract caps uploads at 10 MiB; the environment may only lower it.
return min(_int_env("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES), CONTRACT_MAX_UPLOAD_BYTES)
def max_image_pixels() -> int:
return min(_int_env("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS), CONTRACT_MAX_IMAGE_PIXELS)
def max_concurrent_uploads() -> int:
return _int_env("MAX_CONCURRENT_UPLOADS", 8)
def upload_idle_timeout_seconds() -> int:
return _int_env("UPLOAD_IDLE_TIMEOUT_SECONDS", 10)
def upload_deadline_seconds() -> int:
return _int_env("UPLOAD_DEADLINE_SECONDS", 120)
def upload_grace_seconds() -> int:
value = _int_env("UPLOAD_GRACE_SECONDS", 900)
if value < 2 * upload_deadline_seconds():
raise RuntimeError("UPLOAD_GRACE_SECONDS must be >= 2 * UPLOAD_DEADLINE_SECONDS")
return value
def reconcile_interval_seconds() -> int:
return _int_env("RECONCILE_INTERVAL_SECONDS", 300, minimum=0)
def shutdown_drain_seconds() -> int:
# After SIGTERM: readiness 503 but normal requests served, until HAProxy
# (inter 1s, fall 2) has taken the replica out of rotation.
return _int_env("SHUTDOWN_DRAIN_SECONDS", 3, minimum=0)
def shutdown_timeout_seconds() -> int:
# Upper bound for in-flight requests once the listener is closed.
return _int_env("SHUTDOWN_TIMEOUT_SECONDS", 8)
def spool_dir() -> Path:
return Path(os.environ.get("UPLOAD_SPOOL_DIR", "/tmp/messenger-spool"))
# --- Broker / worker (lab5) ---------------------------------------------------
def broker_url() -> str:
return _read_secret("BROKER_URL", "BROKER_URL_FILE")
def worker_concurrency() -> int:
return _int_env("WORKER_CONCURRENCY", 2)
def job_max_attempts() -> int:
return _int_env("JOB_MAX_ATTEMPTS", 5)
def job_timeout_seconds() -> int:
return _int_env("JOB_TIMEOUT_SECONDS", 60)
def job_backoff_base_seconds() -> float:
raw = os.environ.get("JOB_BACKOFF_BASE_SECONDS", "1").strip()
try:
value = float(raw)
except ValueError as exc:
raise RuntimeError("JOB_BACKOFF_BASE_SECONDS must be a number") from exc
if value <= 0:
raise RuntimeError("JOB_BACKOFF_BASE_SECONDS must be > 0")
return value
def job_backoff_max_seconds() -> float:
raw = os.environ.get("JOB_BACKOFF_MAX_SECONDS", "60").strip()
try:
value = float(raw)
except ValueError as exc:
raise RuntimeError("JOB_BACKOFF_MAX_SECONDS must be a number") from exc
if value < job_backoff_base_seconds():
raise RuntimeError("JOB_BACKOFF_MAX_SECONDS must be >= JOB_BACKOFF_BASE_SECONDS")
return value
def outbox_poll_seconds() -> float:
raw = os.environ.get("OUTBOX_POLL_SECONDS", "0.5").strip()
try:
value = float(raw)
except ValueError as exc:
raise RuntimeError("OUTBOX_POLL_SECONDS must be a number") from exc
if value <= 0:
raise RuntimeError("OUTBOX_POLL_SECONDS must be > 0")
return value
def staging_retention_seconds() -> int:
return _int_env("STAGING_RETENTION_SECONDS", 86400)
def worker_process_timeout_seconds() -> int:
# Wall-clock budget for decode+encode inside the worker for one job.
return _int_env("WORKER_PROCESS_TIMEOUT_SECONDS", 45)
def webp_quality() -> int:
return _int_env("WEBP_QUALITY", 80, minimum=1)
def crash_after_side_effect() -> bool:
"""Fault injection: worker exits after durable side effects, before ack."""
return os.environ.get("WORKER_CRASH_AFTER_SIDE_EFFECT", "").strip() in ("1", "true", "yes")
def crash_before_outbox_publish() -> bool:
"""Fault injection for API/outbox tests (process-local)."""
return os.environ.get("CRASH_BEFORE_OUTBOX_PUBLISH", "").strip() in ("1", "true", "yes")
+218
View File
@@ -0,0 +1,218 @@
"""PostgreSQL connection pool and versioned migrations."""
from __future__ import annotations
import itertools
import logging
import os
import re
import socket
import threading
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Dict, Iterator, List, Optional, Tuple
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from messenger import config
log = logging.getLogger("messenger.db")
# statement_timeout is enforced by the server and does nothing when the server is
# frozen (docker pause, stuck host): the client would wait forever. The watchdog
# bounds every borrowed connection on the client side instead.
QUERY_DEADLINE_S = 6.0
POOL_WAIT_S = 3.0
BREAKER_OPEN_S = 3.0
_pool: Optional[ConnectionPool] = None
_guarded: Optional["GuardedPool"] = None
class _Watchdog:
"""One thread; shuts down the socket of a connection that outlives its deadline."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._armed: Dict[int, Tuple[float, psycopg.Connection]] = {}
self._fired: set = set()
self._ids = itertools.count()
self._thread: Optional[threading.Thread] = None
def arm(self, conn: psycopg.Connection, deadline_s: float) -> int:
token = next(self._ids)
with self._lock:
self._armed[token] = (time.monotonic() + deadline_s, conn)
if self._thread is None:
self._thread = threading.Thread(target=self._run, name="db-watchdog", daemon=True)
self._thread.start()
return token
def disarm(self, token: int) -> bool:
with self._lock:
self._armed.pop(token, None)
if token in self._fired:
self._fired.discard(token)
return True
return False
def _run(self) -> None:
while True:
time.sleep(0.25)
now = time.monotonic()
# Kill under the lock: the borrower must disarm (same lock) before the
# connection goes back to the pool, so its fd cannot be closed and reused
# by the event loop while we touch it.
with self._lock:
overdue = [(t, c) for t, (d, c) in self._armed.items() if d <= now]
for token, conn in overdue:
self._armed.pop(token, None)
self._fired.add(token)
_kill_socket(conn)
def _kill_socket(conn: psycopg.Connection) -> None:
# The blocked libpq call sees EOF, psycopg raises OperationalError and marks the
# connection broken; the pool then discards it instead of reusing a stuck one.
try:
sock = socket.socket(fileno=os.dup(conn.pgconn.socket))
except Exception:
return
# The socket object owns the dup'ed fd and closes it exactly once.
with sock:
try:
sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
log.warning("postgres call exceeded its deadline; connection aborted")
_watchdog = _Watchdog()
class GuardedPool:
"""Pool facade: bounded wait for a connection, bounded time per borrow, breaker."""
def __init__(self, pool: ConnectionPool):
self.raw = pool
self._open_until = 0.0
def _trip(self) -> None:
self._open_until = time.monotonic() + BREAKER_OPEN_S
@contextmanager
def connection(self, deadline: Optional[float] = QUERY_DEADLINE_S) -> Iterator[psycopg.Connection]:
if time.monotonic() < self._open_until:
# Fail fast while PostgreSQL recently hung or vanished; a later call probes again.
raise psycopg.OperationalError("database circuit open")
try:
with self.raw.connection(timeout=POOL_WAIT_S) as conn:
token = _watchdog.arm(conn, deadline) if deadline else None
try:
yield conn
except psycopg.Error:
if conn.broken:
self._trip()
raise
finally:
if token is not None and _watchdog.disarm(token):
self._trip()
except PoolTimeout:
self._trip()
raise
def get_pool() -> GuardedPool:
if _guarded is None:
raise RuntimeError("Database pool is not initialized")
return _guarded
def init_pool() -> GuardedPool:
global _pool, _guarded
if _guarded is not None:
return _guarded
url = config.database_url()
_pool = ConnectionPool(
conninfo=url,
min_size=1,
max_size=10,
kwargs={
"row_factory": dict_row,
"connect_timeout": 3,
"options": "-c statement_timeout=5000",
},
open=True,
timeout=POOL_WAIT_S,
)
_guarded = GuardedPool(_pool)
return _guarded
def close_pool() -> None:
global _pool, _guarded
if _pool is not None:
# Bounded: the whole shutdown must fit into the container's stop_grace_period.
_pool.close(timeout=1.0)
_pool = None
_guarded = None
def _split_sql(script: str) -> List[str]:
"""Split SQL file into statements (no procedure bodies in our migrations)."""
# Drop line comments.
cleaned = re.sub(r"--[^\n]*", "", script)
parts = []
for chunk in cleaned.split(";"):
stmt = chunk.strip()
if stmt:
parts.append(stmt)
return parts
def apply_migrations(migrations_path: Optional[Path] = None) -> None:
path = migrations_path or config.migrations_dir()
sql_files = sorted(path.glob("*.sql"))
if not sql_files:
raise RuntimeError(f"No migrations found in {path}")
pool = get_pool()
with pool.connection(deadline=None) as conn:
with conn.transaction():
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
for sql_file in sql_files:
version = sql_file.name
row = conn.execute(
"SELECT 1 AS ok FROM schema_migrations WHERE version = %s",
(version,),
).fetchone()
if row:
log.info("migration already applied: %s", version)
continue
script = sql_file.read_text(encoding="utf-8")
for stmt in _split_sql(script):
conn.execute(stmt)
conn.execute(
"INSERT INTO schema_migrations (version) VALUES (%s)",
(version,),
)
log.info("applied migration: %s", version)
def ping() -> bool:
try:
with get_pool().connection(deadline=1.5) as conn:
conn.execute("SELECT 1")
return True
except Exception as exc:
log.warning("postgres ping failed: %s", type(exc).__name__)
return False
+411
View File
@@ -0,0 +1,411 @@
"""Incremental PNG/JPEG validation with bounded memory.
The validators consume the upload chunk by chunk. Dimensions are taken from the
header (PNG IHDR, JPEG SOFn) and the pixel limit is enforced as soon as they are
known, before the rest of the body is read.
PNG is decoded here in full except for the arithmetic unfiltering step (which
cannot fail): chunk CRCs, PLTE rules, the whole zlib stream inflated in bounded
slices, its exact size for the declared geometry (incl. Adam7) and the filter
type byte of every scanline. JPEG is only checked structurally here; the entropy
coded data is verified afterwards by a real decoder (see attachments.verify_jpeg).
"""
from __future__ import annotations
import struct
import zlib
from typing import Optional, Tuple
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SOI = b"\xff\xd8\xff"
_INFLATE_SLICE = 64 * 1024
_MAX_PNG_CHUNK = 0x7FFFFFFF
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
_PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
# Baseline, extended sequential and progressive Huffman JPEG: what browsers decode.
_JPEG_SOF_SUPPORTED = {0xC0, 0xC1, 0xC2}
_JPEG_SOF_ALL = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
class ImageRejected(Exception):
"""Upload content is not an acceptable image; maps to an HTTP status."""
def __init__(self, status: int, code: str, message: str):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def _unsupported(message: str) -> ImageRejected:
return ImageRejected(415, "unsupported_media_type", message)
def sniff(prefix: bytes) -> Optional[str]:
if prefix.startswith(PNG_SIGNATURE):
return "image/png"
if prefix.startswith(JPEG_SOI):
return "image/jpeg"
return None
class _Validator:
content_type = ""
def __init__(self, max_pixels: int):
self.max_pixels = max_pixels
self.width: Optional[int] = None
self.height: Optional[int] = None
def _set_dimensions(self, width: int, height: int) -> None:
if width < 1 or height < 1:
raise _unsupported("Invalid image dimensions")
if width * height > self.max_pixels:
raise ImageRejected(413, "payload_too_large", "Image has too many pixels")
self.width, self.height = width, height
def feed(self, data: bytes) -> None:
raise NotImplementedError
def finish(self) -> Tuple[int, int]:
raise NotImplementedError
class PngValidator(_Validator):
content_type = "image/png"
def __init__(self, max_pixels: int):
super().__init__(max_pixels)
self._buf = bytearray()
self._state = "signature"
self._chunk_type = b""
self._chunk_left = 0
self._chunk_small = bytearray()
self._crc = 0
self._seen_plte = False
self._seen_idat = False
self._idat_done = False
self._color_type = 0
self._depth = 0
self._expected_raw = 0
# (start, end, row_len) of every pass in the inflated stream; a scanline
# starts with its filter type byte (0..4).
self._rows: list = []
self._raw = 0
self._inflater = zlib.decompressobj()
def _layout(self, width: int, height: int, depth: int, color: int, interlace: int) -> None:
bpp = _PNG_CHANNELS[color] * depth
def row_bytes(w: int) -> int:
return 1 + (w * bpp + 7) // 8
passes = [(0, 0, 1, 1)] if interlace == 0 else _ADAM7
offset = 0
for x0, y0, dx, dy in passes:
pw = (width - x0 + dx - 1) // dx if width > x0 else 0
ph = (height - y0 + dy - 1) // dy if height > y0 else 0
if pw and ph:
size = ph * row_bytes(pw)
self._rows.append((offset, offset + size, row_bytes(pw)))
offset += size
self._expected_raw = offset
def _check_filters(self, out: bytes, base: int) -> None:
end = base + len(out)
for start, stop, row_len in self._rows:
lo, hi = max(start, base), min(stop, end)
if lo >= hi:
continue
first = start + -(-(lo - start) // row_len) * row_len
if first < hi and max(out[first - base:hi - base:row_len]) > 4:
raise _unsupported("Invalid PNG scanline filter")
def _inflate(self, data: bytes) -> None:
if self._inflater.eof:
if data:
raise _unsupported("Data after end of PNG image stream")
return
try:
out = self._inflater.decompress(data, _INFLATE_SLICE)
self._check_filters(out, self._raw)
self._raw += len(out)
while self._inflater.unconsumed_tail and self._raw <= self._expected_raw:
out = self._inflater.decompress(self._inflater.unconsumed_tail, _INFLATE_SLICE)
self._check_filters(out, self._raw)
self._raw += len(out)
except zlib.error:
raise _unsupported("Corrupted PNG image data") from None
if self._raw > self._expected_raw:
raise _unsupported("PNG image data larger than declared dimensions")
if self._inflater.eof and self._inflater.unused_data:
raise _unsupported("Data after end of PNG image stream")
def _begin_chunk(self, length: int, ctype: bytes) -> None:
if length > _MAX_PNG_CHUNK or not ctype.isalpha():
raise _unsupported("Corrupted PNG chunk")
if self.width is None and ctype != b"IHDR":
raise _unsupported("PNG must start with IHDR")
# Uppercase first letter = critical: a decoder must not skip one it does not know.
if ctype[:1].isupper() and ctype not in (b"IHDR", b"PLTE", b"IDAT", b"IEND"):
raise _unsupported("Unknown critical PNG chunk")
if ctype == b"IHDR" and (self.width is not None or length != 13):
raise _unsupported("Invalid PNG IHDR")
if ctype == b"IDAT":
if self._idat_done:
raise _unsupported("Non-consecutive PNG IDAT chunks")
if self._color_type == 3 and not self._seen_plte:
raise _unsupported("Palette PNG without PLTE")
self._seen_idat = True
elif self._seen_idat:
self._idat_done = True
if ctype == b"PLTE":
entries, rest = divmod(length, 3)
if (
self._seen_plte
or self._seen_idat
or self._color_type in (0, 4)
or rest
or not 1 <= entries <= 256
or (self._color_type == 3 and entries > 1 << self._depth)
):
raise _unsupported("Invalid PNG palette")
self._seen_plte = True
if ctype == b"IEND" and length != 0:
raise _unsupported("Invalid PNG IEND")
self._chunk_type = ctype
self._chunk_left = length
self._chunk_small = bytearray()
self._crc = zlib.crc32(ctype)
def _end_chunk(self) -> None:
ctype = self._chunk_type
if ctype == b"IHDR":
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", bytes(self._chunk_small))
if color not in _PNG_DEPTHS or depth not in _PNG_DEPTHS[color] or comp != 0 or filt != 0 or interlace not in (0, 1):
raise _unsupported("Unsupported PNG header")
if width > _MAX_PNG_CHUNK or height > _MAX_PNG_CHUNK:
raise _unsupported("Invalid image dimensions")
self._set_dimensions(width, height)
self._color_type = color
self._depth = depth
self._layout(width, height, depth, color, interlace)
elif ctype == b"IEND":
self._state = "end"
def feed(self, data: bytes) -> None:
view = memoryview(data)
pos = 0
while pos < len(view):
if self._state == "end":
raise _unsupported("Data after PNG IEND")
if self._state == "signature":
need = 8 - len(self._buf)
self._buf += view[pos:pos + need]
pos += min(need, len(view) - pos)
if len(self._buf) == 8:
if bytes(self._buf) != PNG_SIGNATURE:
raise _unsupported("Not a PNG image")
self._buf.clear()
self._state = "header"
elif self._state == "header":
need = 8 - len(self._buf)
self._buf += view[pos:pos + need]
pos += min(need, len(view) - pos)
if len(self._buf) == 8:
length, ctype = struct.unpack(">I4s", bytes(self._buf))
self._buf.clear()
self._begin_chunk(length, ctype)
self._state = "data" if length else "crc"
elif self._state == "data":
take = min(self._chunk_left, len(view) - pos)
piece = view[pos:pos + take]
pos += take
self._chunk_left -= take
self._crc = zlib.crc32(piece, self._crc)
if self._chunk_type == b"IDAT":
self._inflate(bytes(piece))
elif self._chunk_type == b"IHDR":
self._chunk_small += piece
if self._chunk_left == 0:
self._state = "crc"
elif self._state == "crc":
need = 4 - len(self._buf)
self._buf += view[pos:pos + need]
pos += min(need, len(view) - pos)
if len(self._buf) == 4:
(crc,) = struct.unpack(">I", bytes(self._buf))
self._buf.clear()
if crc != (self._crc & 0xFFFFFFFF):
raise _unsupported("PNG chunk checksum mismatch")
self._state = "header"
self._end_chunk()
def finish(self) -> Tuple[int, int]:
if self._state != "end" or self.width is None or self.height is None:
raise _unsupported("Truncated PNG image")
if not self._seen_idat or not self._inflater.eof or self._raw != self._expected_raw:
raise _unsupported("Incomplete PNG image data")
return self.width, self.height
class JpegValidator(_Validator):
content_type = "image/jpeg"
def __init__(self, max_pixels: int):
super().__init__(max_pixels)
self._buf = bytearray()
self._state = "soi"
self._marker = 0
self._seg_left = 0
self._seg_small = bytearray()
self._pending_ff = False
self._seen_sof = False
self._seen_sos = False
self._done = False
self._sof_ids: set[int] = set()
def _begin_segment(self, marker: int, length: int) -> None:
if length < 2:
raise _unsupported("Corrupted JPEG segment")
if marker in _JPEG_SOF_ALL:
if marker not in _JPEG_SOF_SUPPORTED:
raise _unsupported("Unsupported JPEG coding")
if self._seen_sof:
raise _unsupported("Multiple JPEG frames")
if marker == 0xDA and not self._seen_sof:
raise _unsupported("JPEG scan before frame header")
self._marker = marker
self._seg_left = length - 2
self._seg_small = bytearray()
def _end_segment(self) -> None:
marker = self._marker
if marker in _JPEG_SOF_SUPPORTED:
seg = bytes(self._seg_small)
if len(seg) < 6:
raise _unsupported("Corrupted JPEG frame header")
precision, height, width, components = struct.unpack(">BHHB", seg[:6])
# 8-bit only: what browsers and the verifying decoder (libjpeg-turbo 2.1) support.
if precision != 8 or components not in (1, 3, 4) or len(seg) != 6 + 3 * components:
raise _unsupported("Unsupported JPEG frame header")
if height == 0:
raise _unsupported("JPEG without explicit height is not supported")
ids = set()
for i in range(components):
cid = seg[6 + 3 * i]
if cid in ids:
raise _unsupported("Duplicate JPEG component id")
ids.add(cid)
self._sof_ids = ids
self._set_dimensions(width, height)
self._seen_sof = True
if marker == 0xDA:
# SOS must name only components declared in SOF (djpeg rejects mismatches).
seg = bytes(self._seg_small)
if len(seg) < 1:
raise _unsupported("Corrupted JPEG scan header")
ns = seg[0]
if ns < 1 or ns > 4 or len(seg) < 1 + 2 * ns + 3:
raise _unsupported("Corrupted JPEG scan header")
for i in range(ns):
cid = seg[1 + 2 * i]
if cid not in self._sof_ids:
raise _unsupported("JPEG scan component not in frame header")
self._seen_sos = True
self._state = "entropy"
else:
self._state = "marker"
def feed(self, data: bytes) -> None:
pos = 0
n = len(data)
while pos < n:
if self._done:
# Bytes after EOI are tolerated but never interpreted.
return
state = self._state
if state == "soi":
need = 2 - len(self._buf)
self._buf += data[pos:pos + need]
pos += min(need, n - pos)
if len(self._buf) == 2:
if bytes(self._buf) != b"\xff\xd8":
raise _unsupported("Not a JPEG image")
self._buf.clear()
self._state = "marker"
elif state == "marker":
byte = data[pos]
pos += 1
if not self._pending_ff:
if byte != 0xFF:
raise _unsupported("Corrupted JPEG marker")
self._pending_ff = True
continue
if byte == 0xFF:
continue
self._pending_ff = False
if byte == 0xD9:
if not self._seen_sos:
raise _unsupported("JPEG without image data")
self._done = True
elif byte == 0x01 or 0xD0 <= byte <= 0xD7:
continue
elif byte in (0x00, 0xD8):
raise _unsupported("Corrupted JPEG marker")
else:
self._marker = byte
self._state = "length"
elif state == "length":
need = 2 - len(self._buf)
self._buf += data[pos:pos + need]
pos += min(need, n - pos)
if len(self._buf) == 2:
(length,) = struct.unpack(">H", bytes(self._buf))
self._buf.clear()
self._begin_segment(self._marker, length)
self._state = "segment"
if self._seg_left == 0:
self._end_segment()
elif state == "segment":
take = min(self._seg_left, n - pos)
# Keep SOF and SOS bodies: SOS component ids must match SOF.
if self._marker in _JPEG_SOF_SUPPORTED or self._marker == 0xDA:
self._seg_small += data[pos:pos + take]
pos += take
self._seg_left -= take
if self._seg_left == 0:
self._end_segment()
elif state == "entropy":
if self._pending_ff:
byte = data[pos]
if byte == 0x00 or 0xD0 <= byte <= 0xD7:
pos += 1
self._pending_ff = False
elif byte == 0xFF:
pos += 1
else:
# A real marker ends the scan; reprocess it in marker state.
self._state = "marker"
continue
idx = data.find(b"\xff", pos)
if idx < 0:
return
pos = idx + 1
self._pending_ff = True
def finish(self) -> Tuple[int, int]:
if not self._done or self.width is None or self.height is None:
raise _unsupported("Truncated JPEG image")
return self.width, self.height
def validator_for(content_type: str, max_pixels: int) -> _Validator:
if content_type == "image/png":
return PngValidator(max_pixels)
if content_type == "image/jpeg":
return JpegValidator(max_pixels)
raise _unsupported("Only image/png and image/jpeg are accepted")
+50
View File
@@ -0,0 +1,50 @@
"""Graceful shutdown: readiness drain window before uvicorn stops accepting connections.
Plain uvicorn closes its listening socket on the first SIGTERM, while HAProxy keeps
routing to the replica until `/health/ready` fails `fall` times (inter 1s, fall 2).
Here the first SIGTERM/SIGINT only switches readiness to 503; the replica keeps
serving normally for SHUTDOWN_DRAIN_SECONDS, then uvicorn closes the listener and
waits up to SHUTDOWN_TIMEOUT_SECONDS for in-flight requests before the lifespan
shutdown (reconciler, Redis, DB pool).
"""
from __future__ import annotations
import logging
import time
from types import FrameType
from typing import Optional
import uvicorn
log = logging.getLogger("messenger")
draining = False
class DrainingServer(uvicorn.Server):
def __init__(self, config: uvicorn.Config, drain_seconds: float) -> None:
super().__init__(config)
self.drain_seconds = drain_seconds
self._drain_until: Optional[float] = None
self._drain_logged = False
def handle_exit(self, sig: int, frame: Optional[FrameType]) -> None:
global draining
draining = True
if self._drain_until is None and not self.should_exit and self.drain_seconds > 0:
# Runs as a signal handler: only set flags, logging happens in on_tick.
self._drain_until = time.monotonic() + self.drain_seconds
return
# A second signal ends the drain window at once; a repeated SIGINT forces exit.
super().handle_exit(sig, frame)
async def on_tick(self, counter: int) -> bool:
if self._drain_until is not None and not self.should_exit:
if not self._drain_logged:
self._drain_logged = True
log.info("shutdown signal: readiness 503, serving for %ss drain window", self.drain_seconds)
if time.monotonic() >= self._drain_until:
log.info("drain window over; closing listener, waiting for in-flight requests")
self.should_exit = True
return await super().on_tick(counter)
+28
View File
@@ -0,0 +1,28 @@
"""One-shot migrations entrypoint (run once before API replicas)."""
from __future__ import annotations
import logging
import sys
from messenger import db
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
log = logging.getLogger("messenger.migrate")
def main() -> int:
db.init_pool()
try:
db.apply_migrations()
log.info("migrations complete")
return 0
finally:
db.close_pool()
if __name__ == "__main__":
raise SystemExit(main())
+62
View File
@@ -0,0 +1,62 @@
"""Transactional outbox publisher: closes the COMMIT↔publish gap."""
from __future__ import annotations
import logging
import threading
import time
from typing import Optional
from messenger import broker, config
from messenger.store import Store, StoreUnavailable
log = logging.getLogger("messenger.outbox")
class OutboxPublisher:
def __init__(self, store: Store):
self._store = store
self._stop = threading.Event()
self._thread: Optional[threading.Thread] = None
def start(self) -> None:
if self._thread and self._thread.is_alive():
return
self._stop.clear()
self._thread = threading.Thread(target=self._run, name="outbox-publisher", daemon=True)
self._thread.start()
def stop(self, timeout: float = 2.0) -> None:
self._stop.set()
if self._thread and self._thread.is_alive():
self._thread.join(timeout=timeout)
def _run(self) -> None:
interval = config.outbox_poll_seconds()
while not self._stop.is_set():
try:
self.flush_once(limit=50)
except StoreUnavailable:
log.warning("outbox: database unavailable")
except Exception:
log.exception("outbox flush failed")
self._stop.wait(interval)
def flush_once(self, limit: int = 50) -> int:
"""Publish unpublished outbox rows with broker confirms; returns published count."""
rows = self._store.claim_outbox_batch(limit)
published = 0
for row in rows:
try:
broker.publish_job(row["payload"])
self._store.mark_outbox_published(row["id"])
published += 1
except Exception as exc:
log.warning(
"outbox publish failed id=%s job=%s err=%s",
row["id"],
row["job_id"],
type(exc).__name__,
)
self._store.bump_outbox_attempt(row["id"])
return published
+342
View File
@@ -0,0 +1,342 @@
"""DB/S3 reconciler: removes orphan objects after a safe grace, never live work.
Rules (never violated, also with --grace-seconds 0):
* an object whose key is referenced by attachments.object_key, attachments.staging_key
(for queued/processing), attachment_variants.object_key, or upload_intents is never deleted;
* metadata is never deleted (a missing original is only reported);
* intent / staging cleanup re-reads row state under FOR UPDATE right before DELETE
so replay cannot race into source_missing / ready-without-bytes (TOCTOU);
* staging for queued/processing is retained regardless of STAGING_RETENTION_SECONDS.
CLI: python -m messenger.reconcile [--dry-run] [--grace-seconds N] [--check-missing]
"""
from __future__ import annotations
import argparse
import json
import logging
import random
import sys
import threading
import time
from typing import Dict, List, Optional
from messenger import config, db, s3
log = logging.getLogger("messenger.reconcile")
LOCK_KEY = 0x4C414235 # "LAB5": one reconciler at a time across replicas
BATCH = 500
def _referenced(conn, keys: List[str]) -> set:
"""All storage keys that must not be deleted (live or finished effects)."""
if not keys:
return set()
rows = conn.execute(
"""
SELECT object_key AS k FROM attachments
WHERE object_key = ANY(%s)
UNION
SELECT staging_key FROM attachments
WHERE staging_key = ANY(%s)
AND status IN ('queued', 'processing')
UNION
SELECT object_key FROM attachment_variants
WHERE object_key = ANY(%s)
UNION
SELECT object_key FROM upload_intents
WHERE object_key = ANY(%s)
""",
(keys, keys, keys, keys),
).fetchall()
return {r["k"] for r in rows}
def _resolve_intents(conn, grace: int, dry_run: bool, report: Dict[str, int]) -> None:
if dry_run:
rows = conn.execute(
"""
SELECT i.object_key,
EXISTS(
SELECT 1 FROM attachments a
WHERE a.object_key = i.object_key
OR (a.staging_key = i.object_key AND a.status IN ('queued', 'processing'))
) AS committed
FROM upload_intents i
WHERE i.created_at < now() - make_interval(secs => %s)
""",
(grace,),
).fetchall()
report["stale_intents"] += len(rows)
report["intent_objects_deleted"] += sum(1 for r in rows if not r["committed"])
return
while True:
with conn.transaction():
rows = conn.execute(
"""
SELECT object_key FROM upload_intents
WHERE created_at < now() - make_interval(secs => %s)
ORDER BY created_at
LIMIT %s
FOR UPDATE SKIP LOCKED
""",
(grace, BATCH),
).fetchall()
if not rows:
return
for row in rows:
key = row["object_key"]
report["stale_intents"] += 1
# Re-check under the intent lock: replay/upload may have claimed the key.
live = conn.execute(
"""
SELECT 1 AS ok FROM attachments
WHERE object_key = %s
OR (staging_key = %s AND status IN ('queued', 'processing'))
""",
(key, key),
).fetchone()
if not live:
s3.delete_object(key)
report["intent_objects_deleted"] += 1
conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (key,))
if len(rows) < BATCH:
return
def _sweep_prefix(
conn,
prefix: str,
grace: int,
dry_run: bool,
report: Dict[str, int],
listed: Optional[set],
) -> None:
db_now = conn.execute("SELECT now() AS now").fetchone()["now"]
page: List[dict] = []
def flush() -> None:
keys = [o["Key"] for o in page]
refs = _referenced(conn, keys)
for obj in page:
key = obj["Key"]
if key in refs:
report["referenced_objects"] += 1
continue
age = (db_now - obj["LastModified"]).total_seconds()
if age < grace:
report["fresh_unreferenced_skipped"] += 1
continue
report["orphans_found"] += 1
if not dry_run:
# TOCTOU: lock and re-check in one transaction before DELETE.
# Replay may have moved failed→queued / ready and re-linked the key.
with conn.transaction():
locked = conn.execute(
"""
SELECT id, status FROM attachments
WHERE staging_key = %s OR object_key = %s
FOR UPDATE
""",
(key, key),
).fetchall()
if any(r["status"] in ("queued", "processing", "ready") for r in locked):
report["referenced_objects"] += 1
continue
variant_hit = conn.execute(
"""
SELECT 1 AS ok FROM attachment_variants
WHERE object_key = %s
FOR UPDATE
""",
(key,),
).fetchone()
if variant_hit or _referenced(conn, [key]):
report["referenced_objects"] += 1
continue
s3.delete_object(key)
report["orphans_deleted"] += 1
page.clear()
for obj in s3.list_objects(prefix=prefix):
report["objects_scanned"] += 1
if listed is not None:
listed.add(obj["Key"])
page.append(obj)
if len(page) >= BATCH:
flush()
if page:
flush()
def _cleanup_stale_staging(conn, retention: int, dry_run: bool, report: Dict[str, int]) -> None:
"""Drop staging only for terminal failed attachments older than retention."""
if dry_run:
rows = conn.execute(
"""
SELECT staging_key FROM attachments
WHERE status = 'failed'
AND staging_key IS NOT NULL
AND updated_at < now() - make_interval(secs => %s)
""",
(retention,),
).fetchall()
report["stale_staging_candidates"] = len(rows)
return
while True:
with conn.transaction():
rows = conn.execute(
"""
SELECT id, staging_key FROM attachments
WHERE status = 'failed'
AND staging_key IS NOT NULL
AND updated_at < now() - make_interval(secs => %s)
ORDER BY updated_at
LIMIT %s
FOR UPDATE SKIP LOCKED
""",
(retention, BATCH),
).fetchall()
if not rows:
return
for row in rows:
# Re-read status under row lock: replay may have moved failed→queued.
cur = conn.execute(
"SELECT status, staging_key FROM attachments WHERE id = %s",
(row["id"],),
).fetchone()
if cur is None or cur["status"] != "failed" or not cur["staging_key"]:
continue
key = cur["staging_key"]
s3.delete_object(key)
conn.execute(
"UPDATE attachments SET staging_key = NULL, updated_at = now() WHERE id = %s",
(row["id"],),
)
report["stale_staging_deleted"] += 1
if len(rows) < BATCH:
return
def run(
grace_seconds: int,
dry_run: bool = False,
check_missing: bool = False,
resolve_intents: bool = True,
) -> dict:
started = time.monotonic()
report: Dict[str, int] = {
"stale_intents": 0,
"intent_objects_deleted": 0,
"objects_scanned": 0,
"referenced_objects": 0,
"fresh_unreferenced_skipped": 0,
"orphans_found": 0,
"orphans_deleted": 0,
"stale_staging_candidates": 0,
"stale_staging_deleted": 0,
}
retention = config.staging_retention_seconds()
with db.get_pool().connection(deadline=None) as conn:
conn.autocommit = True
locked = conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (LOCK_KEY,)).fetchone()["ok"]
if not locked:
return {"skipped": "another reconciler holds the lock"}
try:
if resolve_intents:
_resolve_intents(conn, grace_seconds, dry_run, report)
listed = set() if check_missing else None
for prefix in (s3.ORIGINAL_PREFIX, s3.STAGING_PREFIX, s3.VARIANT_PREFIX):
_sweep_prefix(conn, prefix, grace_seconds, dry_run, report, listed)
_cleanup_stale_staging(conn, retention, dry_run, report)
if listed is not None:
rows = conn.execute(
"""
SELECT object_key FROM attachments
WHERE object_key IS NOT NULL AND created_at < now()
"""
).fetchall()
report["missing_originals"] = sum(
1 for r in rows if r["object_key"] not in listed
)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_KEY,))
conn.autocommit = False
result = {"dry_run": dry_run, "grace_seconds": grace_seconds, **report}
result["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
return result
_stop = threading.Event()
_thread: Optional[threading.Thread] = None
def _loop(interval: int, grace: int) -> None:
if _stop.wait(random.uniform(5, 15)):
return
while not _stop.is_set():
try:
result = run(grace)
if (
result.get("intent_objects_deleted")
or result.get("orphans_deleted")
or result.get("stale_staging_deleted")
):
log.info("reconcile %s", json.dumps(result))
except Exception as exc:
log.warning("reconcile failed: %s", type(exc).__name__)
if _stop.wait(interval * random.uniform(0.8, 1.2)):
return
def start_background() -> Optional[threading.Thread]:
global _thread
interval = config.reconcile_interval_seconds()
if interval == 0:
return None
grace = config.upload_grace_seconds()
_stop.clear()
_thread = threading.Thread(target=_loop, args=(interval, grace), name="reconciler", daemon=True)
_thread.start()
return _thread
def stop_background(timeout: float = 1.0) -> None:
_stop.set()
if _thread is not None:
_thread.join(timeout)
if _thread.is_alive():
log.info("reconciler still running at shutdown; leaving it to the next pass")
def main(argv: Optional[List[str]] = None) -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--dry-run", action="store_true", help="report only, change nothing")
parser.add_argument(
"--grace-seconds",
type=int,
default=None,
help="override UPLOAD_GRACE_SECONDS (lab demos only)",
)
parser.add_argument(
"--check-missing",
action="store_true",
help="also report metadata whose original is absent",
)
args = parser.parse_args(argv)
grace = config.upload_grace_seconds() if args.grace_seconds is None else args.grace_seconds
if grace < 0:
parser.error("--grace-seconds must be >= 0")
db.init_pool()
try:
print(json.dumps(run(grace, dry_run=args.dry_run, check_missing=args.check_missing)))
finally:
db.close_pool()
return 0
if __name__ == "__main__":
sys.exit(main())
+71
View File
@@ -0,0 +1,71 @@
"""CLI entry: python -m messenger.replay (same semantics as scripts/replay_failed.py)."""
from __future__ import annotations
import argparse
import sys
def main() -> int:
from messenger import db
from messenger.outbox import OutboxPublisher
from messenger.store import Store
parser = argparse.ArgumentParser(description="Replay failed attachment jobs")
parser.add_argument("--dead-letter-id", type=int, default=None)
parser.add_argument("--attachment-id", default=None)
parser.add_argument("--list", action="store_true", help="list open dead letters")
parser.add_argument("--note", default="operator replay")
args = parser.parse_args()
db.init_pool()
store = Store()
try:
if args.list:
with db.get_pool().connection() as conn:
rows = conn.execute(
"""
SELECT id, job_id, attachment_id, error_code, attempts, failed_at
FROM dead_letter_jobs
WHERE replayed_at IS NULL
ORDER BY failed_at
LIMIT 50
"""
).fetchall()
for r in rows:
print(
f"id={r['id']} attachment={r['attachment_id']} "
f"error={r['error_code']} attempts={r['attempts']}"
)
print(f"open_dead_letters={len(rows)}")
return 0
dead_id = args.dead_letter_id
if dead_id is None and args.attachment_id:
with db.get_pool().connection() as conn:
row = conn.execute(
"""
SELECT id FROM dead_letter_jobs
WHERE attachment_id = %s AND replayed_at IS NULL
ORDER BY failed_at DESC
LIMIT 1
""",
(args.attachment_id,),
).fetchone()
if row is None:
print("no open dead letter for attachment", file=sys.stderr)
return 2
dead_id = int(row["id"])
if dead_id is None:
parser.error("need --dead-letter-id or --attachment-id or --list")
result = store.replay_dead_letter(dead_id, note=args.note)
if result is None:
print("dead letter not found", file=sys.stderr)
return 2
OutboxPublisher(store).flush_once()
print(f"replay_result={result} dead_letter_id={dead_id}")
return 0
finally:
db.close_pool()
if __name__ == "__main__":
raise SystemExit(main())
+193
View File
@@ -0,0 +1,193 @@
"""Private S3 bucket access (SeaweedFS S3 gateway) with bounded timeouts."""
from __future__ import annotations
import io
import logging
import threading
import time
from typing import BinaryIO, Iterator, Optional, Tuple
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from messenger import config
log = logging.getLogger("messenger.s3")
_client = None
ORIGINAL_PREFIX = "original/"
STAGING_PREFIX = "staging/"
VARIANT_PREFIX = "variants/"
DOWNLOAD_CHUNK = 64 * 1024
class S3Unavailable(Exception):
"""S3 is unreachable, timed out or refused the request."""
class S3ObjectMissing(Exception):
"""Object referenced by metadata does not exist in the bucket."""
# Worst case per call: 2 attempts × (2 s connect + 5 s read) + backoff ≈ 15 s, well
# under the LB `timeout server` (45 s), so clients get a JSON 503, not an LB 504.
CONNECT_TIMEOUT_S = 2
READ_TIMEOUT_S = 5
TOTAL_ATTEMPTS = 2
PING_TTL_S = 1.0
_ping_client = None
_ping_lock = threading.Lock()
_ping_cache: Tuple[float, bool] = (0.0, False)
def build_client(access_key: str, secret_key: str, *, connect: float = CONNECT_TIMEOUT_S,
read: float = READ_TIMEOUT_S, attempts: int = TOTAL_ATTEMPTS):
return boto3.client(
"s3",
endpoint_url=config.s3_endpoint(),
region_name=config.s3_region(),
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"total_max_attempts": attempts, "mode": "standard"},
max_pool_connections=32,
),
)
def init_client():
global _client, _ping_client
if _client is None:
access, secret = config.s3_access_key_id(), config.s3_secret_access_key()
_client = build_client(access, secret)
_ping_client = build_client(access, secret, connect=1, read=1.5, attempts=1)
return _client
def client():
return init_client()
def _error_code(exc: ClientError) -> str:
return str(exc.response.get("Error", {}).get("Code", ""))
def put_object(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
try:
client().put_object(
Bucket=config.s3_bucket(),
Key=key,
Body=body,
ContentLength=size,
ContentType=content_type,
Metadata={"sha256": sha256_hex},
)
except (BotoCoreError, ClientError) as exc:
log.warning("s3 put failed: %s", type(exc).__name__)
raise S3Unavailable("s3 put failed") from exc
def put_bytes(key: str, data: bytes, content_type: str, sha256_hex: str) -> None:
put_object(key, io.BytesIO(data), len(data), content_type, sha256_hex)
def get_bytes(key: str) -> bytes:
length, chunks, body = open_object(key)
try:
data = b"".join(chunks)
finally:
body.close()
if length != len(data):
raise S3Unavailable("s3 get incomplete")
return data
def copy_object(src_key: str, dst_key: str, content_type: str) -> None:
try:
client().copy_object(
Bucket=config.s3_bucket(),
Key=dst_key,
CopySource={"Bucket": config.s3_bucket(), "Key": src_key},
ContentType=content_type,
MetadataDirective="REPLACE",
)
except (BotoCoreError, ClientError) as exc:
log.warning("s3 copy failed: %s", type(exc).__name__)
raise S3Unavailable("s3 copy failed") from exc
def delete_object(key: str) -> None:
"""Idempotent: deleting an absent key succeeds."""
try:
client().delete_object(Bucket=config.s3_bucket(), Key=key)
except (BotoCoreError, ClientError) as exc:
log.warning("s3 delete failed: %s", type(exc).__name__)
raise S3Unavailable("s3 delete failed") from exc
def head_object(key: str) -> Optional[dict]:
try:
return client().head_object(Bucket=config.s3_bucket(), Key=key)
except ClientError as exc:
if _error_code(exc) in ("404", "NoSuchKey", "NotFound"):
return None
raise S3Unavailable("s3 head failed") from exc
except BotoCoreError as exc:
raise S3Unavailable("s3 head failed") from exc
def open_object(key: str) -> Tuple[int, Iterator[bytes], object]:
"""Start a GET before response headers are sent so failures can still map to JSON errors."""
try:
resp = client().get_object(Bucket=config.s3_bucket(), Key=key)
except ClientError as exc:
if _error_code(exc) in ("404", "NoSuchKey", "NotFound"):
raise S3ObjectMissing(key) from exc
raise S3Unavailable("s3 get failed") from exc
except BotoCoreError as exc:
raise S3Unavailable("s3 get failed") from exc
body = resp["Body"]
return int(resp["ContentLength"]), body.iter_chunks(DOWNLOAD_CHUNK), body
def list_objects(prefix: str = ORIGINAL_PREFIX) -> Iterator[dict]:
token = None
while True:
kwargs = {"Bucket": config.s3_bucket(), "Prefix": prefix, "MaxKeys": 1000}
if token:
kwargs["ContinuationToken"] = token
try:
page = client().list_objects_v2(**kwargs)
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable("s3 list failed") from exc
for item in page.get("Contents", []) or []:
yield item
if not page.get("IsTruncated"):
return
token = page.get("NextContinuationToken")
def ping() -> bool:
"""Readiness probe: one short request, result shared for PING_TTL_S across callers."""
global _ping_cache
with _ping_lock:
checked_at, ok = _ping_cache
if time.monotonic() - checked_at < PING_TTL_S:
return ok
init_client()
try:
_ping_client.list_objects_v2(Bucket=config.s3_bucket(), Prefix=ORIGINAL_PREFIX, MaxKeys=1)
ok = True
except Exception as exc:
log.warning("s3 ping failed: %s", type(exc).__name__)
ok = False
_ping_cache = (time.monotonic(), ok)
return ok
+67
View File
@@ -0,0 +1,67 @@
"""Idempotent bucket bootstrap (one-shot `s3-init` service, admin credentials).
Creates the private bucket if absent; re-running is a no-op. Access rights are
not stored in the bucket: SeaweedFS reads identities (admin / messenger-api, no
anonymous identity) from the s3 config secret, so the bucket stays private.
"""
from __future__ import annotations
import logging
import sys
import time
import urllib.error
import urllib.request
from botocore.exceptions import BotoCoreError, ClientError
from messenger import config, s3
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("messenger.s3bootstrap")
def _anonymous_status(url: str) -> int:
try:
with urllib.request.urlopen(url, timeout=5) as resp:
return resp.status
except urllib.error.HTTPError as exc:
return exc.code
def main() -> int:
bucket = config.s3_bucket()
admin = s3.build_client(config.s3_admin_access_key_id(), config.s3_admin_secret_access_key())
deadline = time.monotonic() + 90
while True:
try:
admin.head_bucket(Bucket=bucket)
log.info("bucket exists: %s", bucket)
break
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
try:
admin.create_bucket(Bucket=bucket)
log.info("bucket created: %s", bucket)
except ClientError as create_exc:
code = create_exc.response.get("Error", {}).get("Code", "")
if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
raise
log.info("bucket appeared concurrently: %s", bucket)
except BotoCoreError as exc:
if time.monotonic() > deadline:
raise
log.info("waiting for S3 (%s)", type(exc).__name__)
time.sleep(2)
status = _anonymous_status(config.s3_endpoint().rstrip("/") + "/" + bucket + "?list-type=2")
if status != 403:
log.error("bucket is not private: anonymous list returned %s", status)
return 1
log.info("anonymous access denied (HTTP %s); bootstrap complete", status)
return 0
if __name__ == "__main__":
sys.exit(main())
+283
View File
@@ -0,0 +1,283 @@
"""Operator self-checks executed inside an API container (docker compose exec api ...).
python -m messenger.selfcheck s3 private bucket + least-privilege API key
python -m messenger.selfcheck consistency fault injection between S3 PUT and DB commit
python -m messenger.selfcheck spool upload buffers left on the API container
Prints one JSON document; exit code 1 if any check fails. Never prints credentials.
"""
from __future__ import annotations
import hashlib
import json
import os
import socket
import struct
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
import zlib
from typing import Callable, Dict, List
from botocore.exceptions import ClientError
from messenger import attachments, config, db, reconcile, s3
from messenger.store import CommitUncertain, Store
INTERNAL_SEAWEED_PORTS = (8333, 18333, 8888, 18888, 9333, 19333, 8080, 18080)
def _png(width: int = 4, height: int = 3) -> bytes:
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
raw = b"".join(b"\x00" + bytes((x * 40) % 256 for x in range(width * 3)) for _ in range(height))
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
class Checks:
def __init__(self) -> None:
self.results: List[Dict[str, object]] = []
def check(self, name: str, ok: bool, detail: object = "") -> bool:
self.results.append({"check": name, "ok": bool(ok), "detail": detail})
return ok
def expect_denied(self, name: str, fn: Callable[[], object]) -> None:
try:
fn()
self.check(name, False, "allowed")
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
self.check(name, status in (403, 400, 405) and code != "", f"{status} {code}")
def report(self, title: str) -> int:
ok = all(r["ok"] for r in self.results)
print(json.dumps({"selfcheck": title, "passed": ok, "results": self.results}, ensure_ascii=False, indent=1))
return 0 if ok else 1
def _anonymous_get(url: str) -> object:
try:
with urllib.request.urlopen(url, timeout=5) as resp:
return resp.status
except urllib.error.HTTPError as exc:
return exc.code
except OSError as exc:
return type(exc).__name__
def _port_open(host: str, port: int) -> bool:
try:
with socket.create_connection((host, port), timeout=2):
return True
except OSError:
return False
def check_s3() -> int:
c = Checks()
bucket = config.s3_bucket()
endpoint = config.s3_endpoint().rstrip("/")
host = urllib.parse.urlparse(endpoint).hostname or "s3"
app = s3.client()
key = s3.ORIGINAL_PREFIX + "selfcheck-" + uuid.uuid4().hex
body = _png()
app.put_object(Bucket=bucket, Key=key, Body=body, ContentType="image/png")
c.check("api_key_put_object", True)
got = app.get_object(Bucket=bucket, Key=key)["Body"].read()
c.check("api_key_get_object", got == body)
status = _anonymous_get(f"{endpoint}/{bucket}/{key}")
c.check("anonymous_get_object_denied", status == 403, status)
status = _anonymous_get(f"{endpoint}/{bucket}/{key}?x-id=GetObject")
c.check("anonymous_get_object_query_denied", status == 403, status)
status = _anonymous_get(f"{endpoint}/{bucket}?list-type=2")
c.check("anonymous_list_bucket_denied", status == 403, status)
status = _anonymous_get(f"{endpoint}/")
c.check("anonymous_list_buckets_denied", status == 403, status)
c.expect_denied("api_key_list_buckets_denied", lambda: app.list_buckets())
c.expect_denied("api_key_create_bucket_denied", lambda: app.create_bucket(Bucket="lab4-escalation-" + uuid.uuid4().hex[:8]))
c.expect_denied("api_key_delete_bucket_denied", lambda: app.delete_bucket(Bucket=bucket))
c.expect_denied("api_key_put_bucket_acl_denied", lambda: app.put_bucket_acl(Bucket=bucket, ACL="public-read"))
c.expect_denied("api_key_put_object_acl_denied", lambda: app.put_object_acl(Bucket=bucket, Key=key, ACL="public-read"))
c.expect_denied("api_key_other_bucket_denied", lambda: app.put_object(Bucket="lab4-other", Key="x", Body=b"x"))
status = _anonymous_get(f"{endpoint}/{bucket}/{key}")
c.check("anonymous_get_still_denied_after_acl_attempts", status == 403, status)
wrong = s3.build_client(config.s3_access_key_id(), "wrong-" + uuid.uuid4().hex)
c.expect_denied("wrong_secret_denied", lambda: wrong.get_object(Bucket=bucket, Key=key))
open_ports = [p for p in INTERNAL_SEAWEED_PORTS if _port_open(host, p)]
c.check("seaweed_admin_and_grpc_ports_unreachable", not open_ports, {"open": open_ports, "probed": list(INTERNAL_SEAWEED_PORTS)})
c.check("s3_api_port_reachable", _port_open(host, urllib.parse.urlparse(endpoint).port or 80))
app.delete_object(Bucket=bucket, Key=key)
c.check("api_key_delete_object", s3.head_object(key) is None)
return c.report("s3")
def _reconcile(**kwargs) -> dict:
for _ in range(20):
result = reconcile.run(**kwargs)
if "skipped" not in result:
return result
time.sleep(0.5)
raise RuntimeError("reconciler lock busy")
def _intent_exists(key: str) -> bool:
with db.get_pool().connection() as conn:
return conn.execute("SELECT 1 AS ok FROM upload_intents WHERE object_key = %s", (key,)).fetchone() is not None
def _attachment_exists(key: str) -> bool:
with db.get_pool().connection() as conn:
return conn.execute("SELECT 1 AS ok FROM attachments WHERE object_key = %s", (key,)).fetchone() is not None
def _spool(body: bytes) -> attachments.Spooled:
config.spool_dir().mkdir(parents=True, exist_ok=True)
fd, path = tempfile.mkstemp(prefix="selfcheck-", dir=str(config.spool_dir()))
with os.fdopen(fd, "wb") as out:
out.write(body)
return attachments.Spooled(path, len(body), hashlib.sha256(body).hexdigest(), "image/png", 4, 3)
def check_consistency() -> int:
c = Checks()
db.init_pool()
store = Store()
grace = config.upload_grace_seconds()
suffix = uuid.uuid4().hex[:10]
user = store.create_user("selfcheck_" + suffix, "Selfcheck", "Selfcheck-" + uuid.uuid4().hex)
chat = store.create_chat("selfcheck", [user.id])
body = _png()
keys: Dict[str, str] = {}
def persist(fault=None):
spooled = _spool(body)
try:
return attachments.persist(store, spooled, chat_id=chat.id, owner_id=user.id,
filename="selfcheck.png", instance="selfcheck", fault=fault)
finally:
attachments.discard(spooled)
# A. S3 PUT fails: intent is dropped, nothing stays in the bucket.
def fail_put(step: str, key: str) -> None:
keys["put"] = key
if step == "before_put":
raise s3.S3Unavailable("injected")
try:
persist(fail_put)
c.check("A_s3_put_failure_raises", False)
except s3.S3Unavailable:
c.check("A_s3_put_failure_raises", True)
k = keys["put"]
c.check("A_no_object_no_intent_no_metadata",
s3.head_object(k) is None and not _intent_exists(k) and not _attachment_exists(k))
# B. PUT succeeded, DB commit fails: compensation deletes the object at once.
def fail_commit(step: str, key: str) -> None:
keys["commit"] = key
if step == "after_put":
raise RuntimeError("injected commit failure")
try:
persist(fail_commit)
c.check("B_commit_failure_raises", False)
except RuntimeError:
c.check("B_commit_failure_raises", True)
k = keys["commit"]
c.check("B_compensation_removed_object_and_intent",
s3.head_object(k) is None and not _intent_exists(k) and not _attachment_exists(k))
# C. Process dies after PUT (no compensation): object + intent remain until grace expires.
def crash(step: str, key: str) -> None:
keys["crash"] = key
if step == "after_put":
raise attachments.SimulatedCrash()
try:
persist(crash)
except attachments.SimulatedCrash:
pass
k = keys["crash"]
c.check("C_crash_leaves_object_and_intent", s3.head_object(k) is not None and _intent_exists(k))
first = _reconcile(grace_seconds=grace)
c.check("C_fresh_intent_survives_reconcile", s3.head_object(k) is not None and _intent_exists(k), first)
with db.get_pool().connection() as conn:
with conn.transaction():
conn.execute(
"UPDATE upload_intents SET created_at = now() - make_interval(secs => %s) WHERE object_key = %s",
(grace + 60, k),
)
second = _reconcile(grace_seconds=grace)
c.check("C_stale_intent_object_deleted", s3.head_object(k) is None and not _intent_exists(k), second)
again = _reconcile(grace_seconds=grace)
c.check("C_rerun_is_noop", s3.head_object(k) is None, again)
# D. Orphan object without intent/metadata (e.g. intent lost): only the age rule applies.
orphan = attachments.new_object_key()
s3.client().put_object(Bucket=config.s3_bucket(), Key=orphan, Body=body, ContentType="image/png")
dry = _reconcile(grace_seconds=grace, resolve_intents=False, dry_run=True)
c.check("D_fresh_orphan_not_deleted", s3.head_object(orphan) is not None, dry)
time.sleep(3)
swept = _reconcile(grace_seconds=2, resolve_intents=False)
c.check("D_old_orphan_deleted_by_sweep", s3.head_object(orphan) is None, swept)
# F. COMMIT succeeded but its result was lost: the object must NOT be compensated.
def lose_commit_reply(step: str, key: str) -> None:
keys["uncertain"] = key
if step == "after_commit":
raise CommitUncertain("injected lost COMMIT reply")
try:
persist(lose_commit_reply)
c.check("F_uncertain_commit_raises", False)
except CommitUncertain:
c.check("F_uncertain_commit_raises", True)
k = keys["uncertain"]
c.check("F_uncertain_commit_keeps_object_and_metadata",
s3.head_object(k) is not None and _attachment_exists(k) and not _intent_exists(k))
swept = _reconcile(grace_seconds=0, resolve_intents=False, check_missing=True)
c.check("F_reconcile_keeps_committed_object", s3.head_object(k) is not None, swept)
# E. Committed attachments are never touched, even with grace 0.
att = persist()
c.check("E_upload_committed", _attachment_exists(att.object_key) and not _intent_exists(att.object_key))
swept = _reconcile(grace_seconds=0, resolve_intents=False, check_missing=True)
c.check("E_referenced_object_kept", s3.head_object(att.object_key) is not None, swept)
c.check("E_no_missing_originals", swept.get("missing_originals") == 0, swept.get("missing_originals"))
db.close_pool()
return c.report("consistency")
def check_spool() -> int:
c = Checks()
root = config.spool_dir()
files = [p.name for p in root.iterdir()] if root.exists() else []
c.check("spool_dir_empty", not files, {"dir": str(root), "files": len(files)})
return c.report("spool")
def main(argv: List[str]) -> int:
commands = {"s3": check_s3, "consistency": check_consistency, "spool": check_spool}
if len(argv) != 1 or argv[0] not in commands:
print(__doc__)
return 2
return commands[argv[0]]()
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+212
View File
@@ -0,0 +1,212 @@
"""Opaque Bearer sessions in Redis with TTL (durable via Redis AOF).
Revocation is durable in PostgreSQL so logout still wins if Redis is briefly down:
stop Redis → logout 204 → start Redis → old token is 401 (not resurrected from AOF).
"""
from __future__ import annotations
import hashlib
import json
import logging
import secrets
from datetime import datetime, timedelta, timezone
from typing import Optional
import redis
from psycopg import Error as PsycopgError
from messenger import config
from messenger.db import get_pool
log = logging.getLogger("messenger.sessions")
_client: Optional[redis.Redis] = None
TOKEN_BYTES = 32 # 256-bit entropy
KEY_PREFIX = "sess:"
def utc_now() -> datetime:
return datetime.now(timezone.utc).replace(microsecond=0)
def format_ts(value: datetime) -> str:
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def parse_ts(value: str) -> datetime:
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
def token_hash(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def init_client() -> redis.Redis:
global _client
if _client is not None:
return _client
url = config.cache_url()
_client = redis.Redis.from_url(
url,
decode_responses=True,
socket_connect_timeout=3,
socket_timeout=3,
health_check_interval=30,
)
return _client
def close_client() -> None:
global _client
if _client is not None:
try:
_client.close()
except Exception:
pass
_client = None
def get_client() -> redis.Redis:
if _client is None:
raise RuntimeError("Redis client is not initialized")
return _client
def ping() -> bool:
try:
return bool(get_client().ping())
except Exception as exc:
log.warning("redis ping failed: %s", type(exc).__name__)
return False
class SessionStoreError(Exception):
"""Raised when Redis is unavailable for session operations."""
def _is_revoked(token: str) -> bool:
try:
with get_pool().connection() as conn:
row = conn.execute(
"""
SELECT 1 AS ok FROM session_revocations
WHERE token_hash = %s AND expires_at > now()
""",
(token_hash(token),),
).fetchone()
return row is not None
except PsycopgError as exc:
raise SessionStoreError("session revocation store unavailable") from exc
def _record_revocation(token: str, expires_at: Optional[datetime] = None) -> None:
"""Persist revocation so a later Redis restore cannot resurrect the session."""
exp = expires_at or (utc_now() + timedelta(seconds=config.session_ttl_seconds()))
try:
with get_pool().connection() as conn:
with conn.transaction():
conn.execute(
"""
INSERT INTO session_revocations (token_hash, revoked_at, expires_at)
VALUES (%s, now(), %s)
ON CONFLICT (token_hash) DO UPDATE
SET revoked_at = now(), expires_at = GREATEST(session_revocations.expires_at, EXCLUDED.expires_at)
""",
(token_hash(token), exp),
)
except PsycopgError as exc:
raise SessionStoreError("session revocation store unavailable") from exc
def create_session(user_id: str) -> dict:
ttl = config.session_ttl_seconds()
token = secrets.token_urlsafe(TOKEN_BYTES)
now = utc_now()
expires_at = now + timedelta(seconds=ttl)
payload = {
"user_id": user_id,
"expires_at": format_ts(expires_at),
"created_at": format_ts(now),
}
key = KEY_PREFIX + token
try:
client = get_client()
client.set(key, json.dumps(payload), ex=ttl)
except redis.RedisError as exc:
raise SessionStoreError("session store unavailable") from exc
return {
"token": token,
"token_type": "Bearer",
"expires_at": payload["expires_at"],
"user_id": user_id,
}
def get_session(token: str) -> Optional[dict]:
if not token:
return None
if _is_revoked(token):
# Drop a stale Redis copy if present; never trust a revoked token again.
try:
get_client().delete(KEY_PREFIX + token)
except redis.RedisError:
pass
return None
key = KEY_PREFIX + token
try:
raw = get_client().get(key)
except redis.RedisError as exc:
raise SessionStoreError("session store unavailable") from exc
if not raw:
return None
try:
data = json.loads(raw)
expires_at = parse_ts(data["expires_at"])
except (ValueError, KeyError, TypeError, json.JSONDecodeError):
return None
if expires_at <= utc_now():
try:
get_client().delete(key)
except redis.RedisError:
pass
return None
return data
def revoke_session(token: str) -> bool:
"""Revoke durably.
Returns:
True — session existed (or Redis was down) and is now revoked → HTTP 204
False — token already revoked / unknown while Redis is up → HTTP 401
"""
if not token:
return False
if _is_revoked(token):
return False
expires_at = None
key = KEY_PREFIX + token
redis_down = False
raw = None
try:
raw = get_client().get(key)
if raw:
try:
expires_at = parse_ts(json.loads(raw)["expires_at"])
except (ValueError, KeyError, TypeError, json.JSONDecodeError):
expires_at = None
except redis.RedisError:
redis_down = True
if raw is None and not redis_down:
# Redis reachable and no live session → unknown/expired token.
return False
_record_revocation(token, expires_at)
try:
get_client().delete(key)
except redis.RedisError:
pass
return True
File diff suppressed because it is too large Load Diff
+140
View File
@@ -0,0 +1,140 @@
"""Derive attachment variants: thumbnail, crop, optimized WebP (lab5 grade 4)."""
from __future__ import annotations
import hashlib
import io
import logging
from dataclasses import dataclass
from typing import Tuple
from PIL import Image, ImageOps
log = logging.getLogger("messenger.variants")
# Grade-4 optimized WebP: quality/method chosen for size vs fidelity; EXIF stripped.
WEBP_QUALITY = 80
WEBP_METHOD = 4
WEBP_MAX_EDGE = 1280
THUMB_MAX_EDGE = 256
CROP_EDGE = 128
class ProcessingError(Exception):
def __init__(self, code: str, message: str, *, permanent: bool = True):
super().__init__(message)
self.code = code
self.message = message
self.permanent = permanent
@dataclass(frozen=True)
class VariantBytes:
name: str
content_type: str
data: bytes
width: int
height: int
sha256: str
@property
def size_bytes(self) -> int:
return len(self.data)
def _open_oriented(data: bytes) -> Image.Image:
try:
img = Image.open(io.BytesIO(data))
img.load()
except Exception as exc:
raise ProcessingError("invalid_image", "Cannot decode image") from exc
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.split()[-1])
return background
if img.mode != "RGB":
return img.convert("RGB")
return img
def _fit(width: int, height: int, max_edge: int) -> Tuple[int, int]:
"""Letterbox into max_edge×max_edge without upscaling; integer floor, min 1 px.
Uses integer division so 319×200 → 256×160 (not 256×161 from float rounding).
"""
if width <= 0 or height <= 0:
raise ProcessingError("invalid_image", "Invalid dimensions")
if width <= max_edge and height <= max_edge:
return width, height
# min(max_edge/w, max_edge/h) applied with floor: longer side becomes max_edge.
if width >= height:
return max_edge, max(1, (height * max_edge) // width)
return max(1, (width * max_edge) // height), max_edge
def _jpeg_bytes(img: Image.Image) -> bytes:
buf = io.BytesIO()
# No EXIF; progressive off for deterministic small thumbs.
img.save(buf, format="JPEG", quality=85, optimize=True, exif=b"")
return buf.getvalue()
def _webp_bytes(img: Image.Image) -> bytes:
buf = io.BytesIO()
img.save(
buf,
format="WEBP",
quality=WEBP_QUALITY,
method=WEBP_METHOD,
exact=False,
)
return buf.getvalue()
def _pack(name: str, content_type: str, data: bytes, width: int, height: int) -> VariantBytes:
return VariantBytes(
name=name,
content_type=content_type,
data=data,
width=width,
height=height,
sha256=hashlib.sha256(data).hexdigest(),
)
def make_thumbnail(original: bytes) -> VariantBytes:
img = _open_oriented(original)
tw, th = _fit(img.width, img.height, THUMB_MAX_EDGE)
if (tw, th) != (img.width, img.height):
img = img.resize((tw, th), Image.Resampling.LANCZOS)
data = _jpeg_bytes(img)
return _pack("thumbnail", "image/jpeg", data, tw, th)
def make_crop(original: bytes) -> VariantBytes:
"""Centered square 128×128 JPEG; small sources may be upscaled."""
img = _open_oriented(original)
side = min(img.width, img.height)
left = (img.width - side) // 2
top = (img.height - side) // 2
img = img.crop((left, top, left + side, top + side))
if img.size != (CROP_EDGE, CROP_EDGE):
img = img.resize((CROP_EDGE, CROP_EDGE), Image.Resampling.LANCZOS)
data = _jpeg_bytes(img)
return _pack("crop", "image/jpeg", data, CROP_EDGE, CROP_EDGE)
def make_optimized(original: bytes) -> VariantBytes:
"""WebP fitted into 1280×1280 without upscaling; EXIF removed; quality documented."""
img = _open_oriented(original)
ow, oh = _fit(img.width, img.height, WEBP_MAX_EDGE)
if (ow, oh) != (img.width, img.height):
img = img.resize((ow, oh), Image.Resampling.LANCZOS)
data = _webp_bytes(img)
return _pack("optimized", "image/webp", data, ow, oh)
def build_all(original: bytes) -> Tuple[VariantBytes, ...]:
return (make_thumbnail(original), make_crop(original), make_optimized(original))
+276
View File
@@ -0,0 +1,276 @@
"""Attachment processing worker: broker → variants → S3 → ready (lab5)."""
from __future__ import annotations
import logging
import os
import random
import signal
import socket
import sys
import threading
import time
from pathlib import Path
from typing import Any, Callable, Optional
_SRC = Path(__file__).resolve().parents[1]
if str(_SRC) not in sys.path:
sys.path.insert(0, str(_SRC))
from messenger import broker, config, db, s3
from messenger.outbox import OutboxPublisher
from messenger.store import Store, StoreUnavailable, Variant
from messenger.variants import ProcessingError, build_all
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("messenger.worker")
_stop = threading.Event()
_owner = (os.environ.get("HOSTNAME") or socket.gethostname())[:64]
def _backoff_seconds(attempt: int) -> float:
base = config.job_backoff_base_seconds()
cap = config.job_backoff_max_seconds()
exp = min(cap, base * (2 ** max(0, attempt - 1)))
# Full jitter: spread retries so thundering herds do not sync.
return random.uniform(0, exp)
def _variant_key(attachment_id: str, name: str, pipeline_version: int) -> str:
ext = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}[name]
return f"{s3.VARIANT_PREFIX}{attachment_id}/v{pipeline_version}/{name}.{ext}"
def process_job(store: Store, payload: dict[str, Any]) -> str:
"""
Process one job payload.
Returns: 'ack' | 'nack_requeue' (only for unexpected infra blips before side effects).
"""
job_id = str(payload.get("job_id", ""))
attachment_id = str(payload.get("attachment_id", ""))
if not job_id or not attachment_id:
log.error("malformed job payload")
return "ack"
leased = store.try_lease_job(job_id, _owner, config.job_timeout_seconds())
if leased is None:
# Backoff window or another worker holds the lease. Brief pause then
# requeue — do not ack-drop (that permanently loses the delivery).
time.sleep(0.2)
return "nack_requeue"
if leased.get("already_done") or leased.get("already_failed"):
return "ack"
att = leased["attachment"]
pipeline_version = int(leased["pipeline_version"])
attempt = int(leased["attempt"])
staging_key = att.staging_key
if not staging_key:
store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code="processing_failed",
message="missing staging object",
permanent=True,
max_attempts=config.job_max_attempts(),
delay_seconds=0,
payload=payload,
)
return "ack"
# Idempotent short-circuit: if variants+original already present for this version.
if att.status == "ready" and att.object_key and len(att.variants) >= 1:
return "ack"
try:
original_bytes = s3.get_bytes(staging_key)
except s3.S3ObjectMissing:
store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code="invalid_image",
message="staging object missing",
permanent=True,
max_attempts=config.job_max_attempts(),
delay_seconds=0,
payload=payload,
)
return "ack"
except s3.S3Unavailable:
outcome = store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code="processing_failed",
message="s3 unavailable reading staging",
permanent=False,
max_attempts=config.job_max_attempts(),
delay_seconds=_backoff_seconds(attempt),
payload=payload,
)
return "ack" if outcome in ("retry", "failed") else "ack"
if len(original_bytes) != att.size_bytes:
store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code="invalid_image",
message="staging size mismatch",
permanent=True,
max_attempts=config.job_max_attempts(),
delay_seconds=0,
payload=payload,
)
return "ack"
# Hard wall-clock deadline: a hung live decode must not sit in processing forever.
# Lease recovery alone does not interrupt a stuck process; this does.
try:
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
with ThreadPoolExecutor(max_workers=1) as pool:
fut = pool.submit(build_all, original_bytes)
try:
variants = fut.result(timeout=config.worker_process_timeout_seconds())
except FuturesTimeout:
raise ProcessingError(
"processing_failed", "processing timeout", permanent=False
) from None
except ProcessingError as exc:
outcome = store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code=exc.code,
message=exc.message,
permanent=exc.permanent,
max_attempts=config.job_max_attempts(),
delay_seconds=_backoff_seconds(attempt),
payload=payload,
)
log.warning("job %s processing error %s → %s", job_id, exc.code, outcome)
return "ack"
original_key = s3.ORIGINAL_PREFIX + attachment_id.replace("-", "") + uuid_suffix(att.sha256)
stored_variants: list[Variant] = []
try:
# Publish original to final prefix (bytes identical to staging).
s3.put_bytes(original_key, original_bytes, att.content_type, att.sha256)
for vb in variants:
key = _variant_key(attachment_id, vb.name, pipeline_version)
s3.put_bytes(key, vb.data, vb.content_type, vb.sha256)
stored_variants.append(
Variant(
name=vb.name,
content_type=vb.content_type,
size_bytes=vb.size_bytes,
width=vb.width,
height=vb.height,
sha256=vb.sha256,
object_key=key,
)
)
ok = store.complete_job_ready(
job_id=job_id,
attachment_id=attachment_id,
pipeline_version=pipeline_version,
original_key=original_key,
variants=stored_variants,
staging_key=staging_key,
)
if not ok:
log.info("job %s lost race to newer pipeline; acking", job_id)
return "ack"
# Safe to drop staging only after durable ready metadata.
try:
s3.delete_object(staging_key)
except s3.S3Unavailable:
log.warning("staging cleanup deferred for %s", staging_key[-8:])
if config.crash_after_side_effect():
log.error("fault injection: crash after side effect before ack")
os._exit(99)
return "ack"
except s3.S3Unavailable as exc:
outcome = store.fail_or_retry_job(
job_id=job_id,
attachment_id=attachment_id,
error_code="processing_failed",
message=f"s3 write failed: {type(exc).__name__}",
permanent=False,
max_attempts=config.job_max_attempts(),
delay_seconds=_backoff_seconds(attempt),
payload=payload,
)
log.warning("job %s s3 write failed → %s", job_id, outcome)
return "ack"
def uuid_suffix(sha256: str) -> str:
return sha256[:16]
def _handle(
store: Store,
payload: dict[str, Any],
ack: Callable[[], None],
nack: Callable[[bool], None],
) -> None:
try:
result = process_job(store, payload)
except StoreUnavailable:
log.warning("database unavailable during job; requeue")
nack(True)
return
except Exception:
log.exception("unexpected worker error")
nack(True)
return
if result == "ack":
ack()
elif result == "nack_requeue":
nack(True)
else:
nack(True)
def _lease_reaper(store: Store, outbox: OutboxPublisher) -> None:
while not _stop.wait(5.0):
try:
n = store.recover_expired_leases()
if n:
log.info("recovered %s expired leases", n)
outbox.flush_once()
except Exception:
log.exception("lease reaper failed")
def main() -> None:
signal.signal(signal.SIGTERM, lambda *_: _stop.set())
signal.signal(signal.SIGINT, lambda *_: _stop.set())
db.init_pool()
s3.init_client()
store = Store()
outbox = OutboxPublisher(store)
outbox.start()
reaper = threading.Thread(target=_lease_reaper, args=(store, outbox), name="lease-reaper", daemon=True)
reaper.start()
log.info(
"worker started owner=%s concurrency=%s max_attempts=%s",
_owner,
config.worker_concurrency(),
config.job_max_attempts(),
)
try:
broker.consume_forever(
lambda payload, ack, nack: _handle(store, payload, ack, nack),
prefetch=config.worker_concurrency(),
stop_flag=_stop.is_set,
)
finally:
_stop.set()
outbox.stop()
db.close_pool()
log.info("worker stopped")
if __name__ == "__main__":
main()
+19
View File
@@ -0,0 +1,19 @@
# Собственные проверки (lab4)
Нужен уже запущенный стенд (`scripts/up.cmd` / `scripts/up.sh`, ≥2 API за LB). Только stdlib Python.
```sh
# из корня репозитория (Windows PowerShell: $env:PYTHONUTF8=1; $env:RUN_DOCKER_TESTS=1; ...)
BASE_URL=http://127.0.0.1:8080 RUN_DOCKER_TESTS=1 RUN_DEPS_TEST=1 python student-tests/run_all.py
# отдельный модуль:
python student-tests/run_all.py test_attachments
```
- `test_images_unit` — офлайн-проверка потокового валидатора PNG/JPEG (любое разбиение на куски, граница 40 млн пикселей, корпус повреждённых входов ревьюеров).
- `test_attachments` — upload/download/ACL, лимиты 413 (ровно 10 485 760 байт → 201, +1 байт → 413, 8000×5001 → 413), повреждённые изображения и подмена MIME → 415, опасные filename и невалидный UTF-8/повтор `filename` в query → 400, чужое или неизвестное `attachment_id` → 404, порядок и идемпотентность `attachment_ids`.
- `test_attachments_ops` (`RUN_DOCKER_TESTS=1`) — приватность bucket и права ключа API, отсутствие оригиналов на репликах, прерванные/зависшие загрузки, компенсация и reconciler, 16 параллельных загрузок 9.5 MiB без OOM у `s3`.
- `test_shutdown` (`RUN_DOCKER_TESTS=1`, по очереди останавливает и возвращает одну API-реплику) — graceful shutdown: после SIGTERM `/health/ready` сразу 503, а обычные запросы обслуживаются всё окно drain (пробы изнутри реплики, процесс API — PID 1); `docker stop` реплики под нагрузкой через LB — 0 ошибок; загрузка, тело которой ещё идёт при закрытии listener, завершается 201.
- `test_deps` (`RUN_DEPS_TEST=1`, останавливает зависимости!) — stop Redis, stop/pause PostgreSQL, stop S3: JSON 503 за ограниченное время и восстановление; хук `scripts/restart.*` действительно перезапускает API, PostgreSQL, Redis и S3.
- `test_idempotency`, `test_sessions`, `test_concurrency`, `test_api` — перенесены из lab3 и дополнены (область ключа, формат ключа, текст с U+0000).
Для проверки истечения сессии поднимите API с `SESSION_TTL_SECONDS=2` и повторите `test_sessions`.
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

@@ -0,0 +1,17 @@
"""Regenerate synthetic JPEG fixtures (needs Pillow; run once in a throwaway container):
docker run --rm -v "$PWD/student-tests/fixtures:/out" python:3.12.8-slim-bookworm \
sh -c "pip install -q pillow==11.0.0 && python /out/make_jpeg_fixtures.py"
"""
from PIL import Image
im = Image.new("RGB", (320, 200))
px = im.load()
for y in range(200):
for x in range(320):
px[x, y] = (x * 255 // 319, y * 255 // 199, (x + y) % 256)
im.save("/out/sample.jpg", "JPEG", quality=85)
im.save("/out/progressive.jpg", "JPEG", quality=80, progressive=True)
im.convert("L").save("/out/gray.jpg", "JPEG", quality=80)
print("ok")
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.3 KiB

+107
View File
@@ -0,0 +1,107 @@
"""Minimal HTTP helpers for student tests (stdlib only)."""
from __future__ import annotations
import base64
import json
import os
import time
import urllib.error
import urllib.request
import uuid
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
def call(method, path, expected=None, body=None, token=None, headers=None, raw=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
h["Content-Type"] = "application/json"
# Lab3+: POST messages require Idempotency-Key unless caller supplies headers.
if (
method == "POST"
and path.rstrip("/").endswith("/messages")
and (headers is None or "Idempotency-Key" not in headers)
):
h["Idempotency-Key"] = uuid.uuid4().hex
h.update(headers or {})
req = urllib.request.Request(BASE_URL + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=60)
except urllib.error.HTTPError as err:
resp = err
data = resp.read()
status = resp.code
if expected is not None:
assert status == expected, (
f"{method} {path}: expected {expected}, got {status}: {data[:200]!r}"
)
payload = None
ctype = resp.headers.get_content_type()
if ctype == "application/json" and data:
payload = json.loads(data.decode("utf-8"))
assert resp.headers.get("X-Request-Id"), "missing X-Request-Id"
assert resp.headers.get("X-Instance-Id"), "missing X-Instance-Id"
return {"status": status, "headers": resp.headers, "json": payload, "bytes": data}
def upload(token, chat_id, data, filename="photo.png", content_type="image/png", expected=None, headers=None):
from urllib.parse import quote
h = {} if content_type is None else {"Content-Type": content_type}
h.update(headers or {})
query = "" if filename is None else "?filename=" + quote(filename, safe="")
resp = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, token=token, raw=data, headers=h)
if expected is None or expected < 500:
assert resp["status"] < 500, f"user error became {resp['status']}: {resp['json']}"
return resp
def wait_ready(token, att, wait_seconds=60):
deadline = time.monotonic() + wait_seconds
current = att
while time.monotonic() < deadline:
current = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=token)["json"]
assert current["status"] != "failed", current
if current["status"] == "ready":
return current
time.sleep(0.3)
raise AssertionError(f"attachment {att['id']} did not become ready")
def chat_for(*actors, title="attachments"):
return call(
"POST",
"/api/v1/chats",
201,
token=actors[0]["token"],
body={"title": title, "member_ids": [a["id"] for a in actors]},
)["json"]
def register(label: str):
username = f"st_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
body = {
"username": username,
"display_name": f"Student {label}",
"password": password,
}
user = call("POST", "/api/v1/users", 201, body=body)["json"]
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
session = call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + credential},
)["json"]
return {
"id": user["id"],
"user": user,
"token": session["token"],
"registration": body,
}
+172
View File
@@ -0,0 +1,172 @@
"""Synthetic PNG generation (stdlib only) for student tests and experiments."""
from __future__ import annotations
import os
import struct
import zlib
from pathlib import Path
FIXTURES = Path(__file__).resolve().parent / "fixtures"
PUBLIC_PNG = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "sample.png"
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
def png_from_raw(width: int, height: int, raw: bytes, *, depth: int = 8, color: int = 2,
interlace: int = 0, level: int = 6, extra: bytes = b"", plte: bytes = b"") -> bytes:
ihdr = struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)
body = chunk(b"IHDR", ihdr) + extra
if plte:
body += chunk(b"PLTE", plte)
return b"\x89PNG\r\n\x1a\n" + body + chunk(b"IDAT", zlib.compress(raw, level)) + chunk(b"IEND", b"")
def rgb_png(width: int, height: int, *, noise: bool = False, level: int = 6) -> bytes:
if noise:
rows = [b"\x00" + os.urandom(width * 3) for _ in range(height)]
else:
rows = [b"\x00" + bytes(((x + y) * 7) % 256 for x in range(width * 3)) for y in range(height)]
return png_from_raw(width, height, b"".join(rows), level=level)
def interlaced_rgb_png(width: int, height: int) -> bytes:
def pixel(x: int, y: int) -> bytes:
return bytes(((x * 31) % 256, (y * 17) % 256, ((x + y) * 5) % 256))
raw = bytearray()
for x0, y0, dx, dy in ADAM7:
xs = list(range(x0, width, dx))
ys = list(range(y0, height, dy))
if not xs or not ys:
continue
for y in ys:
raw += b"\x00" + b"".join(pixel(x, y) for x in xs)
return png_from_raw(width, height, bytes(raw), interlace=1)
def gray1_png(width: int, height: int) -> bytes:
"""1-bit grayscale: huge pixel counts in a tiny, perfectly valid file."""
row = b"\x00" + b"\x55" * ((width + 7) // 8)
return png_from_raw(width, height, row * height, depth=1, color=0, level=9)
def palette_png(width: int, height: int) -> bytes:
raw = b"".join(b"\x00" + bytes((x + y) % 4 for x in range(width)) for y in range(height))
return png_from_raw(width, height, raw, depth=8, color=3, plte=bytes(range(12)))
def png16_rgba(width: int, height: int) -> bytes:
raw = b"".join(b"\x00" + bytes((x * 3 + y) % 256 for x in range(width * 8)) for y in range(height))
return png_from_raw(width, height, raw, depth=16, color=6)
def large_png(target_bytes: int) -> bytes:
"""Incompressible RGB PNG of roughly target_bytes (stored with zlib level 0)."""
width = 1024
row = 1 + width * 3
height = max(1, target_bytes // row)
return rgb_png(width, height, noise=True, level=0)
def exact_size_png(total_bytes: int) -> bytes:
"""Valid PNG of exactly total_bytes: large_png padded with a tEXt chunk before IEND."""
base = large_png(total_bytes - 64 * 1024)
pad = total_bytes - len(base) - 12 - len(b"Comment\x00")
assert pad >= 0
text = chunk(b"tEXt", b"Comment\x00" + b"x" * pad)
data = base[:-12] + text + base[-12:]
assert len(data) == total_bytes
return data
def unknown_critical_chunk_png() -> bytes:
return png_from_raw(2, 2, b"\x00" + b"\x10" * 6 + b"\x00" + b"\x20" * 6, extra=chunk(b"ABCD", b"x"))
def fixture(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def all_filters_png(width: int, height: int, *, interlace: int = 0) -> bytes:
"""Valid PNG whose scanlines cycle through filter types 0..4 (any bytes unfilter fine)."""
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
raw = bytearray()
row_no = 0
for x0, y0, dx, dy in passes:
pw = len(range(x0, width, dx))
ph = len(range(y0, height, dy))
if not pw or not ph:
continue
for _ in range(ph):
raw.append(row_no % 5)
raw += bytes((row_no * 13 + i) % 256 for i in range(pw * 3))
row_no += 1
return png_from_raw(width, height, bytes(raw), interlace=interlace)
def _with_bad_idat(png: bytes) -> bytes:
"""Same PNG, IDAT payload damaged in the middle, CRC recomputed (valid chunk framing)."""
i = png.index(b"IDAT")
length = struct.unpack(">I", png[i - 4:i])[0]
data = bytearray(png[i + 4:i + 4 + length])
data[len(data) // 2] ^= 0xFF
data[len(data) // 2 + 1] ^= 0x55
return png[:i - 4] + chunk(b"IDAT", bytes(data)) + chunk(b"IEND", b"")
def _idat_not_zlib(png: bytes) -> bytes:
"""Valid PNG chunk framing whose IDAT payload is not a zlib stream."""
i = png.index(b"IDAT")
return png[: i - 4] + chunk(b"IDAT", b"not-a-zlib-stream") + chunk(b"IEND", b"")
def corrupted_pngs() -> dict:
"""Inputs used by reviewers with an independent decoder: all must be 415."""
public = PUBLIC_PNG.read_bytes()
raw = bytearray()
for x0, y0, dx, dy in ADAM7:
pw, ph = len(range(x0, 9, dx)), len(range(y0, 9, dy))
for _ in range(ph if pw else 0):
raw += b"\x00" + bytes(pw * 3)
raw[-(1 + 9 * 3)] = 7 # last scanline of the last Adam7 pass
return {
"first_33_bytes_of_sample_png": public[:33],
"sample_png_truncated_half": public[: len(public) // 2],
"sample_png_truncated_minus_one": public[:-1],
"broken_idat_stream_valid_crc": _with_bad_idat(rgb_png(64, 64)),
"idat_not_zlib": _idat_not_zlib(rgb_png(32, 24)),
"filter_byte_05": png_from_raw(1, 1, b"\x05\x00\x00\x00"),
"filter_byte_ff": png_from_raw(1, 1, b"\xff\x00\x00\x00"),
"interlaced_filter_byte_07": png_from_raw(9, 9, bytes(raw), interlace=1),
"plte_257_entries": png_from_raw(1, 1, b"\x00\x00", color=3, plte=bytes(771)),
"plte_not_multiple_of_3": png_from_raw(1, 1, b"\x00\x00", color=3, plte=bytes(4)),
"plte_too_long_for_1bit": png_from_raw(1, 1, b"\x00\x00", depth=1, color=3, plte=bytes(9)),
"plte_in_grayscale": png_from_raw(1, 1, b"\x00\x00", depth=8, color=0, plte=bytes(3)),
"unknown_critical_chunk": unknown_critical_chunk_png(),
}
def corrupted_jpegs() -> dict:
sample = fixture("sample.jpg")
progressive = fixture("progressive.jpg")
sos = sample.index(b"\xff\xda")
sos_len = struct.unpack(">H", sample[sos + 2:sos + 4])[0]
twelve_bit = bytearray(sample)
twelve_bit[sample.index(b"\xff\xc0") + 4] = 12
# Valid shell, but SOS component id (byte after Ns) not present in SOF → 415.
sos_bad_comp = bytearray(sample)
sos_bad_comp[sos + 5] = 0x7F
return {
"jpeg_27_bytes_no_tables": bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"),
"jpeg_18_bytes": bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"),
"jpeg_no_data_after_sos": sample[: sos + 2 + sos_len] + b"\xff\xd9",
"jpeg_half_plus_eoi": sample[: len(sample) // 2] + b"\xff\xd9",
"progressive_half_plus_eoi": progressive[: len(progressive) // 2] + b"\xff\xd9",
"jpeg_truncated_no_eoi": sample[:-200],
"jpeg_12_bit": bytes(twelve_bit),
"jpeg_sos_unknown_component": bytes(sos_bad_comp),
}
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Run all student tests against a live API (BASE_URL)."""
from __future__ import annotations
import importlib
import sys
import traceback
MODULES = [
"test_images_unit",
"test_api",
"test_concurrency",
"test_sessions",
"test_idempotency",
"test_attachments",
"test_attachments_ops",
"test_shutdown",
"test_deps",
]
def main() -> int:
failed = 0
passed = 0
only = set(sys.argv[1:])
for name in MODULES:
if only and name not in only:
continue
mod = importlib.import_module(name)
tests = [getattr(mod, n) for n in dir(mod) if n.startswith("test_")]
for fn in tests:
label = f"{name}.{fn.__name__}"
try:
fn()
print("OK", label)
passed += 1
except Exception as exc:
failed += 1
print("FAIL", label, ":", exc)
traceback.print_exc()
print(f"Summary: {passed} passed, {failed} failed")
return 1 if failed else 0
if __name__ == "__main__":
sys.path.insert(0, ".")
raise SystemExit(main())
+185
View File
@@ -0,0 +1,185 @@
"""Grade 3/4 API checks: auth, conflict, X-User-Id ignored, logout."""
from __future__ import annotations
import base64
import time
import uuid
from helpers import call, register
def test_username_conflict():
a = register("conflict")
body = dict(a["registration"])
call("POST", "/api/v1/users", 409, body=body)
def test_wrong_password():
a = register("wrongpw")
bad = base64.b64encode(
(a["registration"]["username"] + ":wrong-password-xx").encode()
).decode()
resp = call(
"POST",
"/api/v1/auth/sessions",
401,
headers={"Authorization": "Basic " + bad},
)
assert resp["headers"].get("WWW-Authenticate", "").lower().startswith("basic")
def test_x_user_id_ignored():
a = register("xu")
b = register("xv")
me = call(
"GET",
"/api/v1/users/me",
200,
token=a["token"],
headers={"X-User-Id": b["id"]},
)["json"]
assert me["id"] == a["id"]
def test_basic_on_business_rejected():
a = register("basicbiz")
cred = base64.b64encode(
(a["registration"]["username"] + ":" + a["registration"]["password"]).encode()
).decode()
call(
"GET",
"/api/v1/users/me",
401,
headers={"Authorization": "Basic " + cred},
)
def test_logout_revokes():
a = register("logout")
call("DELETE", "/api/v1/auth/sessions/current", 204, token=a["token"])
call("GET", "/api/v1/users/me", 401, token=a["token"])
def test_chat_requires_caller_member():
a = register("owner")
b = register("peer")
call(
"POST",
"/api/v1/chats",
400,
token=a["token"],
body={"title": "no me", "member_ids": [b["id"]]},
)
def test_unknown_member_rolls_back():
a = register("atom")
fake = str(uuid.uuid4())
call(
"POST",
"/api/v1/chats",
400,
token=a["token"],
body={"title": "bad", "member_ids": [a["id"], fake]},
)
def test_whitespace_only_password_rejected():
# Maintainer remark: required strings must not be whitespace-only.
body = {
"username": "ws_" + uuid.uuid4().hex[:10],
"display_name": "Spaces",
"password": " " * 12,
}
call("POST", "/api/v1/users", 400, body=body)
def test_username_trailing_newline_400():
# The pattern must match the whole value: "name\n" is not a valid username.
for username in ("review123\n", "review123\r\n", "\nreview123"):
resp = call("POST", "/api/v1/users", 400,
body={"username": username, "display_name": "x", "password": "T3st-" + uuid.uuid4().hex})
assert resp["json"]["error"]["code"] == "invalid_request", repr(username)
def test_malformed_cursor_400():
a = register("cursor")
chat = call("POST", "/api/v1/chats", 201, token=a["token"], body={"title": "c", "member_ids": [a["id"]]})["json"]
def b64(raw: str) -> str:
return base64.urlsafe_b64encode(raw.encode()).decode()
bad = ["not-base64!!", b64("messages|not-a-date|" + str(uuid.uuid4())),
b64("messages|2026-01-01T00:00:00Z|not-a-uuid"), b64("chats|2026-01-01T00:00:00Z|" + str(uuid.uuid4()))]
for cursor in bad:
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, token=a["token"])
for cursor in bad[:3]:
call("GET", f"/api/v1/chats?cursor={cursor}", 400, token=a["token"])
def test_nul_in_text_roundtrip():
# U+0000 is valid message text: exact in the response, history and idempotent replay.
a = register("nul")
peer = register("nulb")
chat = call(
"POST",
"/api/v1/chats",
201,
token=a["token"],
body={"title": "nul-chat", "member_ids": [a["id"], peer["id"]]},
)["json"]
text = "abc\u0000def \\0 \\u0000"
key = {"Idempotency-Key": "nul-" + uuid.uuid4().hex}
path = f"/api/v1/chats/{chat['id']}/messages"
first = call("POST", path, 201, token=a["token"], body={"text": text}, headers=key)["json"]
assert first["text"] == text
again = call("POST", path, 201, token=a["token"], body={"text": text}, headers=key)["json"]
assert again == first
items = call("GET", path, 200, token=peer["token"])["json"]["items"]
assert [m["text"] for m in items if m["id"] == first["id"]] == [text]
def test_login_rejects_json_body():
a = register("bodylogin")
cred = base64.b64encode(
(a["registration"]["username"] + ":" + a["registration"]["password"]).encode()
).decode()
call(
"POST",
"/api/v1/auth/sessions",
400,
body={"unexpected": True},
headers={"Authorization": "Basic " + cred},
)
def test_invalid_basic_encoding_is_401():
# Maintainer remark: malformed Basic must be 401, not 500.
call(
"POST",
"/api/v1/auth/sessions",
401,
headers={"Authorization": "Basic " + base64.b64encode("é".encode()).decode()},
)
def test_unsupported_media_type():
a = register("ctype")
peer = register("ctypeb")
chat = call(
"POST",
"/api/v1/chats",
201,
token=a["token"],
body={"title": "ctype", "member_ids": [a["id"], peer["id"]]},
)["json"]
raw = b'{"text":"x"}'
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
415,
token=a["token"],
raw=raw,
headers={"Content-Type": "text/plain"},
)
+368
View File
@@ -0,0 +1,368 @@
"""Lab4 grade 3/4: attachment upload/download, ACL, limits, MIME sniffing, filenames."""
from __future__ import annotations
import concurrent.futures
import hashlib
import http.client
import socket
import time
import urllib.parse
import uuid
import imagegen
from helpers import BASE_URL, call, chat_for, register, upload, wait_ready
MIB = 1024 * 1024
LIMIT = 10 * MIB
def _trio():
alice, bob, eve = register("att_a"), register("att_b"), register("att_e")
return alice, bob, eve, chat_for(alice, bob)
def _png(seed: int = 0) -> bytes:
return imagegen.rgb_png(16 + seed % 7, 9 + seed % 5)
def _download(token, att_id, expected=200, query=""):
return call("GET", f"/api/v1/attachments/{att_id}/content{query}", expected, token=token)
def _message(actor, chat, text, attachment_ids=None, expected=201, key=None):
body = {"text": text}
if attachment_ids is not None:
body["attachment_ids"] = attachment_ids
return call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
expected,
token=actor["token"],
body=body,
headers={"Idempotency-Key": key or uuid.uuid4().hex},
)
def test_upload_roundtrip_png_and_jpeg_variants():
alice, bob, _eve, chat = _trio()
cases = [
(imagegen.PUBLIC_PNG.read_bytes(), "image/png", "sample.png"),
(imagegen.fixture("sample.jpg"), "image/jpeg", "sample.jpg"),
(imagegen.fixture("progressive.jpg"), "image/jpeg", "progressive.jpg"),
(imagegen.fixture("gray.jpg"), "image/jpeg", "gray.jpg"),
(imagegen.interlaced_rgb_png(41, 29), "image/png", "adam7.png"),
(imagegen.palette_png(12, 7), "image/png", "palette.png"),
]
for data, ctype, name in cases:
resp = upload(alice["token"], chat["id"], data, name, ctype, 202)
att = resp["json"]
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, att
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
assert att["content_type"] == ctype and att["filename"] == name
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
assert set(att) == {"id", "chat_id", "owner_id", "filename", "content_type", "size_bytes",
"sha256", "status", "variants", "error_code", "created_at", "updated_at"}
assert resp["headers"].get("Location") == "/api/v1/attachments/" + att["id"]
_download(bob["token"], att["id"], expected=409)
ready = wait_ready(bob["token"], att)
assert ready["status"] == "ready" and ready["error_code"] is None
names = {v["name"] for v in ready["variants"]}
assert {"thumbnail", "crop", "optimized"} <= names, ready["variants"]
dl = _download(bob["token"], att["id"])
assert dl["bytes"] == data, name
assert dl["headers"].get_content_type() == ctype
assert dl["headers"].get("Content-Length") == str(len(data))
assert dl["headers"].get("X-Content-Type-Options") == "nosniff"
assert dl["headers"].get("ETag") == '"' + att["sha256"] + '"'
def test_outsider_and_unknown_get_404():
alice, _bob, eve, chat = _trio()
att = upload(alice["token"], chat["id"], _png(), "secret.png", "image/png", 202)["json"]
for suffix in ("", "/content", "/content?variant=original"):
resp = call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, token=eve["token"])
assert resp["json"]["error"]["code"] == "not_found"
unknown = str(uuid.uuid4())
call("GET", "/api/v1/attachments/" + unknown, 404, token=alice["token"])
call("GET", "/api/v1/attachments/" + unknown + "/content", 404, token=alice["token"])
call("GET", "/api/v1/attachments/not-a-uuid", 400, token=alice["token"])
call("GET", "/api/v1/attachments/" + att["id"], 401)
call("GET", "/api/v1/attachments/" + att["id"] + "/content", 401)
# Outsider cannot upload into the chat either.
upload(eve["token"], chat["id"], _png(), "x.png", "image/png", 404)
def test_upload_auth_and_chat_checks():
alice, _bob, _eve, chat = _trio()
upload(None, chat["id"], _png(), "x.png", "image/png", 401)
upload(alice["token"], str(uuid.uuid4()), _png(), "x.png", "image/png", 404)
upload(alice["token"], "not-a-uuid", _png(), "x.png", "image/png", 400)
def test_message_with_attachments_and_history():
alice, bob, _eve, chat = _trio()
a1 = upload(alice["token"], chat["id"], _png(1), "one.png", "image/png", 202)["json"]
a2 = upload(bob["token"], chat["id"], _png(2), "two.png", "image/png", 202)["json"]
key = uuid.uuid4().hex
msg = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"]
assert msg["attachment_ids"] == [a2["id"], a1["id"]] and msg["text"] == ""
replay = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"]
assert replay == msg
_message(bob, chat, "", [a1["id"], a2["id"]], expected=409, key=key)
plain_key = uuid.uuid4().hex
plain = _message(alice, chat, "без вложений", key=plain_key)["json"]
assert plain["attachment_ids"] == []
# Absent attachment_ids ≡ []: same key replays the same message, another set conflicts.
assert _message(alice, chat, "без вложений", [], key=plain_key)["json"] == plain
_message(alice, chat, "без вложений", [a1["id"]], expected=409, key=plain_key)
# Any member may reference any attachment of the chat, also repeatedly.
again = _message(alice, chat, "ещё раз", [a2["id"]])["json"]
assert again["attachment_ids"] == [a2["id"]]
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=alice["token"])["json"]
by_id = {m["id"]: m for m in page["items"]}
assert by_id[msg["id"]]["attachment_ids"] == [a2["id"], a1["id"]]
assert by_id[plain["id"]]["attachment_ids"] == []
assert by_id[again["id"]]["attachment_ids"] == [a2["id"]]
def test_foreign_and_unknown_attachment_ids_404():
alice, bob, eve, chat = _trio()
other = chat_for(alice, eve, title="other chat")
foreign = upload(alice["token"], other["id"], _png(3), "other.png", "image/png", 202)["json"]
own = upload(alice["token"], chat["id"], _png(4), "own.png", "image/png", 202)["json"]
# File of another chat (even uploaded by the same user) cannot be attached.
resp = _message(alice, chat, "чужой файл", [foreign["id"]], expected=404)
assert resp["json"]["error"]["code"] == "not_found"
_message(alice, chat, "смесь", [own["id"], foreign["id"]], expected=404)
_message(bob, chat, "неизвестный", [str(uuid.uuid4())], expected=404)
# Eve is in `other` only: cannot attach chat's file there.
_message(eve, other, "украсть", [own["id"]], expected=404)
# Failed attempts did not create messages.
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])["json"]
assert page["items"] == []
def test_attachment_ids_validation_400():
alice, _bob, _eve, chat = _trio()
att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 202)["json"]
many = [str(uuid.uuid4()) for _ in range(11)]
for bad in (many, [att["id"], att["id"]], ["not-a-uuid"], [123], "x", {"id": att["id"]}):
_message(alice, chat, "x", bad, expected=400)
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
token=alice["token"],
body={"text": "x", "attachment_ids": None},
)
_message(alice, chat, "", [], expected=400)
_message(alice, chat, " ", None, expected=400)
_message(alice, chat, " ", [att["id"]])
def test_too_large_413():
alice, _bob, _eve, chat = _trio()
big = imagegen.large_png(LIMIT + 64 * 1024)
assert len(big) > LIMIT
resp = upload(alice["token"], chat["id"], big, "big.png", "image/png", 413)
assert resp["json"]["error"]["code"] == "payload_too_large"
# Same oversized body without Content-Length (chunked): detected while streaming.
parsed = urllib.parse.urlparse(BASE_URL)
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=60)
def body():
for i in range(0, len(big), 256 * 1024):
yield big[i:i + 256 * 1024]
conn.request(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=big.png",
body=body(),
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"},
encode_chunked=True,
)
r = conn.getresponse()
r.read()
conn.close()
assert r.status == 413, r.status
def test_near_limit_real_png_accepted():
alice, bob, _eve, chat = _trio()
data = imagegen.large_png(LIMIT - 256 * 1024)
assert len(data) <= LIMIT
att = upload(alice["token"], chat["id"], data, "near-limit.png", "image/png", 202)["json"]
wait_ready(bob["token"], att)
dl = _download(bob["token"], att["id"])
assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"] == hashlib.sha256(data).hexdigest()
def test_exact_limit_png_through_lb_accepted():
# 10 485 760 bytes exactly passes the LB and the API; one more byte is JSON 413 from the API.
alice, bob, _eve, chat = _trio()
data = imagegen.exact_size_png(LIMIT)
att = upload(alice["token"], chat["id"], data, "exact.png", "image/png", 202)["json"]
assert att["size_bytes"] == LIMIT
wait_ready(bob["token"], att)
assert hashlib.sha256(_download(bob["token"], att["id"])["bytes"]).hexdigest() == hashlib.sha256(data).hexdigest()
resp = upload(alice["token"], chat["id"], imagegen.exact_size_png(LIMIT + 1), "plus1.png", "image/png", 413)
assert resp["json"]["error"]["code"] == "payload_too_large"
def test_too_many_pixels_413():
alice, _bob, _eve, chat = _trio()
bomb = imagegen.gray1_png(8000, 5001) # 40 008 000 px in a few KiB
assert len(bomb) < 64 * 1024
resp = upload(alice["token"], chat["id"], bomb, "bomb.png", "image/png", 413)
assert resp["json"]["error"]["code"] == "payload_too_large"
upload(alice["token"], chat["id"], imagegen.gray1_png(8000, 5000), "limit.png", "image/png", 202)
def test_corrupted_images_415():
alice, _bob, _eve, chat = _trio()
png = _png(5)
flipped = bytearray(png)
flipped[-20] ^= 0xFF
jpeg = imagegen.fixture("sample.jpg")
for data, ctype in (
(png[: len(png) // 2], "image/png"),
(bytes(flipped), "image/png"),
(png + b"<script>", "image/png"),
(jpeg[: len(jpeg) - 200], "image/jpeg"),
(b"\xff\xd8\xff\xe0" + bytes(range(256)) * 8, "image/jpeg"),
(b"\x89PNG\r\n\x1a\n" + b"\x00" * 64, "image/png"),
):
resp = upload(alice["token"], chat["id"], data, "broken", ctype, 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type"
def test_reviewer_corrupted_images_415():
# Same inputs reviewers verify with an independent decoder; valid edge cases stay 201.
alice, bob, _eve, chat = _trio()
for name, data in imagegen.corrupted_pngs().items():
resp = upload(alice["token"], chat["id"], data, name + ".png", "image/png", 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type", name
for name, data in imagegen.corrupted_jpegs().items():
resp = upload(alice["token"], chat["id"], data, name + ".jpg", "image/jpeg", 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type", name
for data, ctype in ((imagegen.all_filters_png(33, 20, interlace=1), "image/png"),
(imagegen.fixture("progressive.jpg") + b"trailing", "image/jpeg")):
att = upload(alice["token"], chat["id"], data, "edge", ctype, 202)["json"]
wait_ready(bob["token"], att)
assert _download(bob["token"], att["id"])["bytes"] == data
page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])["json"]
assert page["items"] == []
def test_mime_spoofing_415():
alice, _bob, _eve, chat = _trio()
png = _png(6)
jpeg = imagegen.fixture("sample.jpg")
cases = [
(b"GIF89a\x01\x00\x01\x00\x00\x00\x00;", "image/png"),
(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n", "image/jpeg"),
(b"<html><script>alert(1)</script></html>", "image/png"),
(b"MZ\x90\x00\x03\x00\x00\x00", "image/jpeg"),
(jpeg, "image/png"),
(png, "image/jpeg"),
(png, "text/plain"),
(png, "image/gif"),
(png, "multipart/form-data; boundary=x"),
(png, "application/octet-stream"),
]
for data, ctype in cases:
resp = upload(alice["token"], chat["id"], data, "spoof.png", ctype, 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type", ctype
def test_dangerous_filenames_400_and_unicode_kept():
alice, bob, _eve, chat = _trio()
png = _png(7)
for name in ("../etc/passwd", "../x.png", "a/b.png", "a\\b.png", "x\x00.png", "x\n.png", "x\r.png",
"del\x7f.png", "c1\x85.png",
"tab\t.png", "\u202egnp.exe", "zero\u200bwidth.png", "", " ", ".", "..", "a" * 256):
resp = upload(alice["token"], chat["id"], png, name, "image/png", 400)
assert resp["json"]["error"]["code"] == "invalid_request", repr(name)
upload(alice["token"], chat["id"], png, None, "image/png", 400)
for name in ("фото отпуска 😀.png", "b" * 255, "with space & ampersand;=.png"):
att = upload(alice["token"], chat["id"], png, name, "image/png", 202)["json"]
assert att["filename"] == name
call("GET", "/api/v1/attachments/" + att["id"], 200, token=bob["token"])
def test_filename_query_decoded_strictly_400():
# Invalid UTF-8 must not be "sanitized" into U+FFFD; an ambiguous repeated filename is rejected too.
alice, _bob, _eve, chat = _trio()
png = _png(10)
path = f"/api/v1/chats/{chat['id']}/attachments"
for query in ("?filename=%FF%FE.png", "?filename=%C3.png", "?filename=%ED%A0%80.png",
"?filename=ok.png&filename=a%2Fb", "?filename=a.png&filename=b.png"):
resp = call("POST", path + query, 400, token=alice["token"], raw=png, headers={"Content-Type": "image/png"})
assert resp["json"]["error"]["code"] == "invalid_request", query
att = call("POST", path + "?filename=%D1%84%2B1+2.png", 202, token=alice["token"], raw=png,
headers={"Content-Type": "image/png"})["json"]
assert att["filename"] == "ф+1 2.png"
def test_same_filename_distinct_objects():
alice, bob, _eve, chat = _trio()
first, second = _png(8), _png(9)
a = upload(alice["token"], chat["id"], first, "same.png", "image/png", 202)["json"]
b = upload(alice["token"], chat["id"], second, "same.png", "image/png", 202)["json"]
assert a["id"] != b["id"]
wait_ready(bob["token"], a)
wait_ready(bob["token"], b)
assert _download(bob["token"], a["id"])["bytes"] == first
assert _download(bob["token"], b["id"])["bytes"] == second
def test_empty_body_and_variants():
alice, bob, _eve, chat = _trio()
upload(alice["token"], chat["id"], b"", "empty.png", "image/png", 400)
att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 202)["json"]
_download(bob["token"], att["id"], 409, "?variant=original")
_download(bob["token"], att["id"], 409, "?variant=thumbnail")
ready = wait_ready(bob["token"], att)
assert {v["name"] for v in ready["variants"]} >= {"thumbnail", "crop", "optimized"}
_download(bob["token"], att["id"], 200, "?variant=original")
_download(bob["token"], att["id"], 200, "?variant=thumbnail")
_download(bob["token"], att["id"], 400, "?variant=../../etc")
def test_parallel_uploads_distinct():
alice, bob, _eve, chat = _trio()
blobs = [_png(i) + b"" for i in range(12)]
def send(i):
return upload(alice["token"], chat["id"], blobs[i], f"p{i}.png", "image/png", 202)["json"]
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
results = list(pool.map(send, range(12)))
assert len({r["id"] for r in results}) == 12
for i, att in enumerate(results):
assert att["sha256"] == hashlib.sha256(blobs[i]).hexdigest()
def _raw_partial_upload(token: str, chat_id: str, declared: int, payload: bytes, hold: float = 0.0) -> None:
parsed = urllib.parse.urlparse(BASE_URL)
sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=30)
head = (
f"POST /api/v1/chats/{chat_id}/attachments?filename=cut.png HTTP/1.1\r\n"
f"Host: {parsed.hostname}\r\nAuthorization: Bearer {token}\r\n"
f"Content-Type: image/png\r\nContent-Length: {declared}\r\n\r\n"
).encode()
sock.sendall(head + payload)
if hold:
time.sleep(hold)
sock.close()
def test_interrupted_upload_does_not_break_service():
alice, bob, _eve, chat = _trio()
big = imagegen.large_png(2 * MIB)
for _ in range(3):
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 300 * 1024])
att = upload(alice["token"], chat["id"], _png(), "after.png", "image/png", 202)["json"]
wait_ready(bob["token"], att)
_download(bob["token"], att["id"])
+163
View File
@@ -0,0 +1,163 @@
"""Lab4 grade 4 operator checks that need the Docker CLI (RUN_DOCKER_TESTS=1).
Non-destructive: nothing is stopped; checks run inside the API containers.
"""
from __future__ import annotations
import concurrent.futures
import json
import os
import subprocess
import threading
import time
import imagegen
from helpers import wait_ready, call, chat_for, register, upload
from test_attachments import _raw_partial_upload
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
def _enabled(name: str) -> bool:
if os.environ.get("RUN_DOCKER_TESTS") != "1":
print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)")
return False
return True
def _run(args, check=True):
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}")
return proc
def _api_containers():
out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split()
assert len(out) >= 2, out
return out
def _exec_json(container: str, *args):
proc = _run(["docker", "exec", container, "python", "-m", *args])
return json.loads(proc.stdout[proc.stdout.index("{"):])
def _bucket_state(container: str) -> dict:
return _exec_json(container, "messenger.reconcile", "--dry-run", "--grace-seconds", "0")
def test_bucket_private_and_api_key_least_privilege():
if not _enabled("test_bucket_private_and_api_key_least_privilege"):
return
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "s3")
failed = [r for r in report["results"] if not r["ok"]]
assert report["passed"] and not failed, failed
def test_s3_db_failure_compensation_and_reconcile():
if not _enabled("test_s3_db_failure_compensation_and_reconcile"):
return
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "consistency")
failed = [r for r in report["results"] if not r["ok"]]
assert report["passed"] and not failed, failed
def test_interrupted_and_stalled_uploads_leave_nothing():
if not _enabled("test_interrupted_and_stalled_uploads_leave_nothing"):
return
containers = _api_containers()
alice, bob = register("cut_a"), register("cut_b")
chat = chat_for(alice, bob)
before = _bucket_state(containers[0])
big = imagegen.large_png(3 * 1024 * 1024)
for _ in range(4):
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 512 * 1024])
# Stalled client: headers + part of body, then silence past UPLOAD_IDLE_TIMEOUT_SECONDS (10 s).
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 256 * 1024], hold=12)
time.sleep(1)
after = _bucket_state(containers[0])
assert after["objects_scanned"] == before["objects_scanned"], (before, after)
# Leftovers of earlier S3-outage runs (intents kept for the reconciler) are not ours.
assert after["stale_intents"] == before["stale_intents"], (before, after)
assert after["orphans_found"] == before["orphans_found"], (before, after)
for container in containers:
spool = _exec_json(container, "messenger.selfcheck", "spool")
assert spool["passed"], (container, spool)
att = upload(alice["token"], chat["id"], imagegen.rgb_png(8, 8), "ok.png", "image/png", 202)["json"]
wait_ready(bob["token"], att)
call("GET", "/api/v1/attachments/" + att["id"] + "/content", 200, token=bob["token"])
def test_parallel_large_uploads_keep_s3_within_memory():
# 2 replicas × 8 slots = 16 PUTs in flight; SeaweedFS holds ≈30–40 MiB per 10 MiB PUT.
# With mem_limit 512m it ran at the limit and was OOM-killed (exit 137) under 40 uploads.
if not _enabled("test_parallel_large_uploads_keep_s3_within_memory"):
return
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
limit = int(_run(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", s3]).stdout.strip())
alice = register("par_a")
chat = chat_for(alice)
payload = imagegen.large_png(int(9.5 * 1024 * 1024))
peak = [0]
stop = threading.Event()
def sample():
while not stop.is_set():
stat = _run(["docker", "exec", s3, "cat", "/sys/fs/cgroup/memory.stat"], check=False).stdout
for line in stat.splitlines():
if line.startswith("anon "):
peak[0] = max(peak[0], int(line.split()[1]))
time.sleep(0.1)
def send(i):
return call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=par{i}.png", token=alice["token"],
raw=payload, headers={"Content-Type": "image/png"})["status"]
sampler = threading.Thread(target=sample)
sampler.start()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
statuses = list(pool.map(send, range(16)))
finally:
time.sleep(1)
stop.set()
sampler.join()
state = _run(["docker", "inspect", "-f", "{{.State.Running}} {{.State.OOMKilled}}", s3]).stdout.split()
assert state == ["true", "false"], state
# 503 is the documented backpressure (slot wait > 5 s), never an S3 failure.
assert set(statuses) <= {202, 503} and statuses.count(202) >= 10, statuses
assert peak[0] < 0.8 * limit, (peak[0] // 2**20, limit // 2**20)
call("GET", "/health/ready", 200)
def test_s3_can_grow_volumes():
# SeaweedFS grows a collection by 7 volumes at a time; when no slot is left every PUT
# fails ("No writable volumes") while readiness stays 200. -volume.max=16 hit that at ≈4.7 GiB.
if not _enabled("test_s3_can_grow_volumes"):
return
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
status = json.loads(_run(["docker", "exec", s3, "wget", "-q", "-O", "-", "http://127.0.0.1:9333/dir/status"]).stdout)
topology = status["Topology"]
assert topology["Free"] >= 8, {k: topology[k] for k in ("Max", "Free")}
def test_no_original_kept_on_api_replicas():
if not _enabled("test_no_original_kept_on_api_replicas"):
return
alice, bob = register("loc_a"), register("loc_b")
chat = chat_for(alice, bob)
for i in range(4):
upload(alice["token"], chat["id"], imagegen.rgb_png(10 + i, 10), f"l{i}.png", "image/png", 202)
for container in _api_containers():
spool = _exec_json(container, "messenger.selfcheck", "spool")
assert spool["passed"], (container, spool)
# Any regular file created after the container started, anywhere on its writable layer
# (/etc/hosts, /etc/resolv.conf, /etc/hostname are bind mounts written by Docker itself).
started = _run(["docker", "inspect", "-f", "{{.State.StartedAt}}", container]).stdout.strip()
found = _run(["docker", "exec", "-u", "0", container, "find", "/", "-xdev", "-type", "f",
"-newermt", started, "-not", "-path", "/proc/*",
"-not", "-path", "/etc/hosts", "-not", "-path", "/etc/resolv.conf",
"-not", "-path", "/etc/hostname"]).stdout.split()
assert not found, (container, found[:5])
+66
View File
@@ -0,0 +1,66 @@
"""Concurrent registration race and parallel messages."""
from __future__ import annotations
import concurrent.futures
import uuid
from helpers import BASE_URL, call, register
def test_parallel_messages_unique():
a = register("par")
b = register("parb")
chat = call(
"POST",
"/api/v1/chats",
201,
token=a["token"],
body={"title": "parallel", "member_ids": [a["id"], b["id"]]},
)["json"]
def send(i: int):
return call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": f"msg-{i}"},
)["json"]["id"]
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
ids = list(pool.map(send, range(100)))
assert len(ids) == len(set(ids)) == 100
def test_username_race_one_winner():
username = "race_" + uuid.uuid4().hex[:12]
password = "T3st-" + uuid.uuid4().hex
body = {
"username": username,
"display_name": "Race",
"password": password,
}
def attempt(_):
import json
import urllib.error
import urllib.request
raw = json.dumps(body).encode("utf-8")
req = urllib.request.Request(
BASE_URL + "/api/v1/users",
data=raw,
headers={"Content-Type": "application/json", "Accept": "application/json"},
method="POST",
)
try:
resp = urllib.request.urlopen(req, timeout=30)
except urllib.error.HTTPError as err:
resp = err
return resp.code
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
codes = list(pool.map(attempt, range(8)))
assert codes.count(201) == 1, codes
assert codes.count(409) == 7, codes
+181
View File
@@ -0,0 +1,181 @@
"""Dependency failure checks (Redis/Postgres/S3). Optional: RUN_DEPS_TEST=1."""
from __future__ import annotations
import hashlib
import os
import subprocess
import time
import uuid
import imagegen
from helpers import call, chat_for, register, upload
ROOT = os.path.join(os.path.dirname(__file__), "..")
def _compose(*args: str) -> None:
cmd = ["docker", "compose", "--env-file", ".env", *args]
subprocess.check_call(cmd, cwd=ROOT)
def _started_at(service: str) -> dict:
ids = subprocess.check_output(["docker", "compose", "--env-file", ".env", "ps", "-q", service], cwd=ROOT, text=True).split()
assert ids, service
return {
c: subprocess.check_output(["docker", "inspect", "-f", "{{.State.StartedAt}}", c], text=True).strip()
for c in ids
}
def _wait_ready(timeout_s: int = 60, stable: int = 6) -> None:
# Several consecutive 200s: HAProxy may still finish health checks that started
# during the outage and briefly mark every replica DOWN after the first 200.
deadline = time.time() + timeout_s
streak = 0
while time.time() < deadline:
try:
streak = streak + 1 if call("GET", "/health/ready")["status"] == 200 else 0
except Exception:
streak = 0
if streak >= stable:
return
time.sleep(1)
raise AssertionError("stack did not become ready in time")
def _unavailable(method: str, path: str, token: str | None = None, body=None) -> dict:
# Every 503 — from a replica or from HAProxy itself — is the JSON error envelope
# with X-Request-Id / X-Instance-Id (checked inside call()).
resp = call(method, path, 503, token=token, body=body)
assert resp["json"]["error"]["code"] == "unavailable", resp["json"]
return resp
def _skip(name: str) -> bool:
if os.environ.get("RUN_DEPS_TEST") != "1":
print(f"SKIP {name} (set RUN_DEPS_TEST=1)")
return True
return False
def test_redis_stop_returns_503_or_unready():
if _skip("test_redis_stop_returns_503_or_unready"):
return
actor = register("deps")
other = register("deps_ok")
_compose("stop", "redis")
time.sleep(2)
try:
_unavailable("GET", "/health/ready")
_unavailable("GET", "/api/v1/users/me", actor["token"])
# Logout must still revoke durably while cache is down (204), not suppress
# a failed delete and later trust a resurrected Redis AOF copy.
call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
finally:
_compose("start", "redis")
_wait_ready()
call("GET", "/api/v1/users/me", 401, token=actor["token"])
call("GET", "/api/v1/users/me", 200, token=other["token"])
def test_s3_stop_degrades_only_attachments():
# S3 is on the synchronous attachment path, so readiness reports 503; HAProxy then
# falls back to the liveness-checked backend and text chat/metadata keep working.
if _skip("test_s3_stop_degrades_only_attachments"):
return
alice, bob = register("s3down_a"), register("s3down_b")
chat = chat_for(alice, bob)
data = imagegen.rgb_png(24, 12)
att = upload(alice["token"], chat["id"], data, "before.png", "image/png", 202)["json"]
_compose("stop", "s3")
try:
time.sleep(3)
_unavailable("GET", "/health/ready")
started = time.monotonic()
up = upload(alice["token"], chat["id"], data, "during.png", "image/png", 503)
elapsed = time.monotonic() - started
assert up["json"]["error"]["code"] == "unavailable", up["json"]
assert elapsed < 15, elapsed
_unavailable("GET", f"/api/v1/attachments/{att['id']}/content", bob["token"])
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"])["json"]
assert meta["sha256"] == att["sha256"]
msg = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=bob["token"],
body={"text": "S3 недоступен, но чат работает", "attachment_ids": [att["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)["json"]
assert msg["attachment_ids"] == [att["id"]]
print(" s3 down: ready 503, upload 503 in %.2fs, download 503, metadata/messages OK" % elapsed)
finally:
_compose("start", "s3")
_compose("restart", "s3-gw")
_wait_ready()
wait_ready(bob["token"], att)
dl = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"]
upload(alice["token"], chat["id"], data, "after.png", "image/png", 202)
def test_postgres_stop_returns_503():
if _skip("test_postgres_stop_returns_503"):
return
actor = register("deps_pg")
_compose("stop", "postgres")
time.sleep(2)
try:
_unavailable("GET", "/health/ready")
started = time.monotonic()
_unavailable("GET", "/api/v1/users/me", actor["token"])
_unavailable("GET", "/api/v1/chats", actor["token"])
assert time.monotonic() - started < 15
finally:
_compose("start", "postgres")
_wait_ready(90)
call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_restart_hook_restarts_api_and_stores():
# tests/README: the persistence hook must restart the API and every store.
if _skip("test_restart_hook_restarts_api_and_stores"):
return
actor = register("deps_hook")
services = ("api", "postgres", "redis", "s3")
before = {s: _started_at(s) for s in services}
hook = ["cmd", "/c", r"scripts\restart.cmd"] if os.name == "nt" else ["sh", "scripts/restart.sh"]
subprocess.check_call(hook, cwd=ROOT, stdout=subprocess.DEVNULL)
_wait_ready(90)
for s in services:
after = _started_at(s)
if s == "api":
assert len(after) >= 2, after
stale = [c for c, t in after.items() if before[s].get(c) == t]
assert not stale, (s, stale)
call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_postgres_pause_fails_fast():
# A paused (not stopped) database never answers: no RST, statement_timeout is never
# evaluated. Requests must still end as JSON 503 within a bounded time.
if _skip("test_postgres_pause_fails_fast"):
return
actor = register("deps_pause")
_compose("pause", "postgres")
try:
timings = []
for path in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/users/me"):
started = time.monotonic()
_unavailable("GET", path, actor["token"])
timings.append(time.monotonic() - started)
assert max(timings) < 15, timings
time.sleep(4)
_unavailable("GET", "/health/ready")
print(" postgres paused: 503 after " + ", ".join("%.2fs" % t for t in timings))
finally:
_compose("unpause", "postgres")
_wait_ready(90)
call("GET", "/api/v1/users/me", 200, token=actor["token"])
+165
View File
@@ -0,0 +1,165 @@
"""Lab3: shared Idempotency-Key and multi-replica visibility."""
from __future__ import annotations
import concurrent.futures
import uuid
from helpers import call, register
def _chat_pair():
a = register("idem_a")
b = register("idem_b")
chat = call(
"POST",
"/api/v1/chats",
201,
token=a["token"],
body={"title": "idem", "member_ids": [a["id"], b["id"]]},
)["json"]
return a, b, chat
def test_missing_idempotency_key_400():
a, _b, chat = _chat_pair()
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
token=a["token"],
body={"text": "no-key"},
headers={"Idempotency-Key": ""},
)
def test_replay_same_key_same_body():
a, _b, chat = _chat_pair()
key = uuid.uuid4().hex
first = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": "same"},
headers={"Idempotency-Key": key},
)["json"]
second = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": "same"},
headers={"Idempotency-Key": key},
)["json"]
assert first == second
def test_empty_attachment_ids_equals_absent():
# Peer remark: attachment_ids: [] ≡ absent field; must not 400; same idempotency hash.
a, _b, chat = _chat_pair()
key = uuid.uuid4().hex
first = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": "with-empty-attachments", "attachment_ids": []},
headers={"Idempotency-Key": key},
)["json"]
second = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": "with-empty-attachments"},
headers={"Idempotency-Key": key},
)["json"]
assert first == second
def test_same_key_different_body_409():
a, _b, chat = _chat_pair()
key = uuid.uuid4().hex
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": "one"},
headers={"Idempotency-Key": key},
)
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
409,
token=a["token"],
body={"text": "two"},
headers={"Idempotency-Key": key},
)
def test_key_scoped_to_user_and_chat():
# Bob reusing Alice's key must get his own message, not her stored response.
a, b, chat = _chat_pair()
key = uuid.uuid4().hex
first = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=a["token"],
body={"text": "same"}, headers={"Idempotency-Key": key})["json"]
other = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=b["token"],
body={"text": "same"}, headers={"Idempotency-Key": key})["json"]
assert other["id"] != first["id"] and other["sender_id"] == b["id"]
second_chat = call("POST", "/api/v1/chats", 201, token=a["token"],
body={"title": "idem2", "member_ids": [a["id"], b["id"]]})["json"]
elsewhere = call("POST", f"/api/v1/chats/{second_chat['id']}/messages", 201, token=a["token"],
body={"text": "same"}, headers={"Idempotency-Key": key})["json"]
assert elsewhere["id"] != first["id"] and elsewhere["chat_id"] == second_chat["id"]
def test_invalid_key_format_400():
a, _b, chat = _chat_pair()
for key in ("has space", "x" * 129, "a/b", "a,b", "ключ".encode().decode("latin-1")):
resp = call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=a["token"],
body={"text": "bad key"}, headers={"Idempotency-Key": key})
assert resp["json"]["error"]["code"] == "invalid_request", key
call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=a["token"],
body={"text": "ok key"}, headers={"Idempotency-Key": "A-z0.9_:" + "x" * 120})
def test_parallel_same_key_one_message():
a, b, chat = _chat_pair()
key = uuid.uuid4().hex
text = "parallel-idem-" + key
def send(_):
return call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=a["token"],
body={"text": text},
headers={"Idempotency-Key": key},
)["json"]
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send, range(8)))
ids = {m["id"] for m in results}
assert len(ids) == 1, ids
page = call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=100",
200,
token=b["token"],
)["json"]
assert sum(1 for m in page["items"] if m["text"] == text) == 1
def test_multiple_instance_ids_observed():
a = register("inst")
seen = set()
for _ in range(40):
seen.add(
call("GET", "/api/v1/users/me", 200, token=a["token"])["headers"].get(
"X-Instance-Id"
)
)
assert len(seen) >= 2, f"expected ≥2 replicas via LB, got {seen}"
+139
View File
@@ -0,0 +1,139 @@
"""Offline checks of the streaming PNG/JPEG validators (no server needed)."""
from __future__ import annotations
import struct
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from messenger import images # noqa: E402
import imagegen # noqa: E402
LIMIT = 40_000_000
def _validate(data: bytes, content_type: str, step: int = 0):
v = images.validator_for(content_type, LIMIT)
if step:
for i in range(0, len(data), step):
v.feed(data[i:i + step])
else:
v.feed(data)
return v.finish()
def _rejected(data: bytes, content_type: str, step: int = 0) -> int:
try:
_validate(data, content_type, step)
except images.ImageRejected as exc:
return exc.status
raise AssertionError("image was accepted")
def test_valid_pngs_any_chunking():
cases = {
"public": (imagegen.PUBLIC_PNG.read_bytes(), (320, 200)),
"rgb": (imagegen.rgb_png(33, 17), (33, 17)),
"interlaced": (imagegen.interlaced_rgb_png(37, 23), (37, 23)),
"interlaced_tiny": (imagegen.interlaced_rgb_png(1, 1), (1, 1)),
"gray1": (imagegen.gray1_png(13, 9), (13, 9)),
"palette": (imagegen.palette_png(10, 6), (10, 6)),
"rgba16": (imagegen.png16_rgba(5, 4), (5, 4)),
}
for name, (data, size) in cases.items():
for step in (0, 1, 7, 4096):
assert _validate(data, "image/png", step) == size, (name, step)
def test_valid_jpegs_any_chunking():
for name in ("sample.jpg", "progressive.jpg", "gray.jpg"):
data = imagegen.fixture(name)
for step in (0, 1, 5, 4096):
assert _validate(data, "image/jpeg", step) == (320, 200), (name, step)
def test_pixel_limit_detected_from_header_only():
data = imagegen.gray1_png(8000, 5001)
v = images.validator_for("image/png", LIMIT)
try:
v.feed(data[:33]) # signature + IHDR only
except images.ImageRejected as exc:
assert exc.status == 413
else:
raise AssertionError("pixel limit not enforced on IHDR")
jpeg = bytearray(imagegen.fixture("sample.jpg"))
sof = jpeg.find(b"\xff\xc0")
struct.pack_into(">HH", jpeg, sof + 5, 6000, 8000)
v = images.validator_for("image/jpeg", LIMIT)
try:
v.feed(bytes(jpeg[: sof + 32]))
except images.ImageRejected as exc:
assert exc.status == 413
else:
raise AssertionError("pixel limit not enforced on SOF")
def test_exactly_at_pixel_limit_accepted():
assert _validate(imagegen.gray1_png(8000, 5000), "image/png") == (8000, 5000)
def test_corrupted_png_rejected():
good = imagegen.rgb_png(20, 10)
flipped = bytearray(good)
flipped[-20] ^= 0xFF # inside IDAT → CRC mismatch
assert _rejected(bytes(flipped), "image/png") == 415
assert _rejected(good[: len(good) // 2], "image/png") == 415
assert _rejected(good + b"trailing", "image/png") == 415
ihdr = imagegen.chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 2, 0, 0, 0))
bad_zlib = b"\x89PNG\r\n\x1a\n" + ihdr + imagegen.chunk(b"IDAT", b"\x78\x9c" + b"\xff" * 40) + imagegen.chunk(b"IEND", b"")
assert _rejected(bad_zlib, "image/png") == 415
short = imagegen.png_from_raw(10, 10, b"\x00" * 10) # fewer rows than IHDR declares
assert _rejected(short, "image/png") == 415
long = imagegen.png_from_raw(2, 2, b"\x00" * 100) # decompression bomb style overflow
assert _rejected(long, "image/png") == 415
assert _rejected(imagegen.palette_png(4, 4).replace(b"PLTE", b"tEXt"), "image/png") == 415
def test_corrupted_jpeg_rejected():
good = imagegen.fixture("sample.jpg")
assert _rejected(good[: len(good) // 2], "image/jpeg") == 415
assert _rejected(b"\xff\xd8\xff\xe0" + bytes(range(256)) * 4, "image/jpeg") == 415
no_sof = b"\xff\xd8\xff\xda\x00\x08\x01\x01\x00\x00\x3f\x00" + b"\x12" * 64 + b"\xff\xd9"
assert _rejected(no_sof, "image/jpeg") == 415
assert _rejected(b"\xff\xd8\xff\xd9", "image/jpeg") == 415
arithmetic = bytearray(good)
arithmetic[good.find(b"\xff\xc0") + 1] = 0xC9
assert _rejected(bytes(arithmetic), "image/jpeg") == 415
def test_reviewer_corrupted_pngs_rejected():
for name, data in imagegen.corrupted_pngs().items():
for step in (0, 1, 4096):
assert _rejected(data, "image/png", step) == 415, (name, step)
def test_all_filter_types_accepted():
for interlace in (0, 1):
for size in ((1, 1), (9, 7), (33, 20)):
data = imagegen.all_filters_png(*size, interlace=interlace)
for step in (0, 1, 7):
assert _validate(data, "image/png", step) == size, (interlace, size, step)
def test_reviewer_corrupted_jpegs_structural_part():
# The rest (no tables, missing scan data, truncated + EOI) needs a real decoder:
# covered over HTTP by test_attachments.test_reviewer_corrupted_images_415.
corpus = imagegen.corrupted_jpegs()
for name in ("jpeg_18_bytes", "jpeg_truncated_no_eoi", "jpeg_12_bit"):
assert _rejected(corpus[name], "image/jpeg") == 415, name
def test_foreign_formats_rejected():
for blob in (b"GIF89a" + b"\x00" * 100, b"%PDF-1.7\n", b"<html></html>", b"MZ\x90\x00", b""):
assert _rejected(blob, "image/png") == 415
assert _rejected(blob, "image/jpeg") == 415
assert images.sniff(imagegen.fixture("sample.jpg")[:8]) == "image/jpeg"
assert images.sniff(imagegen.PUBLIC_PNG.read_bytes()[:8]) == "image/png"
+120
View File
@@ -0,0 +1,120 @@
"""Lab5 grade 3/4: async states, thumbnail/crop/webp, worker-stopped 409, replay safety."""
from __future__ import annotations
import hashlib
import subprocess
import time
from pathlib import Path
import imagegen
from helpers import call, chat_for, register, upload, wait_ready
ROOT = Path(__file__).resolve().parents[1]
def _jpeg_dims(data: bytes):
try:
from PIL import Image
import io
img = Image.open(io.BytesIO(data))
return img.size
except Exception:
# Minimal SOF parser
assert data[:2] == b"\xff\xd8"
i = 2
while i < len(data) - 8:
if data[i] != 0xFF:
i += 1
continue
marker = data[i + 1]
if marker in (0xC0, 0xC1, 0xC2):
height = int.from_bytes(data[i + 5:i + 7], "big")
width = int.from_bytes(data[i + 7:i + 9], "big")
return width, height
if marker == 0xD9 or marker == 0xDA:
break
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
i += 2
continue
length = int.from_bytes(data[i + 2:i + 4], "big")
i += 2 + length
raise AssertionError("JPEG SOF not found")
def test_queued_linkable_then_ready_variants():
alice, bob = register("q_a"), register("q_b")
chat = chat_for(alice, bob, title="lab5")
data = imagegen.PUBLIC_PNG.read_bytes() # 320x200
att = upload(alice["token"], chat["id"], data, "sample.png", "image/png", 202)["json"]
assert att["status"] == "queued"
call("GET", f"/api/v1/attachments/{att['id']}/content", 409, token=bob["token"])
msg = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]},
)["json"]
assert msg["attachment_ids"] == [att["id"]]
ready = wait_ready(bob["token"], att)
names = {v["name"]: v for v in ready["variants"]}
assert set(names) >= {"thumbnail", "crop", "optimized"}
thumb = call(
"GET",
f"/api/v1/attachments/{att['id']}/content?variant=thumbnail",
200,
token=bob["token"],
)
assert thumb["headers"].get_content_type() == "image/jpeg"
assert _jpeg_dims(thumb["bytes"]) == (256, 160)
assert hashlib.sha256(thumb["bytes"]).hexdigest() == names["thumbnail"]["sha256"]
crop = call(
"GET",
f"/api/v1/attachments/{att['id']}/content?variant=crop",
200,
token=bob["token"],
)
assert crop["headers"].get_content_type() == "image/jpeg"
assert _jpeg_dims(crop["bytes"]) == (128, 128)
opt = call(
"GET",
f"/api/v1/attachments/{att['id']}/content?variant=optimized",
200,
token=bob["token"],
)
assert opt["headers"].get_content_type() == "image/webp"
assert opt["bytes"][:4] == b"RIFF"
def test_worker_stopped_upload_stays_queued():
import os
if os.environ.get("RUN_DOCKER_TESTS") != "1":
print("SKIP test_worker_stopped_upload_stays_queued (set RUN_DOCKER_TESTS=1)")
return
stop = ROOT / "scripts" / ("stop-worker.cmd" if os.name == "nt" else "stop-worker.sh")
start = ROOT / "scripts" / ("start-worker.cmd" if os.name == "nt" else "start-worker.sh")
subprocess.run([str(stop)], check=True, cwd=str(ROOT))
try:
alice = register("wstop")
chat = chat_for(alice, title="worker-stop")
data = imagegen.rgb_png(64, 48)
att = upload(alice["token"], chat["id"], data, "park.png", "image/png", 202)["json"]
assert att["status"] == "queued"
# Text path still works while worker is down.
call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
201,
token=alice["token"],
body={"text": "still-alive"},
)
call("GET", f"/api/v1/attachments/{att['id']}/content", 409, token=alice["token"])
time.sleep(2)
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"])["json"]
assert meta["status"] in ("queued", "processing")
finally:
subprocess.run([str(start)], check=True, cwd=str(ROOT))
ready = wait_ready(alice["token"], att, wait_seconds=90)
assert ready["status"] == "ready"
+58
View File
@@ -0,0 +1,58 @@
"""Targeted checks for lab5 reviewer remarks (415, thumbnail floor, keys)."""
from __future__ import annotations
import hashlib
import io
import imagegen
from helpers import call, chat_for, register, upload, wait_ready
def _jpeg_dims(data: bytes):
from PIL import Image
img = Image.open(io.BytesIO(data))
return img.size
def test_corrupted_and_truncated_rejected_415():
alice = register("rem_a")
chat = chat_for(alice)
for name, data in imagegen.corrupted_pngs().items():
resp = upload(alice["token"], chat["id"], data, "bad.png", "image/png", 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type", (name, resp["json"])
for name, data in imagegen.corrupted_jpegs().items():
resp = upload(alice["token"], chat["id"], data, "bad.jpg", "image/jpeg", 415)
assert resp["json"]["error"]["code"] == "unsupported_media_type", (name, resp["json"])
def test_thumbnail_319x200_is_256x160():
alice, bob = register("th_a"), register("th_b")
chat = chat_for(alice, bob)
data = imagegen.rgb_png(319, 200)
att = upload(alice["token"], chat["id"], data, "odd.png", "image/png", 202)["json"]
ready = wait_ready(bob["token"], att)
thumbs = [v for v in ready["variants"] if v["name"] == "thumbnail"]
assert len(thumbs) == 1
assert (thumbs[0]["width"], thumbs[0]["height"]) == (256, 160), thumbs[0]
body = call(
"GET",
f"/api/v1/attachments/{att['id']}/content?variant=thumbnail",
200,
token=bob["token"],
)["bytes"]
assert _jpeg_dims(body) == (256, 160)
assert hashlib.sha256(body).hexdigest() == thumbs[0]["sha256"]
def test_jpeg_ready_download_uses_stored_key():
alice = register("jpg_a")
chat = chat_for(alice)
data = imagegen.fixture("sample.jpg")
att = upload(alice["token"], chat["id"], data, "s.jpg", "image/jpeg", 202)["json"]
ready = wait_ready(alice["token"], att)
assert ready["status"] == "ready"
dl = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
assert dl["bytes"] == data
assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"]
+34
View File
@@ -0,0 +1,34 @@
"""Session expiry (short TTL) — run against stack with SESSION_TTL_SECONDS=2."""
from __future__ import annotations
import base64
import os
import time
from helpers import call, register
def test_session_expiry_if_short_ttl():
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
if ttl > 5:
print("SKIP expiry: SESSION_TTL_SECONDS=%s (need <=5 for this test)" % ttl)
return
a = register("expire")
time.sleep(ttl + 1.5)
call("GET", "/api/v1/users/me", 401, token=a["token"])
def test_login_after_register():
a = register("relogin")
cred = base64.b64encode(
(a["registration"]["username"] + ":" + a["registration"]["password"]).encode()
).decode()
session = call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + cred},
)["json"]
assert session["token_type"] == "Bearer"
assert session["user"]["id"] == a["id"]
+249
View File
@@ -0,0 +1,249 @@
"""Graceful shutdown of an API replica (RUN_DOCKER_TESTS=1).
Each test stops ONE API replica and brings it back; dependencies are not touched.
"""
from __future__ import annotations
import hashlib
import json
import os
import subprocess
import threading
import time
import uuid
from datetime import datetime
import imagegen
from helpers import wait_ready, call, chat_for, register
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
STOP_TIMEOUT_S = 15 # = stop_grace_period of the api service
# Runs inside the replica: SIGTERM to PID 1 (as `docker stop` does), then probes the
# same process directly while HAProxy is still expected to route to it.
PROBE = r"""
import json, os, signal, time, urllib.error, urllib.request
def get(path, token=None):
req = urllib.request.Request("http://127.0.0.1:8080" + path)
if token:
req.add_header("Authorization", "Bearer " + token)
try:
with urllib.request.urlopen(req, timeout=2) as resp:
return resp.status
except urllib.error.HTTPError as err:
return err.code
except Exception as exc:
return type(exc).__name__
token = os.environ["DRAIN_TOKEN"]
drain = float(os.environ.get("SHUTDOWN_DRAIN_SECONDS", "3"))
with open("/proc/1/cmdline", "rb") as f:
pid1 = f.read().replace(b"\0", b" ").decode().strip()
before = [get("/health/ready"), get("/api/v1/users/me", token)]
sent_at = time.time()
t0 = time.monotonic()
os.kill(1, signal.SIGTERM)
samples = []
while time.monotonic() - t0 < drain - 0.8:
samples.append([round(time.monotonic() - t0, 2), get("/health/ready"), get("/api/v1/users/me", token)])
time.sleep(0.1)
print(json.dumps({"pid1": pid1, "drain": drain, "before": before, "sent_at": sent_at, "samples": samples}))
"""
# Runs inside the replica: a slow upload (body from stdin) whose body is still being sent
# when the drain window ends and the listener is closed.
SLOW_UPLOAD = r"""
import http.client, json, os, signal, sys, time
body = sys.stdin.buffer.read()
drain = float(os.environ.get("SHUTDOWN_DRAIN_SECONDS", "3"))
conn = http.client.HTTPConnection("127.0.0.1", 8080, timeout=30)
conn.putrequest("POST", "/api/v1/chats/%s/attachments?filename=drain.png" % os.environ["DRAIN_CHAT"])
conn.putheader("Authorization", "Bearer " + os.environ["DRAIN_TOKEN"])
conn.putheader("Content-Type", "image/png")
conn.putheader("Content-Length", str(len(body)))
conn.endheaders()
t0 = time.monotonic()
step = len(body) // 40 + 1
for i in range(0, len(body), step):
conn.send(body[i:i + step])
if i == 0:
os.kill(1, signal.SIGTERM)
time.sleep((drain + 2) / 40)
resp = conn.getresponse()
print(json.dumps({"status": resp.status, "body_sent_s": round(time.monotonic() - t0, 2),
"json": json.loads(resp.read() or b"null")}))
"""
def _enabled(name: str) -> bool:
if os.environ.get("RUN_DOCKER_TESTS") != "1":
print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)")
return False
return True
def _run(args, check=True):
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}")
return proc
def _api_containers():
out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split()
assert len(out) >= 2, out
return out
def _inspect(container: str, fmt: str) -> str:
return _run(["docker", "inspect", "-f", fmt, container]).stdout.strip()
def _docker_time(value: str) -> float:
# 2026-11-06T15:37:12.123456789Z → POSIX seconds (nanoseconds cut to microseconds).
head, _, frac = value.rstrip("Z").partition(".")
stamp = datetime.fromisoformat(head + "+00:00").timestamp()
return stamp + (float("0." + frac[:6]) if frac else 0.0)
def _wait_in_rotation(host: str, timeout_s: int = 90) -> float:
started = time.monotonic()
while time.monotonic() - started < timeout_s:
try:
if call("GET", "/health/ready")["headers"].get("X-Instance-Id") == host:
return time.monotonic() - started
except Exception:
pass
time.sleep(0.5)
raise AssertionError(f"replica {host} not back in LB rotation after {timeout_s}s")
def _wait_restarted(container: str, restarts: int) -> None:
# unless-stopped restarts the container after a SIGTERM sent from inside it.
deadline = time.monotonic() + 60
while int(_inspect(container, "{{.RestartCount}}")) == restarts:
assert time.monotonic() < deadline, "replica did not exit after SIGTERM"
time.sleep(0.5)
def test_sigterm_drains_readiness_but_keeps_serving():
# After SIGTERM the replica must report /health/ready 503 at once (HAProxy takes it
# out after `inter 1s fall 2`) but keep answering normal requests until then.
if not _enabled("test_sigterm_drains_readiness_but_keeps_serving"):
return
actor = register("drain")
target = _api_containers()[0]
host = _inspect(target, "{{.Config.Hostname}}")
restarts = int(_inspect(target, "{{.RestartCount}}"))
proc = subprocess.run(["docker", "exec", "-e", "DRAIN_TOKEN", target, "python", "-c", PROBE],
cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
env=dict(os.environ, DRAIN_TOKEN=actor["token"]))
if proc.returncode != 0:
_wait_in_rotation(host)
raise AssertionError(f"replica exited during the drain window (probe rc={proc.returncode}) {proc.stderr[-300:]}")
probe = json.loads(proc.stdout[proc.stdout.index("{"):])
samples = probe["samples"]
assert probe["pid1"].startswith("python -m messenger"), probe["pid1"]
assert probe["drain"] >= 2, f"drain window {probe['drain']}s < HAProxy inter 1s x fall 2"
assert probe["before"] == [200, 200], probe["before"]
assert len(samples) >= 5, samples
not_served = [s for s in samples if s[2] != 200]
assert not not_served, f"requests failed during the drain window: {not_served}"
still_ready = [s for s in samples if s[0] >= 0.3 and s[1] != 503]
assert not still_ready, f"/health/ready not 503 while draining: {still_ready}"
_wait_restarted(target, restarts)
# FinishedAt is the exit of the drained process (same clock as the probe in the container).
stop_s = _docker_time(_inspect(target, "{{.State.FinishedAt}}")) - probe["sent_at"]
assert probe["drain"] - 0.2 <= stop_s < STOP_TIMEOUT_S, stop_s
back_s = _wait_in_rotation(host)
print(" drain: %d probes, ready 503 / users/me 200; exit %.2fs after SIGTERM; back in LB in %.1fs"
% (len(samples), stop_s, back_s))
def test_inflight_upload_finishes_after_listener_closed():
# The body is still arriving when the drain window ends: the listener closes, but the
# in-flight upload (spool → S3 PUT → DB commit) must complete before the process exits.
if not _enabled("test_inflight_upload_finishes_after_listener_closed"):
return
alice, bob = register("drain_up_a"), register("drain_up_b")
chat = chat_for(alice, bob, title="drain upload")
data = imagegen.large_png(2 * 1024 * 1024)
target = _api_containers()[0]
host = _inspect(target, "{{.Config.Hostname}}")
restarts = int(_inspect(target, "{{.RestartCount}}"))
proc = subprocess.run(["docker", "exec", "-i", "-e", "DRAIN_TOKEN", "-e", "DRAIN_CHAT", target,
"python", "-c", SLOW_UPLOAD],
cwd=ROOT, input=data, capture_output=True,
env=dict(os.environ, DRAIN_TOKEN=alice["token"], DRAIN_CHAT=chat["id"]))
out = proc.stdout.decode("utf-8", "replace")
_wait_restarted(target, restarts)
back_s = _wait_in_rotation(host)
assert proc.returncode == 0, f"upload probe rc={proc.returncode}: {proc.stderr[-300:]!r}"
result = json.loads(out[out.index("{"):])
assert result["status"] == 202, result
att = result["json"]
assert att["sha256"] == hashlib.sha256(data).hexdigest(), att
dl = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"]
print(" in-flight upload: body sent over %.1fs across listener close -> 202, download via LB OK; back in LB in %.1fs"
% (result["body_sent_s"], back_s))
def test_replica_stop_under_load_has_no_errors():
# Closed-loop POSTs through the LB while one replica gets `docker stop`: requests
# that HAProxy still sends to it during the drain window must succeed.
if not _enabled("test_replica_stop_under_load_has_no_errors"):
return
alice, bob = register("drain_load_a"), register("drain_load_b")
chat = chat_for(alice, bob, title="drain")
target = _api_containers()[0]
host = _inspect(target, "{{.Config.Hostname}}")
stats = {"ok": 0, "target_after_stop": 0, "errors": []}
lock = threading.Lock()
stop = threading.Event()
stopping = threading.Event()
def worker():
while not stop.is_set():
key = uuid.uuid4().hex
try:
r = call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "drain-" + key}, headers={"Idempotency-Key": key})
status, instance = r["status"], r["headers"].get("X-Instance-Id")
except Exception as exc:
status, instance = type(exc).__name__, None
with lock:
if status == 201:
stats["ok"] += 1
if stopping.is_set() and instance == host:
stats["target_after_stop"] += 1
else:
stats["errors"].append(status)
time.sleep(0.05)
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
for th in threads:
th.start()
try:
time.sleep(1.5)
stopping.set()
started = time.monotonic()
_run(["docker", "stop", "-t", str(STOP_TIMEOUT_S), target])
stop_s = time.monotonic() - started
time.sleep(1.5)
finally:
stop.set()
for th in threads:
th.join(timeout=10)
_run(["docker", "start", target])
back_s = _wait_in_rotation(host)
assert not stats["errors"], f"{len(stats['errors'])} errors during replica stop: {stats['errors'][:10]}"
assert stats["ok"] >= 20, stats
assert stats["target_after_stop"] >= 1, "stopped replica served nothing during its drain window"
assert stop_s < STOP_TIMEOUT_S, stop_s
print(" stop under load: %d x 201, 0 errors, %d served by stopping replica; docker stop %.2fs; back in LB in %.1fs"
% (stats["ok"], stats["target_after_stop"], stop_s, back_s))